def save():
from lib.core.state import State
ads = State.get_ads()
file = Config.ADS_FILE
with open(file, "w") as stream:
json.dump(ads, stream)
def scrape(source,
notif_agents_list,
include=[],
exclude=[],
colour_flag="",
notify=True,
force_tasks=False,
force_agents=False,
recent_ads=0,
save_ads=True,
ignore_old_ads=False):
from lib.core.state import State
import lib.core.notif_agent as notif_agent
ads = State.get_ads()
source_modules = State.get_source_modules()
notif_agent_modules = State.get_notif_agent_modules()
log.info_print(f"Source: {source.name}")
log.info_print(f"Module: {source.module}")
log.info_print(f"Module Properties: {source.module_properties}")
if len(include):
print(f"Including: {include}")
if len(exclude):
print(f"Excluding: {exclude}")
module = source_modules[source.module]
old_ads = []
if ignore_old_ads == False:
if source.module in ads:
old_ads = ads[source.module]
log.debug(f"Total old ads: {len(old_ads)}")
else:
log.debug(f"No old ads found for module: {source.module}")
else:
log.info_print("Ignoring old ads...")
new_ads, ad_title = module.scrape_for_ads(old_ads,
exclude=exclude,
**source.module_properties)
info_string = f"Found {len(new_ads)} new ads" \
if len(new_ads) != 1 else "Found 1 new ad"
log.info_print(info_string)
num_ads = len(new_ads)
if notify and num_ads:
ads_to_send = new_ads
if recent_ads > 0:
# only notify the most recent notify_recent new_ads
ads_to_send = ct.get_most_recent_items(recent_ads, new_ads)
log.debug(
f"Recent ads set to: {recent_ads} got: {len(ads_to_send)}")
log.info_print(f"Total ads to notify about: {len(ads_to_send)}")
if len(notif_agents_list) == 0:
log.warning_print(
"No notification agents set... nothing to notify")
else:
if len(notif_agents_list) > 1:
log.info_print(
f"Notifying agents: {notif_agent.get_names(notif_agents_list)}"
)
for agent in notif_agents_list:
if agent.enabled or force_agents == True:
if agent.enabled == False and force_agents == True:
log.info_print(
"Notification agent was disabled but forcing...")
notif_agent_modules[agent.module].send_ads(
ads_to_send, ad_title, colour_flag,
**agent.module_properties)
else:
log.info_print(
f"Skipping... Notification agent disabled: {agent.name}"
)
elif not notify and num_ads:
log.info_print("Skipping notification")
if save_ads:
ads[source.module] = module.old_ad_ids
log.debug(f"Total all-time processed ads: {len(module.old_ad_ids)}")
else:
log.info_print(f"Saving ads disabled. Skipping...")
print()
return ScrapeSummary(new_ads=new_ads,
latest_ads=list(new_ads)[-3:],
total_new_ads=len(new_ads))
