def validate(task, sources, notif_agents, stay_alive=True): for s in task.source_ids: if not s in sources: if stay_alive: task.source_ids.remove(s) log.warning_print( f"Source not found: '{s}'. Removing from task '{task.id} - {task.name}'" ) else: raise ValueError( f"Error validating task '{task.id} - {task.name}': Source '{s}' not found" ) for n in task.notif_agent_ids: if not n in notif_agents: if stay_alive: task.notif_agent_ids.remove(s) log.warning_print( f"Notification agent not found: '{s}'. Removing from task '{task.id} - {task.name}'" ) else: raise ValueError( f"Error validating task '{task.id} - {task.name}': Notification Agent '{s}' not found" )
def notif_agents_enabled_check(notif_agents): if len(get_enabled(notif_agents)) == 0: log.warning_print( "There are no enabled agents... no notifications will be sent")
def scrape(source, notif_agents_list, include=[], exclude=[], colour_flag="", notify=True, force_tasks=False, force_agents=False, recent_ads=0, save_ads=True, ignore_old_ads=False): from lib.core.state import State import lib.core.notif_agent as notif_agent ads = State.get_ads() source_modules = State.get_source_modules() notif_agent_modules = State.get_notif_agent_modules() log.info_print(f"Source: {source.name}") log.info_print(f"Module: {source.module}") log.info_print(f"Module Properties: {source.module_properties}") if len(include): print(f"Including: {include}") if len(exclude): print(f"Excluding: {exclude}") module = source_modules[source.module] old_ads = [] if ignore_old_ads == False: if source.module in ads: old_ads = ads[source.module] log.debug(f"Total old ads: {len(old_ads)}") else: log.debug(f"No old ads found for module: {source.module}") else: log.info_print("Ignoring old ads...") new_ads, ad_title = module.scrape_for_ads(old_ads, exclude=exclude, **source.module_properties) info_string = f"Found {len(new_ads)} new ads" \ if len(new_ads) != 1 else "Found 1 new ad" log.info_print(info_string) num_ads = len(new_ads) if notify and num_ads: ads_to_send = new_ads if recent_ads > 0: # only notify the most recent notify_recent new_ads ads_to_send = ct.get_most_recent_items(recent_ads, new_ads) log.debug( f"Recent ads set to: {recent_ads} got: {len(ads_to_send)}") log.info_print(f"Total ads to notify about: {len(ads_to_send)}") if len(notif_agents_list) == 0: log.warning_print( "No notification agents set... nothing to notify") else: if len(notif_agents_list) > 1: log.info_print( f"Notifying agents: {notif_agent.get_names(notif_agents_list)}" ) for agent in notif_agents_list: if agent.enabled or force_agents == True: if agent.enabled == False and force_agents == True: log.info_print( "Notification agent was disabled but forcing...") notif_agent_modules[agent.module].send_ads( ads_to_send, ad_title, colour_flag, **agent.module_properties) else: log.info_print( f"Skipping... Notification agent disabled: {agent.name}" ) elif not notify and num_ads: log.info_print("Skipping notification") if save_ads: ads[source.module] = module.old_ad_ids log.debug(f"Total all-time processed ads: {len(module.old_ad_ids)}") else: log.info_print(f"Saving ads disabled. Skipping...") print() return ScrapeSummary(new_ads=new_ads, latest_ads=list(new_ads)[-3:], total_new_ads=len(new_ads))