def test_notif_agent(notif_agent): from lib.core.state import State notif_agents = State.get_notif_agents() modules = State.get_notif_agent_modules() log.debug_print(f"Testing notification agent: '{notif_agent.name}'") module = modules[notif_agent.module] module.send(title=f"Testing '{notif_agent.name}'", message="This is a test message", **notif_agent.module_properties) print("Done!")
def create_notif_agent_choose_module(default=None): from lib.core.state import State modules = State.get_notif_agent_modules() default_str = "" if default is None and len(modules) == 1: default = list(modules)[0] if default is not None: default_str = f" [{default}]" while True: modules_list = [] i = 0 for m in modules: modules_list.append(m) print(f"{i} - {m}") i = i + 1 choices = "0" if len(modules_list) > 1: choices = f"0-{len(modules_list) - 1}" module_index_str = input( f"Module [Choose from {choices}]:{default_str} ") if default is not None and module_index_str == "": return default if len(modules_list) == 0 and module_index_str == "": module_index = 0 break if re.match("[0-9]+$", module_index_str): module_index = int(module_index_str) if module_index >= 0 and module_index < len(modules_list): return modules_list[module_index]
def notif_agent_creator(notif_agent): from lib.core.state import State n = notif_agent cur_notif_agents = State.get_notif_agents() modules = State.get_notif_agent_modules() while True: n.name = creator.prompt_string("Name", default=n.name) n.module = create_notif_agent_choose_module(n.module) module = modules[n.module] props = module.get_properties() print("Module Properties") for p in props: default = None if n.module_properties is not None: default = n.module_properties.get(p, None) else: n.module_properties = {} while True: n.module_properties[p] = creator.prompt_string(f"{p}", default=default) is_valid, error_msg = module.is_property_valid( p, n.module_properties[p]) if is_valid: break else: print(error_msg) default = None print() print(f"Id: {n.id}") print(f"Name: {n.name}") print(f"Module: {n.module}") print("-----------------------------") for p in props: print(f"{p}: {n.module_properties[p]}") print("-----------------------------") while True: confirm = creator.prompt_options("Choose an option", ["save", "edit", "test", "quit"]) if confirm == "test": test_notif_agent(notif_agent) continue else: break if confirm == "save": break elif confirm == "edit": continue elif confirm == "quit": return cur_notif_agents[n.id] = notif_agent State.save_notif_agents()
def scrape(source, notif_agents_list, include=[], exclude=[], colour_flag="", notify=True, force_tasks=False, force_agents=False, recent_ads=0, save_ads=True, ignore_old_ads=False): from lib.core.state import State import lib.core.notif_agent as notif_agent ads = State.get_ads() source_modules = State.get_source_modules() notif_agent_modules = State.get_notif_agent_modules() log.info_print(f"Source: {source.name}") log.info_print(f"Module: {source.module}") log.info_print(f"Module Properties: {source.module_properties}") if len(include): print(f"Including: {include}") if len(exclude): print(f"Excluding: {exclude}") module = source_modules[source.module] old_ads = [] if ignore_old_ads == False: if source.module in ads: old_ads = ads[source.module] log.debug(f"Total old ads: {len(old_ads)}") else: log.debug(f"No old ads found for module: {source.module}") else: log.info_print("Ignoring old ads...") new_ads, ad_title = module.scrape_for_ads(old_ads, exclude=exclude, **source.module_properties) info_string = f"Found {len(new_ads)} new ads" \ if len(new_ads) != 1 else "Found 1 new ad" log.info_print(info_string) num_ads = len(new_ads) if notify and num_ads: ads_to_send = new_ads if recent_ads > 0: # only notify the most recent notify_recent new_ads ads_to_send = ct.get_most_recent_items(recent_ads, new_ads) log.debug( f"Recent ads set to: {recent_ads} got: {len(ads_to_send)}") log.info_print(f"Total ads to notify about: {len(ads_to_send)}") if len(notif_agents_list) == 0: log.warning_print( "No notification agents set... nothing to notify") else: if len(notif_agents_list) > 1: log.info_print( f"Notifying agents: {notif_agent.get_names(notif_agents_list)}" ) for agent in notif_agents_list: if agent.enabled or force_agents == True: if agent.enabled == False and force_agents == True: log.info_print( "Notification agent was disabled but forcing...") notif_agent_modules[agent.module].send_ads( ads_to_send, ad_title, colour_flag, **agent.module_properties) else: log.info_print( f"Skipping... Notification agent disabled: {agent.name}" ) elif not notify and num_ads: log.info_print("Skipping notification") if save_ads: ads[source.module] = module.old_ad_ids log.debug(f"Total all-time processed ads: {len(module.old_ad_ids)}") else: log.info_print(f"Saving ads disabled. Skipping...") print() return ScrapeSummary(new_ads=new_ads, latest_ads=list(new_ads)[-3:], total_new_ads=len(new_ads))