def source_creator(source): from lib.core.state import State s = source sources = State.get_sources() modules = State.get_source_modules() while True: name = s.name s.name = creator.prompt_string("Name", default=name) s.module = create_source_choose_module(s.module) module = modules[s.module] props = module.get_properties() print("Module Properties") for p in props: default = None if s.module_properties is not None: default = s.module_properties.get(p, None) if s.module_properties is None: s.module_properties = {} s.module_properties[p] = creator.prompt_string(f"{p}", default=default) print() print(f"Id: {s.id}") print(f"Name: {s.name}") print(f"Module: {s.module}") print("-----------------------------") for p in s.module_properties: print(f"{p}: {s.module_properties[p]}") print("-----------------------------") while True: confirm = creator.prompt_options("Choose an option", ["save", "edit", "test", "quit"]) if confirm == "test": test_source(source, modules) continue else: break if confirm == "save": break elif confirm == "edit": continue elif confirm == "quit": return sources[source.id] = source save()
def test_webui_source(source): from lib.core.state import State modules = State.get_source_modules() module = modules[source.module] return scrape(source, None, include="", exclude="", notify=False, save_ads=False)
def edit_source(): from lib.core.state import State sources = State.get_sources() modules = State.get_source_modules() creator.print_title("Edit Source") source = creator.prompt_complex_dict("Choose a source", sources, "name", extra_options=["d"], extra_options_desc=["done"]) if source == "d": return else: source_creator(source)
def test_source(source): from lib.core.state import State modules = State.get_source_modules() include = [] exclude = [] if creator.yes_no("Would you like to add inlcudes/excludes", "n") == "y": include = creator.prompt_string("Include", allow_empty=True) exclude = creator.prompt_string("Exclude", allow_empty=True) module = modules[source.module] return scrape(source, None, include=include, exclude=exclude, notify=False, save_ads=False)
def create_source_choose_module(default=None): from lib.core.state import State modules = State.get_source_modules() default_str = "" if default is None and len(modules) == 1: default = list(modules)[0] if default is not None: default_str = f" [{default}]" while True: modules_list = [] i = 0 for s in modules: modules_list.append(s) print(f"{i} - {s}") i = i + 1 choices = "0" if len(modules_list) > 1: choices = f"0-{len(modules_list) - 1}" module_index_str = input( f"Module [Choose from {choices}]:{default_str} ") if default is not None and module_index_str == "": return default if len(modules_list) == 0 and module_index_str == "": module_index = 0 break if re.match("[0-9]+$", module_index_str): module_index = int(module_index_str) if module_index >= 0 and module_index < len(modules_list): return modules_list[module_index]
def scrape(source, notif_agents_list, include=[], exclude=[], colour_flag="", notify=True, force_tasks=False, force_agents=False, recent_ads=0, save_ads=True, ignore_old_ads=False): from lib.core.state import State import lib.core.notif_agent as notif_agent ads = State.get_ads() source_modules = State.get_source_modules() notif_agent_modules = State.get_notif_agent_modules() log.info_print(f"Source: {source.name}") log.info_print(f"Module: {source.module}") log.info_print(f"Module Properties: {source.module_properties}") if len(include): print(f"Including: {include}") if len(exclude): print(f"Excluding: {exclude}") module = source_modules[source.module] old_ads = [] if ignore_old_ads == False: if source.module in ads: old_ads = ads[source.module] log.debug(f"Total old ads: {len(old_ads)}") else: log.debug(f"No old ads found for module: {source.module}") else: log.info_print("Ignoring old ads...") new_ads, ad_title = module.scrape_for_ads(old_ads, exclude=exclude, **source.module_properties) info_string = f"Found {len(new_ads)} new ads" \ if len(new_ads) != 1 else "Found 1 new ad" log.info_print(info_string) num_ads = len(new_ads) if notify and num_ads: ads_to_send = new_ads if recent_ads > 0: # only notify the most recent notify_recent new_ads ads_to_send = ct.get_most_recent_items(recent_ads, new_ads) log.debug( f"Recent ads set to: {recent_ads} got: {len(ads_to_send)}") log.info_print(f"Total ads to notify about: {len(ads_to_send)}") if len(notif_agents_list) == 0: log.warning_print( "No notification agents set... nothing to notify") else: if len(notif_agents_list) > 1: log.info_print( f"Notifying agents: {notif_agent.get_names(notif_agents_list)}" ) for agent in notif_agents_list: if agent.enabled or force_agents == True: if agent.enabled == False and force_agents == True: log.info_print( "Notification agent was disabled but forcing...") notif_agent_modules[agent.module].send_ads( ads_to_send, ad_title, colour_flag, **agent.module_properties) else: log.info_print( f"Skipping... Notification agent disabled: {agent.name}" ) elif not notify and num_ads: log.info_print("Skipping notification") if save_ads: ads[source.module] = module.old_ad_ids log.debug(f"Total all-time processed ads: {len(module.old_ad_ids)}") else: log.info_print(f"Saving ads disabled. Skipping...") print() return ScrapeSummary(new_ads=new_ads, latest_ads=list(new_ads)[-3:], total_new_ads=len(new_ads))