def source_creator(source):
from lib.core.state import State
s = source
sources = State.get_sources()
modules = State.get_source_modules()
while True:
name = s.name
s.name = creator.prompt_string("Name", default=name)
s.module = create_source_choose_module(s.module)
module = modules[s.module]
props = module.get_properties()
print("Module Properties")
for p in props:
default = None
if s.module_properties is not None:
default = s.module_properties.get(p, None)
if s.module_properties is None:
s.module_properties = {}
s.module_properties[p] = creator.prompt_string(f"{p}",
default=default)
print()
print(f"Id: {s.id}")
print(f"Name: {s.name}")
print(f"Module: {s.module}")
print("-----------------------------")
for p in s.module_properties:
print(f"{p}: {s.module_properties[p]}")
print("-----------------------------")
while True:
confirm = creator.prompt_options("Choose an option",
["save", "edit", "test", "quit"])
if confirm == "test":
test_source(source, modules)
continue
else:
break
if confirm == "save":
break
elif confirm == "edit":
continue
elif confirm == "quit":
return
sources[source.id] = source
save()
def test_webui_source(source):
from lib.core.state import State
modules = State.get_source_modules()
module = modules[source.module]
return scrape(source,
None,
include="",
exclude="",
notify=False,
save_ads=False)
def edit_source():
from lib.core.state import State
sources = State.get_sources()
modules = State.get_source_modules()
creator.print_title("Edit Source")
source = creator.prompt_complex_dict("Choose a source",
sources,
"name",
extra_options=["d"],
extra_options_desc=["done"])
if source == "d":
return
else:
source_creator(source)
def test_source(source):
from lib.core.state import State
modules = State.get_source_modules()
include = []
exclude = []
if creator.yes_no("Would you like to add inlcudes/excludes", "n") == "y":
include = creator.prompt_string("Include", allow_empty=True)
exclude = creator.prompt_string("Exclude", allow_empty=True)
module = modules[source.module]
return scrape(source,
None,
include=include,
exclude=exclude,
notify=False,
save_ads=False)
def create_source_choose_module(default=None):
from lib.core.state import State
modules = State.get_source_modules()
default_str = ""
if default is None and len(modules) == 1:
default = list(modules)[0]
if default is not None:
default_str = f" [{default}]"
while True:
modules_list = []
i = 0
for s in modules:
modules_list.append(s)
print(f"{i} - {s}")
i = i + 1
choices = "0"
if len(modules_list) > 1:
choices = f"0-{len(modules_list) - 1}"
module_index_str = input(
f"Module [Choose from {choices}]:{default_str} ")
if default is not None and module_index_str == "":
return default
if len(modules_list) == 0 and module_index_str == "":
module_index = 0
break
if re.match("[0-9]+$", module_index_str):
module_index = int(module_index_str)
if module_index >= 0 and module_index < len(modules_list):
return modules_list[module_index]
def scrape(source,
notif_agents_list,
include=[],
exclude=[],
colour_flag="",
notify=True,
force_tasks=False,
force_agents=False,
recent_ads=0,
save_ads=True,
ignore_old_ads=False):
from lib.core.state import State
import lib.core.notif_agent as notif_agent
ads = State.get_ads()
source_modules = State.get_source_modules()
notif_agent_modules = State.get_notif_agent_modules()
log.info_print(f"Source: {source.name}")
log.info_print(f"Module: {source.module}")
log.info_print(f"Module Properties: {source.module_properties}")
if len(include):
print(f"Including: {include}")
if len(exclude):
print(f"Excluding: {exclude}")
module = source_modules[source.module]
old_ads = []
if ignore_old_ads == False:
if source.module in ads:
old_ads = ads[source.module]
log.debug(f"Total old ads: {len(old_ads)}")
else:
log.debug(f"No old ads found for module: {source.module}")
else:
log.info_print("Ignoring old ads...")
new_ads, ad_title = module.scrape_for_ads(old_ads,
exclude=exclude,
**source.module_properties)
info_string = f"Found {len(new_ads)} new ads" \
if len(new_ads) != 1 else "Found 1 new ad"
log.info_print(info_string)
num_ads = len(new_ads)
if notify and num_ads:
ads_to_send = new_ads
if recent_ads > 0:
# only notify the most recent notify_recent new_ads
ads_to_send = ct.get_most_recent_items(recent_ads, new_ads)
log.debug(
f"Recent ads set to: {recent_ads} got: {len(ads_to_send)}")
log.info_print(f"Total ads to notify about: {len(ads_to_send)}")
if len(notif_agents_list) == 0:
log.warning_print(
"No notification agents set... nothing to notify")
else:
if len(notif_agents_list) > 1:
log.info_print(
f"Notifying agents: {notif_agent.get_names(notif_agents_list)}"
)
for agent in notif_agents_list:
if agent.enabled or force_agents == True:
if agent.enabled == False and force_agents == True:
log.info_print(
"Notification agent was disabled but forcing...")
notif_agent_modules[agent.module].send_ads(
ads_to_send, ad_title, colour_flag,
**agent.module_properties)
else:
log.info_print(
f"Skipping... Notification agent disabled: {agent.name}"
)
elif not notify and num_ads:
log.info_print("Skipping notification")
if save_ads:
ads[source.module] = module.old_ad_ids
log.debug(f"Total all-time processed ads: {len(module.old_ad_ids)}")
else:
log.info_print(f"Saving ads disabled. Skipping...")
print()
return ScrapeSummary(new_ads=new_ads,
latest_ads=list(new_ads)[-3:],
total_new_ads=len(new_ads))
