def test_notif_agent(notif_agent):
from lib.core.state import State
notif_agents = State.get_notif_agents()
modules = State.get_notif_agent_modules()
log.debug_print(f"Testing notification agent: '{notif_agent.name}'")
module = modules[notif_agent.module]
module.send(title=f"Testing '{notif_agent.name}'",
message="This is a test message",
**notif_agent.module_properties)
print("Done!")
def create_notif_agent_choose_module(default=None):
from lib.core.state import State
modules = State.get_notif_agent_modules()
default_str = ""
if default is None and len(modules) == 1:
default = list(modules)[0]
if default is not None:
default_str = f" [{default}]"
while True:
modules_list = []
i = 0
for m in modules:
modules_list.append(m)
print(f"{i} - {m}")
i = i + 1
choices = "0"
if len(modules_list) > 1:
choices = f"0-{len(modules_list) - 1}"
module_index_str = input(
f"Module [Choose from {choices}]:{default_str} ")
if default is not None and module_index_str == "":
return default
if len(modules_list) == 0 and module_index_str == "":
module_index = 0
break
if re.match("[0-9]+$", module_index_str):
module_index = int(module_index_str)
if module_index >= 0 and module_index < len(modules_list):
return modules_list[module_index]
def notif_agent_creator(notif_agent):
from lib.core.state import State
n = notif_agent
cur_notif_agents = State.get_notif_agents()
modules = State.get_notif_agent_modules()
while True:
n.name = creator.prompt_string("Name", default=n.name)
n.module = create_notif_agent_choose_module(n.module)
module = modules[n.module]
props = module.get_properties()
print("Module Properties")
for p in props:
default = None
if n.module_properties is not None:
default = n.module_properties.get(p, None)
else:
n.module_properties = {}
while True:
n.module_properties[p] = creator.prompt_string(f"{p}",
default=default)
is_valid, error_msg = module.is_property_valid(
p, n.module_properties[p])
if is_valid:
break
else:
print(error_msg)
default = None
print()
print(f"Id: {n.id}")
print(f"Name: {n.name}")
print(f"Module: {n.module}")
print("-----------------------------")
for p in props:
print(f"{p}: {n.module_properties[p]}")
print("-----------------------------")
while True:
confirm = creator.prompt_options("Choose an option",
["save", "edit", "test", "quit"])
if confirm == "test":
test_notif_agent(notif_agent)
continue
else:
break
if confirm == "save":
break
elif confirm == "edit":
continue
elif confirm == "quit":
return
cur_notif_agents[n.id] = notif_agent
State.save_notif_agents()
def scrape(source,
notif_agents_list,
include=[],
exclude=[],
colour_flag="",
notify=True,
force_tasks=False,
force_agents=False,
recent_ads=0,
save_ads=True,
ignore_old_ads=False):
from lib.core.state import State
import lib.core.notif_agent as notif_agent
ads = State.get_ads()
source_modules = State.get_source_modules()
notif_agent_modules = State.get_notif_agent_modules()
log.info_print(f"Source: {source.name}")
log.info_print(f"Module: {source.module}")
log.info_print(f"Module Properties: {source.module_properties}")
if len(include):
print(f"Including: {include}")
if len(exclude):
print(f"Excluding: {exclude}")
module = source_modules[source.module]
old_ads = []
if ignore_old_ads == False:
if source.module in ads:
old_ads = ads[source.module]
log.debug(f"Total old ads: {len(old_ads)}")
else:
log.debug(f"No old ads found for module: {source.module}")
else:
log.info_print("Ignoring old ads...")
new_ads, ad_title = module.scrape_for_ads(old_ads,
exclude=exclude,
**source.module_properties)
info_string = f"Found {len(new_ads)} new ads" \
if len(new_ads) != 1 else "Found 1 new ad"
log.info_print(info_string)
num_ads = len(new_ads)
if notify and num_ads:
ads_to_send = new_ads
if recent_ads > 0:
# only notify the most recent notify_recent new_ads
ads_to_send = ct.get_most_recent_items(recent_ads, new_ads)
log.debug(
f"Recent ads set to: {recent_ads} got: {len(ads_to_send)}")
log.info_print(f"Total ads to notify about: {len(ads_to_send)}")
if len(notif_agents_list) == 0:
log.warning_print(
"No notification agents set... nothing to notify")
else:
if len(notif_agents_list) > 1:
log.info_print(
f"Notifying agents: {notif_agent.get_names(notif_agents_list)}"
)
for agent in notif_agents_list:
if agent.enabled or force_agents == True:
if agent.enabled == False and force_agents == True:
log.info_print(
"Notification agent was disabled but forcing...")
notif_agent_modules[agent.module].send_ads(
ads_to_send, ad_title, colour_flag,
**agent.module_properties)
else:
log.info_print(
f"Skipping... Notification agent disabled: {agent.name}"
)
elif not notify and num_ads:
log.info_print("Skipping notification")
if save_ads:
ads[source.module] = module.old_ad_ids
log.debug(f"Total all-time processed ads: {len(module.old_ad_ids)}")
else:
log.info_print(f"Saving ads disabled. Skipping...")
print()
return ScrapeSummary(new_ads=new_ads,
latest_ads=list(new_ads)[-3:],
total_new_ads=len(new_ads))
