def run(task,
sources=None,
notif_agents=None,
notify=True,
force_tasks=False,
force_agents=False,
recent_ads=0,
save_ads=True,
ignore_old_ads=False):
from lib.core.state import State
if sources is None:
sources = State.get_sources()
if notif_agents is None:
notif_agents = State.get_notif_agents()
exclude_words = task.exclude
log.info_print(f"Task: {task.name}")
if task.enabled == False:
if force_tasks == False:
log.info_print("Task disabled. Skipping...")
print()
return
else:
log.info_print("Task disabled but forcing task to run...")
task_notif_agents = notif_agent.get_notif_agents_by_ids(
task.notif_agent_ids)
if notify == True and force_agents == False:
notif_agent.notif_agents_enabled_check(task_notif_agents)
source_results = {}
for source_id in task.source_ids:
source_results[source_id] = source.scrape(
sources[source_id],
task_notif_agents,
include=task.include,
exclude=task.exclude,
colour_flag=task.colour_flag,
notify=notify,
force_tasks=force_tasks,
force_agents=force_agents,
recent_ads=recent_ads,
save_ads=save_ads,
ignore_old_ads=ignore_old_ads)
if save_ads:
ad.save()
result = RunResult(source_results=source_results)
return result
def delete_source():
from lib.core.state import State
sources_dict = State.get_sources()
tasks_dict = State.get_tasks()
creator.print_title("Delete Source")
changes_made = False
while True:
sources_list = []
for s in sources_dict:
sources_list.append(sources_dict[s])
for i in range(len(sources_list)):
print(f"{i} - {sources_list[i].name}")
print("s - save and quit")
print("q - quit without saving")
tnum_str = creator.prompt_string("Delete source")
if tnum_str == "s":
save(sources_dict, sources_file)
core.task.save(tasks_dict, tasks_file)
return
elif tnum_str == "q":
if changes_made and creator.yes_no("Quit without saving",
"n") == "y":
return
elif not changes_made:
return
if re.match("[0-9]+$", tnum_str):
tnum = int(tnum_str)
if tnum >= 0 and tnum < len(sources_list):
if creator.yes_no(f"Delete {sources_list[tnum].name}",
"y") == "y":
used_by = get_tasks_using_source(
sources_dict[sources_list[tnum].id], tasks_dict)
if len(used_by) > 0:
task_names = []
for u in used_by:
task_names.append(f"'{u.name}'")
print(
f"Cannot delete source. It is being used by task(s): {','.join(task_names)}"
)
print(
"Delete those tasks or remove this notification agent from them first before deleting."
)
else:
do_delete_source(sources_list[tnum].id)
changes_made = True
def source_creator(source):
from lib.core.state import State
s = source
sources = State.get_sources()
modules = State.get_source_modules()
while True:
name = s.name
s.name = creator.prompt_string("Name", default=name)
s.module = create_source_choose_module(s.module)
module = modules[s.module]
props = module.get_properties()
print("Module Properties")
for p in props:
default = None
if s.module_properties is not None:
default = s.module_properties.get(p, None)
if s.module_properties is None:
s.module_properties = {}
s.module_properties[p] = creator.prompt_string(f"{p}",
default=default)
print()
print(f"Id: {s.id}")
print(f"Name: {s.name}")
print(f"Module: {s.module}")
print("-----------------------------")
for p in s.module_properties:
print(f"{p}: {s.module_properties[p]}")
print("-----------------------------")
while True:
confirm = creator.prompt_options("Choose an option",
["save", "edit", "test", "quit"])
if confirm == "test":
test_source(source, modules)
continue
else:
break
if confirm == "save":
break
elif confirm == "edit":
continue
elif confirm == "quit":
return
sources[source.id] = source
save()
def list_sources(pause_after=0):
from lib.core.state import State
sources = State.get_sources()
i = 0
for id in sources:
print(sources[id])
i = i + 1
if pause_after > 0 and i == pause_after:
i = 0
if input(":") == "q":
break
def __repr__(self):
from lib.core.state import State
source_names = []
sources = State.get_sources()
for id in self.source_ids:
source_names.append(f"'{sources[id].name}'")
notif_agent_names = []
notif_agents = State.get_notif_agents()
for id in self.notif_agent_ids:
notif_agent_names.append(f"'{notif_agents[id].name}'")
return f"""
def __init__(self,
id=None,
name="New Source",
module=None,
module_properties=None):
from lib.core.state import State
if id is None:
id = creator.create_simple_id(State.get_sources())
self.id = id
self.name = name
self.module = module
self.module_properties = module_properties
def do_delete_source(id):
import lib.core.hooks as hooks
from lib.core.state import State
tasks = State.get_tasks()
sources = State.get_sources()
for task_id in tasks:
task = tasks[task_id]
if id in task.source_ids:
task.source_ids.remove(id)
hooks.delete_source_model(sources[id])
del sources[id]
def edit_source():
from lib.core.state import State
sources = State.get_sources()
modules = State.get_source_modules()
creator.print_title("Edit Source")
source = creator.prompt_complex_dict("Choose a source",
sources,
"name",
extra_options=["d"],
extra_options_desc=["done"])
if source == "d":
return
else:
source_creator(source)
def create_task_add_sources(default=None):
from lib.core.state import State
sources_dict = State.get_sources()
default_str = ""
if default is None and len(sources_dict) == 1:
default = [list(sources_dict)[0]]
if default is not None:
first = True
for s in default:
if not s in sources_dict:
continue
if first:
default_str = f"[{sources_dict[s].name}"
first = False
else:
default_str = f"{default_str}, {sources_dict[s].name}"
default_str = f" {default_str}]"
add_sources = []
if len(sources_dict) == 0:
log.error_print(f"No sources found. Please add a source ")
return
sources_list = list(sources_dict.values())
remaining_sources = sources_list.copy()
while len(remaining_sources) > 0:
i = 0
for s in remaining_sources:
print(f"{i} - {s.name}")
i = i + 1
choices = "0"
if len(remaining_sources) > 1:
choices = f"0-{len(remaining_sources) - 1}"
if len(add_sources) > 0:
print("r - reset")
print("d - done")
if default is None or len(add_sources) > 0:
source_index_str = input(f"Source [{choices}]: ")
else:
source_index_str = input(f"Source [{choices}]:{default_str} ")
if default is not None and source_index_str == "" and len(
add_sources) == 0:
return default
if len(remaining_sources) == 0 and source_index_str == "":
add_sources.append(remaining_sources[source_index])
break
if len(add_sources):
if source_index_str == "d":
break
elif source_index_str == "r":
print("Resetting...")
remaining_sources = sources_list.copy()
add_sources = []
if re.match("[0-9]+$", source_index_str):
source_index = int(source_index_str)
if source_index >= 0 and source_index < len(sources_list):
add_sources.append(remaining_sources[source_index])
del (remaining_sources[source_index])
if len(remaining_sources) == 0:
break
confirm = creator.yes_no("Add another?", "y")
if confirm == "n":
break
result = []
for s in add_sources:
result.append(s.id)
return result
def task_creator(task):
from lib.core.state import State
cur_tasks = State.get_tasks()
sources = State.get_sources()
notif_agents = State.get_notif_agents()
t = task
while True:
while True:
print(f"Id: {t.id}")
t.name = creator.prompt_string("Name", default=t.name)
t.frequency = creator.prompt_num("Frequency", default=t.frequency)
t.frequency_unit = creator.prompt_options("Frequency Unit",
["minutes", "hours"],
default=t.frequency_unit)
t.source_ids = create_task_add_sources(default=t.source_ids)
if t.source_ids == None:
return
t.include = creator.prompt_string(
"Include [list seperated by commas]",
allow_empty=True,
default=t.include)
t.exclude = creator.prompt_string(
"Exclude [list seperated by commas]",
allow_empty=True,
default=t.exclude)
t.notif_agent_ids = create_task_add_notif_agents(
default=t.notif_agent_ids)
if t.notif_agent_ids == None:
return
print()
print(f"Name: {t.name}")
print(f"Frequency: {t.frequency} {t.frequency_unit}")
print(f"Sources")
print(f"----------------------------")
for source_id in t.source_ids:
print(f"{sources[source_id].name}")
print("-----------------------------")
print(f"Include: {t.include}")
print(f"Exclude: {t.exclude}")
while True:
confirm = creator.prompt_options(
"Choose an option", ["save", "edit", "dryrun", "quit"])
if confirm == "quit":
if creator.yes_no("Quit without saving?", "n") == "y":
return
else:
continue
elif confirm == "dryrun":
if creator.yes_no("Execute dry run?", "y"):
log.debug_print("Executing dry run...")
test(task)
continue
else:
break
if confirm == "save":
break
elif confirm == "edit":
continue
cur_tasks[task.id] = task
save()
if creator.yes_no("Prime this task?", "y") == "y":
recent = int(
creator.prompt_num(
"How many of the latest ads do you want notified?",
default="3"))
if recent == 0:
notify = False
else:
notify = True
prime(task, notify=notify, recent_ads=recent)
if not cron.exists(task.frequency, task.frequency_unit):
if creator.yes_no(
f"Add cronjob for '{task.frequency} {task.frequency_unit}'",
"y") == "y":
refresh_cron()
else:
print(
f"Cronjob already exists for '{task.frequency} {task.frequency_unit}'... skipping"
)
print("Done!")
return
def test_source():
option = simple_cmd("Choose a Source to Test", "Test Source",
State.get_sources(), source.Source)
if option is not None:
source.test_source(option)
