def edit_notification_agent(notification_agent_id):
State.load()
notification_agents = NotificationAgent.query.get_or_404(
notification_agent_id)
form = NotificationAgentForm()
if form.validate_on_submit():
notification_agents.id = notification_agent_id
notification_agents.module = form.module.data
notification_agents.name = form.name.data
notification_agents.webhook_url = form.webhook_url.data
notification_agents.username = form.username.data
notification_agents.icon = form.icon.data
notification_agents.channel = form.channel.data
db.session.commit()
State.refresh_notif_agents()
flash('Your notification agent has been updated!', 'top_flash_success')
return redirect(
url_for('main.notification_agents',
notification_agent_id=notification_agents.id))
elif request.method == 'GET':
form.module.data = notification_agents.module
form.name.data = notification_agents.name
form.webhook_url.data = notification_agents.webhook_url
form.username.data = notification_agents.username
form.icon.data = notification_agents.icon
form.channel.data = notification_agents.channel
return render_template('create-notification-agent.html',
title='Update Notification Agent',
form=form,
legend='Update Notification Agent')
def run(task,
sources=None,
notif_agents=None,
notify=True,
force_tasks=False,
force_agents=False,
recent_ads=0,
save_ads=True,
ignore_old_ads=False):
from lib.core.state import State
if sources is None:
sources = State.get_sources()
if notif_agents is None:
notif_agents = State.get_notif_agents()
exclude_words = task.exclude
log.info_print(f"Task: {task.name}")
if task.enabled == False:
if force_tasks == False:
log.info_print("Task disabled. Skipping...")
print()
return
else:
log.info_print("Task disabled but forcing task to run...")
task_notif_agents = notif_agent.get_notif_agents_by_ids(
task.notif_agent_ids)
if notify == True and force_agents == False:
notif_agent.notif_agents_enabled_check(task_notif_agents)
source_results = {}
for source_id in task.source_ids:
source_results[source_id] = source.scrape(
sources[source_id],
task_notif_agents,
include=task.include,
exclude=task.exclude,
colour_flag=task.colour_flag,
notify=notify,
force_tasks=force_tasks,
force_agents=force_agents,
recent_ads=recent_ads,
save_ads=save_ads,
ignore_old_ads=ignore_old_ads)
if save_ads:
ad.save()
result = RunResult(source_results=source_results)
return result
def delete_source(source_id):
source = Source.query.get_or_404(source_id)
db.session.delete(source)
db.session.commit()
State.refresh_sources()
flash('Your source has been deleted.', 'top_flash_success')
return redirect(url_for('main.sources'))
def delete_source():
from lib.core.state import State
sources_dict = State.get_sources()
tasks_dict = State.get_tasks()
creator.print_title("Delete Source")
changes_made = False
while True:
sources_list = []
for s in sources_dict:
sources_list.append(sources_dict[s])
for i in range(len(sources_list)):
print(f"{i} - {sources_list[i].name}")
print("s - save and quit")
print("q - quit without saving")
tnum_str = creator.prompt_string("Delete source")
if tnum_str == "s":
save(sources_dict, sources_file)
core.task.save(tasks_dict, tasks_file)
return
elif tnum_str == "q":
if changes_made and creator.yes_no("Quit without saving",
"n") == "y":
return
elif not changes_made:
return
if re.match("[0-9]+$", tnum_str):
tnum = int(tnum_str)
if tnum >= 0 and tnum < len(sources_list):
if creator.yes_no(f"Delete {sources_list[tnum].name}",
"y") == "y":
used_by = get_tasks_using_source(
sources_dict[sources_list[tnum].id], tasks_dict)
if len(used_by) > 0:
task_names = []
for u in used_by:
task_names.append(f"'{u.name}'")
print(
f"Cannot delete source. It is being used by task(s): {','.join(task_names)}"
)
print(
"Delete those tasks or remove this notification agent from them first before deleting."
)
else:
do_delete_source(sources_list[tnum].id)
changes_made = True
def source_creator(source):
from lib.core.state import State
s = source
sources = State.get_sources()
modules = State.get_source_modules()
while True:
name = s.name
s.name = creator.prompt_string("Name", default=name)
s.module = create_source_choose_module(s.module)
module = modules[s.module]
props = module.get_properties()
print("Module Properties")
for p in props:
default = None
if s.module_properties is not None:
default = s.module_properties.get(p, None)
if s.module_properties is None:
s.module_properties = {}
s.module_properties[p] = creator.prompt_string(f"{p}",
default=default)
print()
print(f"Id: {s.id}")
print(f"Name: {s.name}")
print(f"Module: {s.module}")
print("-----------------------------")
for p in s.module_properties:
print(f"{p}: {s.module_properties[p]}")
print("-----------------------------")
while True:
confirm = creator.prompt_options("Choose an option",
["save", "edit", "test", "quit"])
if confirm == "test":
test_source(source, modules)
continue
else:
break
if confirm == "save":
break
elif confirm == "edit":
continue
elif confirm == "quit":
return
sources[source.id] = source
save()
def delete_notification_agent(notification_agent_id):
notification_agent = NotificationAgent.query.get_or_404(
notification_agent_id)
db.session.delete(notification_agent)
db.session.commit()
State.refresh_notif_agents()
flash('Your notification agent has been deleted.', 'top_flash_success')
return redirect(url_for('main.notification_agents'))
def delete_task(task_id):
delete_task = Task.query.get_or_404(task_id)
db.session.delete(delete_task)
db.session.commit()
State.refresh_tasks()
task.refresh_cron()
flash('Your task has been deleted.', 'top_flash_success')
return redirect(url_for('main.tasks'))
def edit_source(source_id):
State.load()
source = Source.query.get_or_404(source_id)
form = SourceForm()
if form.validate_on_submit():
if form.test.data:
web_url = form.website.data
Dict = {1: 'kijiji'}
prime_source = prime.Source(module=Dict.get(form.module.data),
module_properties={
'url': web_url,
'botname': "prime"
})
try:
total_ads = prime.test_webui_source(prime_source).total_new_ads
except:
message = "Not a valid source"
else:
message = f"Found {total_ads} new ads" \
if total_ads != 1 else "Found 1 new ad"
finally:
if web_url == "":
message = "Not a valid source"
flash(message, "notification")
else:
source.module = form.module.data
source.name = form.name.data
source.website = form.website.data
source.location = form.location.data
source.range = form.range.data
# source.subreddit = form.subreddit.data
db.session.commit()
State.refresh_sources()
flash('Your source has been updated!', 'top_flash_success')
return redirect(url_for('main.sources', source_id=source.id))
elif request.method == 'GET':
form.module.data = source.module
form.name.data = source.name
form.website.data = source.website
form.location.data = source.location
form.range.data = source.range
# form.subreddit.data = source.subreddit
return render_template('create-source.html',
title='Update Source',
form=form,
legend='Update Source')
def delete_notif_agent():
notif_agents_dict = State.get_notif_agents()
creator.print_title("Delete Notification Agent")
changes_made = False
while True:
notif_agents_list = []
for n in notif_agents_dict:
notif_agents_list.append(notif_agents_dict[n])
for i in range(len(notif_agents_list)):
print(f"{i} - {notif_agents_list[i].name}")
print("s - save and quit")
print("q - quit without saving")
tnum_str = creator.prompt_string("Delete notif_agent")
if tnum_str == "s":
State.save_notif_agents()
State.save_tasks()
break
elif tnum_str == "q":
if changes_made and creator.yes_no("Quit without saving",
"n") == "y":
return
elif not changes_made:
return
if re.match("[0-9]+$", tnum_str):
tnum = int(tnum_str)
if tnum >= 0 and tnum < len(notif_agents_list):
if creator.yes_no(f"Delete {notif_agents_list[tnum].name}",
"y") == "y":
used_by = get_tasks_using_notif_agent(
notif_agents_dict[notif_agents_list[tnum].id])
if used_by is not None and len(used_by) > 0:
task_names = []
for u in used_by:
task_names.append(f"'{u.name}'")
print(
f"Cannot delete notification agent '{notif_agents_list[tnum].name}'. It is being used by: {','.join(task_names)}"
)
print(
"Delete tasks using this notification agent or remove this notification agent from those tasks first before deleting."
)
else:
do_delete_notif_agent(notif_agents_list[tnum].id)
changes_made = True
def do_delete_notif_agent_yaml(id):
from lib.core.state import State
tasks_dict = State.get_tasks()
notif_agents_dict = State.get_notif_agents()
for task_id in tasks_dict:
t = tasks_dict[task_id]
if id in t.notif_agent_ids:
t.notif_agent_ids.remove(id)
del notif_agents_dict[id]
def __repr__(self):
from lib.core.state import State
source_names = []
sources = State.get_sources()
for id in self.source_ids:
source_names.append(f"'{sources[id].name}'")
notif_agent_names = []
notif_agents = State.get_notif_agents()
for id in self.notif_agent_ids:
notif_agent_names.append(f"'{notif_agents[id].name}'")
return f"""
def test_notif_agent(notif_agent):
from lib.core.state import State
notif_agents = State.get_notif_agents()
modules = State.get_notif_agent_modules()
log.debug_print(f"Testing notification agent: '{notif_agent.name}'")
module = modules[notif_agent.module]
module.send(title=f"Testing '{notif_agent.name}'",
message="This is a test message",
**notif_agent.module_properties)
print("Done!")
def do_delete_notif_agent(id):
from lib.core.state import State
import lib.core.hooks as hooks
tasks_dict = State.get_tasks()
notif_agents_dict = State.get_notif_agents()
for task_id in tasks_dict:
t = tasks_dict[task_id]
if id in t.notif_agent_ids:
t.notif_agent_ids.remove(id)
hooks.delete_notif_agent_model(notif_agents_dict[id])
del notif_agents_dict[id]
def do_delete_source(id):
import lib.core.hooks as hooks
from lib.core.state import State
tasks = State.get_tasks()
sources = State.get_sources()
for task_id in tasks:
task = tasks[task_id]
if id in task.source_ids:
task.source_ids.remove(id)
hooks.delete_source_model(sources[id])
del sources[id]
def edit_source():
from lib.core.state import State
sources = State.get_sources()
modules = State.get_source_modules()
creator.print_title("Edit Source")
source = creator.prompt_complex_dict("Choose a source",
sources,
"name",
extra_options=["d"],
extra_options_desc=["done"])
if source == "d":
return
else:
source_creator(source)
def __init__(self,
id=None,
name="New Task",
enabled=True,
frequency=15,
frequency_unit="minutes",
source_ids=None,
notif_agent_ids=None,
include=None,
exclude=None,
colour_flag=""):
if id is None:
from lib.core.state import State
id = creator.create_simple_id(State.get_tasks())
self.id = id
self.name = name
self.enabled = enabled
self.frequency = frequency
self.frequency_unit = frequency_unit
self.source_ids = source_ids
self.notif_agent_ids = notif_agent_ids
self.include = include
self.exclude = exclude
self.colour_flag = colour_flag
def run(cron_time,
cron_unit,
notify=True,
force_tasks=False,
force_agents=False,
recent_ads=3):
tasks = State.get_tasks()
log.add_handler("CRON_HANDLER")
log.info_print(f"Running cronjob for schedule: {cron_time} {cron_unit}")
# Scrape each url given in tasks file
for id in tasks:
t = tasks[id]
freq = t.frequency
freq_unit = t.frequency_unit
# skip tasks that dont correspond with the cron schedule
if int(freq) != int(cron_time) or freq_unit[:1] != cron_unit[:1]:
continue
task.run(t,
notify=notify,
force_tasks=force_tasks,
force_agents=force_agents,
recent_ads=recent_ads)
def create_notification_agents():
State.load()
form = NotificationAgentForm()
if form.validate_on_submit():
if form.test.data:
# This button will send a test notification to the Notification Agent.
# Will need to add IF statements to send different notification messages to different agents.
webhook_url = form.webhook_url.data
if form.username.data:
username = form.username.data
else:
username = "******"
Dict = {1: 'discord'}
test_notify = notifAgent.NotifAgent(module=Dict.get(
form.module.data),
module_properties={
'webhook': webhook_url,
'botname': username
})
notifAgent.test_notif_agent(test_notify)
# Notification message on webui
flash("A test message has been sent to your notification agent",
"notification")
else:
notification_agent = NotificationAgent(
module=form.module.data,
name=form.name.data,
webhook_url=form.webhook_url.data,
username=form.username.data,
icon=form.icon.data,
channel=form.channel.data)
db.session.add(notification_agent)
db.session.commit()
State.refresh_notif_agents()
flash('Your notification agent has been saved!',
'top_flash_success')
return redirect(url_for('main.notification_agents'))
return render_template('create-notification-agent.html',
title='Create Notification Agent',
form=form,
legend='Create Notification Agent')
def create_source():
State.load()
form = SourceForm()
if form.validate_on_submit():
if form.test.data:
web_url = form.website.data
Dict = {1: 'kijiji'}
prime_source = prime.Source(module=Dict.get(form.module.data),
module_properties={
'url': web_url,
'botname': "prime"
})
try:
total_ads = prime.test_webui_source(prime_source).total_new_ads
except:
message = "Not a valid source"
else:
message = f"Found {total_ads} new ads" \
if total_ads != 1 else "Found 1 new ad"
finally:
if web_url == "":
message = "Not a valid source"
flash(message, "notification")
else:
source = Source(
module=form.module.data,
name=form.name.data,
website=form.website.data,
location=form.location.data,
range=form.range.data,
# subreddit=form.subreddit.data
)
db.session.add(source)
db.session.commit()
State.refresh_sources()
flash('Your source has been saved!', 'top_flash_success')
return redirect(url_for('main.sources'))
return render_template('create-source.html',
title='Create Source',
form=form,
legend='Create Source')
def do_delete_task(id):
from lib.core.state import State
import lib.core.hooks as hooks
tasks = State.get_tasks()
hooks.delete_task_model(tasks[id])
del tasks[id]
def save():
from lib.core.state import State
ads = State.get_ads()
file = Config.ADS_FILE
with open(file, "w") as stream:
json.dump(ads, stream)
def delete_source(source_id):
State.load()
source = Source.query.get_or_404(source_id)
tasks = Task.query.all()
for task in tasks:
task.source = [s for s in task.source if s != source_id]
# if this causes a task to go down to 0 sources, then it should be deleted.
if len(task.source) == 0:
db.session.delete(task)
db.session.delete(source)
db.session.commit()
flash('Your source has been deleted.', 'top_flash_success')
return redirect(url_for('main.sources'))
def get_tasks_using_notif_agent(notif_agent):
result = []
tasks = State.get_tasks()
for id in tasks:
if notif_agent.id in tasks[id].notif_agent_ids:
result.append(tasks[id])
return result
def prime_all(tasks=None, notify=True, recent_ads=3):
if tasks is None:
from lib.core.state import State
tasks = State.get_tasks()
results = []
for id in tasks:
results.append(prime(tasks[id], notify=notify, recent_ads=recent_ads))
return results
def get_notif_agents_by_ids(ids):
from lib.core.state import State
notif_agents = State.get_notif_agents()
result = []
for id in ids:
result.append(notif_agents[id])
return result
def edit_notif_agent():
creator.print_title("Edit Notification Agent")
notif_agent = creator.prompt_complex_dict("Choose a notification agent",
State.get_notif_agents(),
"name",
extra_options=["d"],
extra_options_desc=["done"])
if notif_agent == "d":
return
else:
notif_agent_creator(notif_agent)
def list_sources(pause_after=0):
from lib.core.state import State
sources = State.get_sources()
i = 0
for id in sources:
print(sources[id])
i = i + 1
if pause_after > 0 and i == pause_after:
i = 0
if input(":") == "q":
break
def edit_task():
from lib.core.state import State
creator.print_title("Edit Task")
task = creator.prompt_complex_dict("Choose a task",
State.get_tasks(),
"name",
extra_options=["d"],
extra_options_desc=["done"])
if task == "d":
return
else:
task_creator(task)
def list_tasks(pause_after=0):
from lib.core.state import State
tasks = State.get_tasks()
i = 0
for t in tasks:
print(tasks[t])
i = i + 1
if pause_after > 0 and i == pause_after:
i = 0
if input(":") == "q":
break
def get_tasks_using_source(source, tasks=None):
if tasks is None:
from lib.core.state import State
tasks = State.get_tasks()
used_by = []
for id in tasks:
if source.id in tasks[id].source_ids:
used_by.append(tasks[id])
return used_by
