def dashboard():
jobs = Job.find(by={'user_id': current_user.id})
form = NewPredictionForm()
return render_template('dashboard.html',
title='Dashboard',
jobs=jobs,
form=form)
def detail_page_job(*args, **kwargs):
"""
Job function of scraping specified page with details about offer. It will be scheduled by main_page_job
args = [job_id, website, offer_id, datasource]
"""
print('*** detail_page_job ***', args)
job_id = args[0]
website = args[1]
offer_id = args[2]
datasource = args[3]
print('initializing scraper', job_id, website, offer_id)
scraper = Scraper(website=website, service_struct=datasource)
print('scraping')
scraper.run_detail_page_scrapping(offer_id)
print('saving')
# scraper.save_data() # save to file
print('get data')
scraper.get_data() # get data as dict
# TODO save result in db as Offer
job_id = args[0]
j: Job = Job.find(job_id)
print(j.name, j.run_date)
sleep(1)
print('site id:', website)
sleep(1)
def main_page_job(*args, **kwargs):
"""
Job function of scraping main page. It will by executed if user schedule scraping.
args = [job_id, website, offers_count_number, datasource]
"""
print('*** main_page_job ***')
# get args
job_id = args[0]
website = args[1]
offers_count_number = args[2]
datasource = args[3]
j: Job = Job.find(job_id)
# init scraper
print('initializing scraper')
scraper = Scraper(website=website, service_struct=datasource)
# run scrapping on main page
print('scraping')
scraper.run_main_page_scrapping(offers_count_number)
# get basic offer data
print('get data')
data = scraper.get_data()
print('saving')
# scraper.save_data()
print('creating details page jobs')
seconds = 60
for i in range(scraper.output.__len__()):
offer_id = list(data.keys())[i]
detail: Job = create_job(
'detail_page_job',
detail_page_job,
args=[datasource["service_name"], offer_id, datasource],
seconds=seconds)
print('detail run: ', detail.run_date)
seconds += 60
sleep(1)
def job(idx):
j = Job.find(idx)
if j is None:
return jsonify(error="Not Found"), 404
return jsonify(data=j.dict())
def job_missed_listener(event):
job = Job.find(event.job_id)
print(f"Listener: job {event.job_id} missed.")
job.set_status(Job.MISSED)
def job_error_listener(event):
job = Job.find(event.job_id)
print(f"Listener: job {event.job_id} failed.")
job.set_status(Job.FAILED)
def job_executed_listener(event):
job = Job.find(event.job_id)
print(f"Listener: job {event.job_id} executed. Task {job.name} finished")
job.set_status(Job.DONE)
def job_submitted_listener(event):
job = Job.find(event.job_id)
print(f"Listener: job {event.job_id} started")
job.set_status(Job.RUNNING)