def create():
form = NewPredictionForm(request.form)
if form.validate_on_submit():
flash(form.data.data)
Job.create(name='fruits',
data=form.data.data,
current_user=current_user)
return redirect(url_for('index.dashboard'))
def print_jobs():
jobs = Job.all()
data = {}
for j in jobs:
data[j.id] = j.dict()
pass
return jsonify(data=data)
def dashboard():
jobs = Job.find(by={'user_id': current_user.id})
form = NewPredictionForm()
return render_template('dashboard.html',
title='Dashboard',
jobs=jobs,
form=form)
def detail_page_job(*args, **kwargs):
"""
Job function of scraping specified page with details about offer. It will be scheduled by main_page_job
args = [job_id, website, offer_id, datasource]
"""
print('*** detail_page_job ***', args)
job_id = args[0]
website = args[1]
offer_id = args[2]
datasource = args[3]
print('initializing scraper', job_id, website, offer_id)
scraper = Scraper(website=website, service_struct=datasource)
print('scraping')
scraper.run_detail_page_scrapping(offer_id)
print('saving')
# scraper.save_data() # save to file
print('get data')
scraper.get_data() # get data as dict
# TODO save result in db as Offer
job_id = args[0]
j: Job = Job.find(job_id)
print(j.name, j.run_date)
sleep(1)
print('site id:', website)
sleep(1)
def queue():
if 'api_key' in request.args:
api_key = request.args.get('api_key')
token = Token.find_one(by={'api_key_hash': api_key})
if token is not None:
job = Job.pop()
if job is None:
return jsonify(error='Not Found'), 404
job.set_status(Job.RUNNING)
return jsonify(job.to_dict())
else:
return jsonify(error='Wrong API key'), 401
else:
return jsonify(error='No API key'), 401
def create_job(name, func, args=None, seconds=0, minutes=0):
run_date = datetime.now() + timedelta(seconds=seconds, minutes=minutes)
job = Job.create(name=name, run_date=run_date)
if args is None:
args = [job.id]
else:
args.insert(0, job.id)
scheduler.add_job(id=str(job.id),
func=func,
args=args,
trigger='date',
run_date=run_date)
return job
def update(idx):
if 'api_key' in request.args:
api_key = request.args.get('api_key')
token = Token.find_one(by={'api_key_hash': api_key})
if token is not None:
job = Job.find_one(idx)
if job is None:
return jsonify(error='Not Found'), 404
data = request.get_json()
if 'status' in data:
job.set_status(data.get('status'))
if 'result' in data:
job.set_result(data.get('result'))
return jsonify(job.to_dict())
else:
return jsonify(error='Wrong API key'), 401
else:
return jsonify(error='No API key'), 401
def main_page_job(*args, **kwargs):
"""
Job function of scraping main page. It will by executed if user schedule scraping.
args = [job_id, website, offers_count_number, datasource]
"""
print('*** main_page_job ***')
# get args
job_id = args[0]
website = args[1]
offers_count_number = args[2]
datasource = args[3]
j: Job = Job.find(job_id)
# init scraper
print('initializing scraper')
scraper = Scraper(website=website, service_struct=datasource)
# run scrapping on main page
print('scraping')
scraper.run_main_page_scrapping(offers_count_number)
# get basic offer data
print('get data')
data = scraper.get_data()
print('saving')
# scraper.save_data()
print('creating details page jobs')
seconds = 60
for i in range(scraper.output.__len__()):
offer_id = list(data.keys())[i]
detail: Job = create_job(
'detail_page_job',
detail_page_job,
args=[datasource["service_name"], offer_id, datasource],
seconds=seconds)
print('detail run: ', detail.run_date)
seconds += 60
sleep(1)
def job(idx):
j = Job.find(idx)
if j is None:
return jsonify(error="Not Found"), 404
return jsonify(data=j.dict())
def read(idx):
job = Job.find_one(by={'id': idx, 'user_id': current_user.id})
if job is None:
return jsonify(error='Not Found'), 404
return jsonify(job=job.to_dict())
def job_missed_listener(event):
job = Job.find(event.job_id)
print(f"Listener: job {event.job_id} missed.")
job.set_status(Job.MISSED)
def job_error_listener(event):
job = Job.find(event.job_id)
print(f"Listener: job {event.job_id} failed.")
job.set_status(Job.FAILED)
def job_executed_listener(event):
job = Job.find(event.job_id)
print(f"Listener: job {event.job_id} executed. Task {job.name} finished")
job.set_status(Job.DONE)
def job_submitted_listener(event):
job = Job.find(event.job_id)
print(f"Listener: job {event.job_id} started")
job.set_status(Job.RUNNING)