def load_job(job_id):
"""
:return:
"""
data = request.get_json(force=True)
name = data['name']
trigger_name = data['trigger_name']
job_name = data['job_name']
try:
job = current_app.apscheduler.load_job(job_id, name, trigger_name,
job_name)
return jsonify(dict(message='Job: %s load success!' % job_id,
title='Success',
type='success'),
status=200)
# return jsonify(job)
except ConflictingIdError:
return jsonify(dict(message='Job: %s already exists.' % job_id,
title='Warning',
type='warning'),
status=200)
except Exception as e:
return jsonify(dict(message=str(e), title='Error', type='error'),
status=202)
def remove_jobs():
try:
jobs = current_app.apscheduler.remove_all_jobs()
return jsonify(dict(message='success', status=200))
except Exception as e:
return jsonify(dict(message=str(e)), status=202)
def update_task(name):
scheduler.remove_job(name)
script = request.files.get('script')
if not script:
return jsonify(dict(error_message='script file can not be none.'),
status=409)
path = 'crawlers/' + script.filename
script.save(path)
task = request.form.copy().to_dict()
for key, value in task.items():
print key
print value
if value.isdigit():
task[key] = int(value)
try:
job = scheduler.add_job(**task)
return jsonify(job)
except ConflictingIdError:
return jsonify(dict(error_message='Job %s already exists.' %
task.get('id')),
status=409)
except Exception as e:
LOG.exception(e)
return jsonify(dict(error_message=str(e)), status=500)
def add_job_from_dict(data):
try:
job = current_app.apscheduler.add_job(**data)
return jsonify(job)
except ConflictingIdError:
return jsonify(dict(error_message='Job %s already exists.' % data.get('id')), status=409)
except Exception as e:
return jsonify(dict(error_message=str(e)), status=500)
def get_job(job_id):
"""Gets a job."""
job = current_app.apscheduler.get_job(job_id)
if not job:
return jsonify(dict(message='Job %s not found' % job_id), status=202)
return jsonify(job)
def delete_job(job_id):
"""Deletes a job."""
try:
current_app.apscheduler.remove_job(job_id)
return Response(status=200)
except JobLookupError:
return jsonify(dict(message='Job %s not found' % job_id), status=202)
except Exception as e:
return jsonify(dict(message=str(e)), status=202)
def pause_job(job_id):
"""Pauses a job."""
try:
current_app.apscheduler.pause_job(job_id)
job = current_app.apscheduler.get_job(job_id)
return jsonify(job)
except JobLookupError:
return jsonify(dict(message='Job %s not found' % job_id), status=202)
except Exception as e:
return jsonify(dict(message=str(e)), status=202)
def add_task():
"""
:param url: str target url
:param name: task name
:param url_type: desc info
:param script: python script for handler url response
:param start method: script scheduler method
:param depends: requirments txt
:param scheduler_type: interval or cron
:return:
"""
script = request.files.get('script')
task = request.form.copy().to_dict()
if script:
path = 'crawlers/' + script.filename
script.save(path)
else:
if 'script' in task:
del task['script']
depends = task.get('depends', None)
if depends:
for depend in depends.split(','):
c_o = commands.getoutput('pip install %s' % depend)
LOG.info('c_o:%s', c_o)
del task['depends']
trigger_value = task['trigger_value']
trigger_dict = {}
for entry in trigger_value.split(','):
key, value = entry.split(':', 1)
trigger_dict[key] = int(value) if value.isdigit() else value
del task['trigger_value']
task.update(trigger_dict)
try:
if scheduler.get_job(task['id']):
scheduler.remove_job(task['id'])
# if crawler file change, we should reload it
crawler_module = util.import_module(task['func'])
reload(crawler_module)
job = scheduler.add_job(**task)
return redirect('/index.html')
except ConflictingIdError:
return jsonify(dict(error_message='Job %s already exists.' %
task.get('id')),
status=409)
except Exception as e:
LOG.exception(e)
return jsonify(dict(error_message=str(e)), status=500)
def add_job():
"""Adds a new job."""
data = request.get_json(force=True)
try:
job = current_app.apscheduler.add_job(**data)
return jsonify(job)
except ConflictingIdError:
return jsonify(dict(message='Job %s already exists.' % data.get('id')),
status=202)
except Exception as e:
return jsonify(dict(message=str(e)), status=202)
def update_job(job_id):
"""Updates a job."""
data = request.get_json(force=True)
try:
current_app.apscheduler.modify_job(job_id, **data)
job = current_app.apscheduler.get_job(job_id)
return jsonify(job)
except JobLookupError:
return jsonify(dict(message='Job %s not found' % job_id), status=202)
except Exception as e:
return jsonify(dict(message=str(e)), status=202)
def get_jobs():
"""Gets all scheduled jobs."""
job_states = []
jobs = current_app.apscheduler.get_jobs()
for job in jobs:
desc = AvailableJobs.get_description(job.id)
temp = loads(dumps(job))
temp.update({'desc': desc})
job_states.append(temp)
return jsonify(job_states)
def get_scheduler_info():
"""Gets the scheduler info."""
scheduler = current_app.apscheduler
d = OrderedDict([('current_host', scheduler.host_name),
('allowed_hosts', scheduler.allowed_hosts),
('running', scheduler.running)])
return jsonify(d)
def get_jobs():
"""Gets all scheduled jobs."""
jobs = current_app.apscheduler.get_jobs()
job_states = []
for job in jobs:
job_states.append(job)
return jsonify(job_states)
def unload_job(job_id):
"""
:return:
"""
data = request.get_json(force=True)
try:
job = current_app.apscheduler.unload_job(job_id)
return jsonify(dict(message='Job: %s unload success!' % job_id,
title='Success',
type='success'),
status=200)
except ConflictingIdError:
return jsonify(dict(message='Job: %s already exists.' % job_id,
title='Warning',
type='warning'),
status=200)
except Exception as e:
return jsonify(dict(message=str(e), title='Error', type='error'),
status=202)
def log_detail():
reg = re.compile('^[\d]{4}-[\d]{2}-[\d]{2}')
f = codecs.open('spider.log', 'rb', 'utf-8')
# f = open('spider.log', 'rb')
lines = f.readlines()
f.close()
lines = lines[-1000:]
res = []
for line in lines:
if reg.match(line):
line_array = line.strip().split(' ', 4)
line_info = {
'time': line_array[0] + ' ' + line_array[1],
'level': line_array[2],
'module': line_array[3],
'message': line_array[4]
}
res.append(line_info)
return jsonify(res)
def start():
try:
current_app.apscheduler.start()
return jsonify(dict(message='success', status=200))
except Exception as e:
return jsonify(dict(message=str(e)), status=202)
def get(self, job_id):
job = scheduler.get_job(job_id)
return jsonify(job)
def run_job(job_id):
current_app.apscheduler.get_job(id=job_id).modify(
next_run_time=datetime.datetime.now() + datetime.timedelta(seconds=3))
job = current_app.apscheduler.get_job(job_id)
return jsonify(job)
def get(self):
jobs = scheduler.get_jobs()
return jsonify(jobs)
