def load_job(job_id): """ :return: """ data = request.get_json(force=True) name = data['name'] trigger_name = data['trigger_name'] job_name = data['job_name'] try: job = current_app.apscheduler.load_job(job_id, name, trigger_name, job_name) return jsonify(dict(message='Job: %s load success!' % job_id, title='Success', type='success'), status=200) # return jsonify(job) except ConflictingIdError: return jsonify(dict(message='Job: %s already exists.' % job_id, title='Warning', type='warning'), status=200) except Exception as e: return jsonify(dict(message=str(e), title='Error', type='error'), status=202)
def remove_jobs(): try: jobs = current_app.apscheduler.remove_all_jobs() return jsonify(dict(message='success', status=200)) except Exception as e: return jsonify(dict(message=str(e)), status=202)
def update_task(name): scheduler.remove_job(name) script = request.files.get('script') if not script: return jsonify(dict(error_message='script file can not be none.'), status=409) path = 'crawlers/' + script.filename script.save(path) task = request.form.copy().to_dict() for key, value in task.items(): print key print value if value.isdigit(): task[key] = int(value) try: job = scheduler.add_job(**task) return jsonify(job) except ConflictingIdError: return jsonify(dict(error_message='Job %s already exists.' % task.get('id')), status=409) except Exception as e: LOG.exception(e) return jsonify(dict(error_message=str(e)), status=500)
def add_job_from_dict(data): try: job = current_app.apscheduler.add_job(**data) return jsonify(job) except ConflictingIdError: return jsonify(dict(error_message='Job %s already exists.' % data.get('id')), status=409) except Exception as e: return jsonify(dict(error_message=str(e)), status=500)
def get_job(job_id): """Gets a job.""" job = current_app.apscheduler.get_job(job_id) if not job: return jsonify(dict(message='Job %s not found' % job_id), status=202) return jsonify(job)
def delete_job(job_id): """Deletes a job.""" try: current_app.apscheduler.remove_job(job_id) return Response(status=200) except JobLookupError: return jsonify(dict(message='Job %s not found' % job_id), status=202) except Exception as e: return jsonify(dict(message=str(e)), status=202)
def pause_job(job_id): """Pauses a job.""" try: current_app.apscheduler.pause_job(job_id) job = current_app.apscheduler.get_job(job_id) return jsonify(job) except JobLookupError: return jsonify(dict(message='Job %s not found' % job_id), status=202) except Exception as e: return jsonify(dict(message=str(e)), status=202)
def add_task(): """ :param url: str target url :param name: task name :param url_type: desc info :param script: python script for handler url response :param start method: script scheduler method :param depends: requirments txt :param scheduler_type: interval or cron :return: """ script = request.files.get('script') task = request.form.copy().to_dict() if script: path = 'crawlers/' + script.filename script.save(path) else: if 'script' in task: del task['script'] depends = task.get('depends', None) if depends: for depend in depends.split(','): c_o = commands.getoutput('pip install %s' % depend) LOG.info('c_o:%s', c_o) del task['depends'] trigger_value = task['trigger_value'] trigger_dict = {} for entry in trigger_value.split(','): key, value = entry.split(':', 1) trigger_dict[key] = int(value) if value.isdigit() else value del task['trigger_value'] task.update(trigger_dict) try: if scheduler.get_job(task['id']): scheduler.remove_job(task['id']) # if crawler file change, we should reload it crawler_module = util.import_module(task['func']) reload(crawler_module) job = scheduler.add_job(**task) return redirect('/index.html') except ConflictingIdError: return jsonify(dict(error_message='Job %s already exists.' % task.get('id')), status=409) except Exception as e: LOG.exception(e) return jsonify(dict(error_message=str(e)), status=500)
def add_job(): """Adds a new job.""" data = request.get_json(force=True) try: job = current_app.apscheduler.add_job(**data) return jsonify(job) except ConflictingIdError: return jsonify(dict(message='Job %s already exists.' % data.get('id')), status=202) except Exception as e: return jsonify(dict(message=str(e)), status=202)
def update_job(job_id): """Updates a job.""" data = request.get_json(force=True) try: current_app.apscheduler.modify_job(job_id, **data) job = current_app.apscheduler.get_job(job_id) return jsonify(job) except JobLookupError: return jsonify(dict(message='Job %s not found' % job_id), status=202) except Exception as e: return jsonify(dict(message=str(e)), status=202)
def get_jobs(): """Gets all scheduled jobs.""" job_states = [] jobs = current_app.apscheduler.get_jobs() for job in jobs: desc = AvailableJobs.get_description(job.id) temp = loads(dumps(job)) temp.update({'desc': desc}) job_states.append(temp) return jsonify(job_states)
def get_scheduler_info(): """Gets the scheduler info.""" scheduler = current_app.apscheduler d = OrderedDict([('current_host', scheduler.host_name), ('allowed_hosts', scheduler.allowed_hosts), ('running', scheduler.running)]) return jsonify(d)
def get_jobs(): """Gets all scheduled jobs.""" jobs = current_app.apscheduler.get_jobs() job_states = [] for job in jobs: job_states.append(job) return jsonify(job_states)
def unload_job(job_id): """ :return: """ data = request.get_json(force=True) try: job = current_app.apscheduler.unload_job(job_id) return jsonify(dict(message='Job: %s unload success!' % job_id, title='Success', type='success'), status=200) except ConflictingIdError: return jsonify(dict(message='Job: %s already exists.' % job_id, title='Warning', type='warning'), status=200) except Exception as e: return jsonify(dict(message=str(e), title='Error', type='error'), status=202)
def log_detail(): reg = re.compile('^[\d]{4}-[\d]{2}-[\d]{2}') f = codecs.open('spider.log', 'rb', 'utf-8') # f = open('spider.log', 'rb') lines = f.readlines() f.close() lines = lines[-1000:] res = [] for line in lines: if reg.match(line): line_array = line.strip().split(' ', 4) line_info = { 'time': line_array[0] + ' ' + line_array[1], 'level': line_array[2], 'module': line_array[3], 'message': line_array[4] } res.append(line_info) return jsonify(res)
def start(): try: current_app.apscheduler.start() return jsonify(dict(message='success', status=200)) except Exception as e: return jsonify(dict(message=str(e)), status=202)
def get(self, job_id): job = scheduler.get_job(job_id) return jsonify(job)
def run_job(job_id): current_app.apscheduler.get_job(id=job_id).modify( next_run_time=datetime.datetime.now() + datetime.timedelta(seconds=3)) job = current_app.apscheduler.get_job(job_id) return jsonify(job)
def get(self): jobs = scheduler.get_jobs() return jsonify(jobs)