def consume(self, msg): print "****** STARTING CONSUME" # ignore the message, we do what we want lock = LockFile(app.config['RUNNER_LOCKFILE']) with lock: # get list of open jobs while True: jobs = Job.query.filter_by(status=dgrepm.STATUS_FREE) if jobs.count() == 0: break for job in jobs: # run query on jobs dq = DataQuery.from_database(job) job.set_status(dgrepm.STATUS_OPEN) try: job.filename = dq.run_query( 'datagrepper_{0}'.format(job.id)) except: job.set_status(dgrepm.STATUS_FAILED) else: job.set_status(dgrepm.STATUS_DONE) # get list of completed jobs to be deleted jobs = Job.query.filter( Job.status == dgrepm.STATUS_DONE, Job.complete_time < (datetime.now() - app.config['JOB_EXPIRY']) ) for job in jobs: os.remove(os.path.join(app.config['JOB_OUTPUT_DIR'], job.filename)) job.set_status(dgrepm.STATUS_DELETED) print "****** FINISHING CONSUME"
def submit(): try: job = Job(flask.g.auth, DataQuery.from_request_args(flask.request.args)) db.session.add(job) db.session.commit() fedmsg.publish(topic='job.new', msg=job) status = 200 msg = {'id': job.id, 'options': job.dataquery['options']} except ValueError as exc: msg = {'error': 'invalid_arg', 'value': exc.message} status = 400 return flask.Response( response=fedmsg.encoding.dumps(msg), status=status, mimetype='application/json', )