def consume(self, msg):
print "****** STARTING CONSUME"
# ignore the message, we do what we want
lock = LockFile(app.config['RUNNER_LOCKFILE'])
with lock:
# get list of open jobs
while True:
jobs = Job.query.filter_by(status=dgrepm.STATUS_FREE)
if jobs.count() == 0:
break
for job in jobs:
# run query on jobs
dq = DataQuery.from_database(job)
job.set_status(dgrepm.STATUS_OPEN)
try:
job.filename = dq.run_query(
'datagrepper_{0}'.format(job.id))
except:
job.set_status(dgrepm.STATUS_FAILED)
else:
job.set_status(dgrepm.STATUS_DONE)
# get list of completed jobs to be deleted
jobs = Job.query.filter(
Job.status == dgrepm.STATUS_DONE,
Job.complete_time < (datetime.now() - app.config['JOB_EXPIRY'])
)
for job in jobs:
os.remove(os.path.join(app.config['JOB_OUTPUT_DIR'],
job.filename))
job.set_status(dgrepm.STATUS_DELETED)
print "****** FINISHING CONSUME"
def submit():
try:
job = Job(flask.g.auth,
DataQuery.from_request_args(flask.request.args))
db.session.add(job)
db.session.commit()
fedmsg.publish(topic='job.new', msg=job)
status = 200
msg = {'id': job.id, 'options': job.dataquery['options']}
except ValueError as exc:
msg = {'error': 'invalid_arg', 'value': exc.message}
status = 400
return flask.Response(
response=fedmsg.encoding.dumps(msg),
status=status,
mimetype='application/json',
)
def submit():
try:
job = Job(flask.g.auth,
DataQuery.from_request_args(flask.request.args))
db.session.add(job)
db.session.commit()
fedmsg.publish(topic='job.new', msg=job)
status = 200
msg = {'id': job.id,
'options': job.dataquery['options']}
except ValueError as exc:
msg = {'error': 'invalid_arg',
'value': exc.message}
status = 400
return flask.Response(
response=fedmsg.encoding.dumps(msg),
status=status,
mimetype='application/json',
)
