def v1_folder_extract_post(fid, sid):
conf = yakonfig.get_global_config('coordinate')
tm = coordinate.TaskMaster(conf)
key = cbor.dumps((fid, sid))
wu_status = tm.get_work_unit_status('ingest', key)
if wu_status and wu_status['status'] in (AVAILABLE, BLOCKED, PENDING):
return {'state': 'pending'}
else:
logger.info('launching async work unit for %r', (fid, sid))
conf = yakonfig.get_global_config('coordinate')
tm = coordinate.TaskMaster(conf)
tm.add_work_units('ingest', [(cbor.dumps((fid, sid)), {})])
return {'state': 'submitted'}
def v1_folder_extract_get(request, response, kvlclient, store, fid, sid):
conf = yakonfig.get_global_config('coordinate')
tm = coordinate.TaskMaster(conf)
key = cbor.dumps((fid, sid))
wu_status = tm.get_work_unit_status('ingest', key)
status = wu_status['status']
if status in (AVAILABLE, BLOCKED, PENDING):
return {'state': 'pending'}
elif status in (FINISHED, ):
kvlclient.setup_namespace({'openquery': (str, )})
data = None
try:
data = list(kvlclient.get('openquery', (key, )))
assert len(data) == 1, data
logger.info('got data of len 1: %r', data)
assert data[0], data
assert data[0][1], data
data = data[0][1]
data = json.loads(data)
data['state'] = 'done'
return data
except:
logger.info('kvlclient: %r', kvlclient)
logger.error('Failed to get openquery data: %r',
data,
exc_info=True)
return {'state': 'failed'}
else:
return {'state': 'failed'}
def dragnet_status():
conf = yakonfig.get_global_config('coordinate')
tm = coordinate.TaskMaster(conf)
wu_status = tm.get_work_unit_status('dragnet', DRAGNET_KEY)
if not wu_status: return None
status = wu_status['status']
return status
def v1_dragnet():
status = dragnet_status()
if not status or status in (FINISHED, FAILED):
logger.info('launching dragnet async work unit')
conf = yakonfig.get_global_config('coordinate')
tm = coordinate.TaskMaster(conf)
tm.add_work_units('dragnet', [(DRAGNET_KEY, {})])
return {'state': 'submitted'}
else:
return {'state': 'pending'}
def main():
'''Launch the AMQP worker.'''
filters = {
'already_labeled': already_labeled,
'geotime': geotime,
}
worker = AMQPWorker(filters)
parser = argparse.ArgumentParser()
modules = [yakonfig, kvlayer, dblogger, coordinate, worker]
yakonfig.parse_args(parser, modules)
worker.task_master = coordinate.TaskMaster(
yakonfig.get_global_config(coordinate.config_name))
worker.start()
kvlclient.put('files', (file_id, json.dumps(data)))
payload = {
'state': HIGHLIGHTS_PENDING,
'id': file_id_str,
'delay': delay,
'start': time.time()
}
# store the payload, so that it gets returned during
# polling until replaced by the work unit.
payload_str = json.dumps(payload)
kvlclient.put('highlights', (file_id, payload_str))
logger.info('launching highlights async work unit')
if task_master is None:
conf = yakonfig.get_global_config('coordinate')
task_master = coordinate.TaskMaster(conf)
task_master.add_work_units('highlights', [(file_id_str, {})])
return payload
return maybe_store_highlights(file_id, data, tfidf, kvlclient)
def highlights_worker(work_unit):
'''coordinate worker wrapper around :func:`maybe_create_highlights`
'''
if 'config' not in work_unit.spec:
raise coordinate.exceptions.ProgrammerError(
'could not run `create_highlights` without global config')
web_conf = Config()