def v1_folder_extract_post(fid, sid): conf = yakonfig.get_global_config('coordinate') tm = coordinate.TaskMaster(conf) key = cbor.dumps((fid, sid)) wu_status = tm.get_work_unit_status('ingest', key) if wu_status and wu_status['status'] in (AVAILABLE, BLOCKED, PENDING): return {'state': 'pending'} else: logger.info('launching async work unit for %r', (fid, sid)) conf = yakonfig.get_global_config('coordinate') tm = coordinate.TaskMaster(conf) tm.add_work_units('ingest', [(cbor.dumps((fid, sid)), {})]) return {'state': 'submitted'}
def v1_folder_extract_get(request, response, kvlclient, store, fid, sid): conf = yakonfig.get_global_config('coordinate') tm = coordinate.TaskMaster(conf) key = cbor.dumps((fid, sid)) wu_status = tm.get_work_unit_status('ingest', key) status = wu_status['status'] if status in (AVAILABLE, BLOCKED, PENDING): return {'state': 'pending'} elif status in (FINISHED, ): kvlclient.setup_namespace({'openquery': (str, )}) data = None try: data = list(kvlclient.get('openquery', (key, ))) assert len(data) == 1, data logger.info('got data of len 1: %r', data) assert data[0], data assert data[0][1], data data = data[0][1] data = json.loads(data) data['state'] = 'done' return data except: logger.info('kvlclient: %r', kvlclient) logger.error('Failed to get openquery data: %r', data, exc_info=True) return {'state': 'failed'} else: return {'state': 'failed'}
def dragnet_status(): conf = yakonfig.get_global_config('coordinate') tm = coordinate.TaskMaster(conf) wu_status = tm.get_work_unit_status('dragnet', DRAGNET_KEY) if not wu_status: return None status = wu_status['status'] return status
def v1_dragnet(): status = dragnet_status() if not status or status in (FINISHED, FAILED): logger.info('launching dragnet async work unit') conf = yakonfig.get_global_config('coordinate') tm = coordinate.TaskMaster(conf) tm.add_work_units('dragnet', [(DRAGNET_KEY, {})]) return {'state': 'submitted'} else: return {'state': 'pending'}
def main(): '''Launch the AMQP worker.''' filters = { 'already_labeled': already_labeled, 'geotime': geotime, } worker = AMQPWorker(filters) parser = argparse.ArgumentParser() modules = [yakonfig, kvlayer, dblogger, coordinate, worker] yakonfig.parse_args(parser, modules) worker.task_master = coordinate.TaskMaster( yakonfig.get_global_config(coordinate.config_name)) worker.start()
kvlclient.put('files', (file_id, json.dumps(data))) payload = { 'state': HIGHLIGHTS_PENDING, 'id': file_id_str, 'delay': delay, 'start': time.time() } # store the payload, so that it gets returned during # polling until replaced by the work unit. payload_str = json.dumps(payload) kvlclient.put('highlights', (file_id, payload_str)) logger.info('launching highlights async work unit') if task_master is None: conf = yakonfig.get_global_config('coordinate') task_master = coordinate.TaskMaster(conf) task_master.add_work_units('highlights', [(file_id_str, {})]) return payload return maybe_store_highlights(file_id, data, tfidf, kvlclient) def highlights_worker(work_unit): '''coordinate worker wrapper around :func:`maybe_create_highlights` ''' if 'config' not in work_unit.spec: raise coordinate.exceptions.ProgrammerError( 'could not run `create_highlights` without global config') web_conf = Config()