def get_executor(self, job):
executor = job.get('executor')
t = job.get('type')
if executor is None:
executor = self.job_type_conf(t).get('executor')
DEBUG('type: %s executor: %s' % (t, executor))
return create_obj_from_config(executor)
def get_state_saver():
global _state_saver
if _state_saver is None:
d = etcd['quant_executor/executor/state_saver']
if d is None:
return None
_state_saver = create_obj_from_config(d)
return _state_saver
def init_plugins(self, plugin_confs, context=None):
self.job_plugins = []
if not plugin_confs:
return
if context is None:
context = globals()
for p in plugin_confs:
kwargs = { 'worker': self }
x = create_obj_from_config(p, kwargs=kwargs)
self.job_plugins.append(x)
def execute(self, job):
debug = job.get('debug')
if job.get('debug'):
crawler_server = None
else:
crawler_server = serviced.get_service_client('quant_crawler')
kwargs = {
'crawler_server': crawler_server
}
self.crawler = create_obj_from_config(job['crawler'], kwargs=kwargs)
stream = self.crawler.feed(job)
output = job.get('output')
if debug or output is None:
#start generator
for x in stream:
print json_ext.dumps(x)
return None
else:
tsdb = self.tsdb()
tsdb.create_series(output)
tsdb.write_series(output, None, stream, timeout=None)
