def get_executor(self, job): executor = job.get('executor') t = job.get('type') if executor is None: executor = self.job_type_conf(t).get('executor') DEBUG('type: %s executor: %s' % (t, executor)) return create_obj_from_config(executor)
def get_state_saver(): global _state_saver if _state_saver is None: d = etcd['quant_executor/executor/state_saver'] if d is None: return None _state_saver = create_obj_from_config(d) return _state_saver
def init_plugins(self, plugin_confs, context=None): self.job_plugins = [] if not plugin_confs: return if context is None: context = globals() for p in plugin_confs: kwargs = { 'worker': self } x = create_obj_from_config(p, kwargs=kwargs) self.job_plugins.append(x)
def execute(self, job): debug = job.get('debug') if job.get('debug'): crawler_server = None else: crawler_server = serviced.get_service_client('quant_crawler') kwargs = { 'crawler_server': crawler_server } self.crawler = create_obj_from_config(job['crawler'], kwargs=kwargs) stream = self.crawler.feed(job) output = job.get('output') if debug or output is None: #start generator for x in stream: print json_ext.dumps(x) return None else: tsdb = self.tsdb() tsdb.create_series(output) tsdb.write_series(output, None, stream, timeout=None)