class Collection: def __init__(self, collection_name, query_name, queue_name, table_name): self.collection_name = collection_name self.redis_conn = get_rc() self.table_name = table_name self.queue = Queue('collections',connection = self.redis_conn) self.query_name = query_name self.collector = Collector(query_name, collection_name, queue_name) def schedule(self, job_parameters, input_type): self.collector.schedule_jobs(job_parameters, input_type) finishings = map(self.add_finisher, self.collector.jobs) return finishings def add_finisher(self, job): return self.queue.enqueue( consume_data, kwargs = {'job_id':job.id,'table_name':self.table_name}, depends_on = job )
def run_collector(self): '''instigates a collector, adds the specified jobs to it, then returns the job uuids & queue name''' details = request.json or {} _input = details.pop('Input') input_type = details.pop('InputType') collector = Collector(**details) jobs = collector.schedule_jobs(_input, input_type) res = { 'data':{ 'jobs':jobs, 'log':collector.log_data, }, 'message':'collector initiated successfully', } return Response( response = dumps(res), status = 200, mimetype = 'application/json' )