class Collection:
def __init__(self, collection_name, query_name, queue_name, table_name):
self.collection_name = collection_name
self.redis_conn = get_rc()
self.table_name = table_name
self.queue = Queue('collections',connection = self.redis_conn)
self.query_name = query_name
self.collector = Collector(query_name, collection_name, queue_name)
def schedule(self, job_parameters, input_type):
self.collector.schedule_jobs(job_parameters, input_type)
finishings = map(self.add_finisher, self.collector.jobs)
return finishings
def add_finisher(self, job):
return self.queue.enqueue(
consume_data,
kwargs = {'job_id':job.id,'table_name':self.table_name},
depends_on = job
)
def run_collector(self):
'''instigates a collector, adds the specified jobs to it,
then returns the job uuids & queue name'''
details = request.json or {}
_input = details.pop('Input')
input_type = details.pop('InputType')
collector = Collector(**details)
jobs = collector.schedule_jobs(_input, input_type)
res = {
'data':{
'jobs':jobs,
'log':collector.log_data,
},
'message':'collector initiated successfully',
}
return Response(
response = dumps(res),
status = 200,
mimetype = 'application/json'
)
