def create_from_json(self, j, user=None): if self.process is None: self.process = DVAPQL() if not (user is None): self.process.user = user if j['process_type'] == DVAPQL.QUERY: image_data = None if j['image_data_b64'].strip(): image_data = base64.decodestring(j['image_data_b64']) j['image_data_b64'] = None self.process.process_type = DVAPQL.QUERY self.process.script = j self.process.save() if image_data: query_path = "{}/queries/{}.png".format(settings.MEDIA_ROOT, self.process.uuid) redis_client.set("/queries/{}.png".format(self.process.uuid), image_data, ex=1200) with open(query_path, 'w') as fh: fh.write(image_data) if settings.ENABLE_CLOUDFS: query_key = "/queries/{}.png".format(self.process.uuid) fs.upload_file_to_remote(query_key) os.remove(query_path) elif j['process_type'] == DVAPQL.PROCESS: self.process.process_type = DVAPQL.PROCESS self.process.script = j self.process.save() elif j['process_type'] == DVAPQL.SCHEDULE: raise NotImplementedError else: raise ValueError return self.process
def create_from_json(self, j, user=None): if self.process is None: self.process = DVAPQL() if not (user is None): self.process.user = user if j['process_type'] == DVAPQL.QUERY: if j['image_data_b64'].strip(): image_data = base64.decodestring(j['image_data_b64']) j['image_data_b64'] = None self.process.image_data = image_data self.process.process_type = DVAPQL.QUERY self.process.script = j self.process.save() self.store() elif j['process_type'] == DVAPQL.PROCESS: self.process.process_type = DVAPQL.PROCESS self.process.script = j self.process.save() elif j['process_type'] == DVAPQL.INGEST: raise NotImplementedError else: raise ValueError return self.process
class DVAPQLProcess(object): def __init__(self, process=None, media_dir=None): self.process = process self.media_dir = media_dir self.task_results = {} self.created_objects = [] def store(self): query_path = "{}/queries/{}.png".format(settings.MEDIA_ROOT, self.process.pk) with open(query_path, 'w') as fh: fh.write(self.process.image_data) if settings.DISABLE_NFS: query_key = "/queries/{}.png".format(self.process.pk) fs.upload_file_to_remote(query_key) os.remove(query_path) def create_from_json(self, j, user=None): if self.process is None: self.process = DVAPQL() if not (user is None): self.process.user = user if j['process_type'] == DVAPQL.QUERY: if j['image_data_b64'].strip(): image_data = base64.decodestring(j['image_data_b64']) j['image_data_b64'] = None self.process.image_data = image_data self.process.process_type = DVAPQL.QUERY self.process.script = j self.process.save() self.store() elif j['process_type'] == DVAPQL.PROCESS: self.process.process_type = DVAPQL.PROCESS self.process.script = j self.process.save() elif j['process_type'] == DVAPQL.INGEST: raise NotImplementedError else: raise ValueError return self.process def validate(self): pass def launch(self): if self.process.script['process_type'] == DVAPQL.PROCESS: for c in self.process.script.get('create', []): m = apps.get_model(app_label='dvaapp', model_name=c['MODEL']) for k, v in c['spec'].iteritems(): if v == '__timezone.now__': c['spec'][k] = timezone.now() instance = m.objects.create(**c['spec']) self.created_objects.append(instance) for t in copy.deepcopy(c.get('tasks', [])): self.launch_task(t, instance.pk) for t in self.process.script.get('tasks', []): self.launch_task(t) elif self.process.script['process_type'] == DVAPQL.QUERY: for t in self.process.script['tasks']: operation = t['operation'] arguments = t.get('arguments', {}) queue_name, operation = get_queue_name_and_operation( operation, arguments) next_task = TEvent.objects.create(parent_process=self.process, operation=operation, arguments=arguments, queue=queue_name) self.task_results[next_task.pk] = app.send_task( name=operation, args=[ next_task.pk, ], queue=queue_name, priority=5) else: raise NotImplementedError def wait(self, timeout=60): for _, result in self.task_results.iteritems(): try: next_task_ids = result.get(timeout=timeout) if next_task_ids: for next_task_id in next_task_ids: next_result = AsyncResult(id=next_task_id) _ = next_result.get(timeout=timeout) except Exception, e: raise ValueError(e)
class DVAPQLProcess(object): def __init__(self, process=None, media_dir=None): self.process = process self.media_dir = media_dir self.task_results = {} self.created_objects = [] self.task_group_index = 0 self.task_group_name_to_index = {} self.parent_task_group_index = {} self.root_task = None def launch_root_task(self): pass def create_from_json(self, j, user=None): if self.process is None: self.process = DVAPQL() if not (user is None): self.process.user = user if j['process_type'] == DVAPQL.QUERY: image_data = None if j['image_data_b64'].strip(): image_data = base64.decodestring(j['image_data_b64']) j['image_data_b64'] = None self.process.process_type = DVAPQL.QUERY self.process.script = j self.process.save() if image_data: query_path = "{}/queries/{}.png".format(settings.MEDIA_ROOT, self.process.uuid) redis_client.set("/queries/{}.png".format(self.process.uuid), image_data, ex=1200) with open(query_path, 'w') as fh: fh.write(image_data) if settings.ENABLE_CLOUDFS: query_key = "/queries/{}.png".format(self.process.uuid) fs.upload_file_to_remote(query_key) os.remove(query_path) elif j['process_type'] == DVAPQL.PROCESS: self.process.process_type = DVAPQL.PROCESS self.process.script = j self.process.save() elif j['process_type'] == DVAPQL.SCHEDULE: raise NotImplementedError else: raise ValueError return self.process def validate(self): pass def assign_task_group_id(self, tasks, parent_group_index=None): for t in tasks: t['task_group_id'] = self.task_group_index self.task_group_index += 1 if parent_group_index: self.parent_task_group_index[t['task_group_id']] = parent_group_index task_group_name = t['arguments'].get('task_group_name', None) if task_group_name: if task_group_name in self.task_group_name_to_index: self.process.failed = True self.process.error_message = "Repeated task group name." else: self.task_group_name_to_index[task_group_name] = t['task_group_id'] if 'map' in t.get('arguments', {}): self.assign_task_group_id(t['arguments']['map'], t['task_group_id']) if 'reduce' in t.get('arguments', {}): self.assign_task_group_id(t['arguments']['reduce'], t['task_group_id']) def launch(self): if self.process.script['process_type'] == DVAPQL.PROCESS: self.delete_instances() self.create_root_task() self.create_instances() self.launch_processing_tasks() self.launch_process_monitor() elif self.process.script['process_type'] == DVAPQL.QUERY: self.launch_query_tasks() else: raise NotImplementedError self.process.script['task_group_name_to_index'] = self.task_group_name_to_index self.process.script['parent_task_group_index'] = self.parent_task_group_index self.process.save() def delete_instances(self): for d in self.process.script.get('delete', []): if d['MODEL'] == 'Video': d_copy = copy.deepcopy(d) m = apps.get_model(app_label='dvaapp', model_name=d['MODEL']) instance = m.objects.get(**d_copy['selector']) DeletedVideo.objects.create(deleter=self.process.user, video_uuid=instance.pk) instance.delete() else: self.process.failed = True self.process.error_message = "Cannot delete {}; Only video deletion implemented.".format(d['MODEL']) def create_instances(self): for c in self.process.script.get('create', []): c_copy = copy.deepcopy(c) m = apps.get_model(app_label='dvaapp', model_name=c['MODEL']) for k, v in c['spec'].iteritems(): if v == '__timezone.now__': c_copy['spec'][k] = timezone.now() if c['MODEL'] != 'Video' and c['MODEL'] != 'TrainingSet': c_copy['spec']['event_id'] = self.root_task.pk instance = m.objects.create(**c_copy['spec']) self.created_objects.append(instance) def create_root_task(self): self.root_task = TEvent.objects.create(operation="perform_launch", task_group_id=self.task_group_index, completed=True, started=True, parent_process_id=self.process.pk, queue="sync") self.task_group_index += 1 def launch_processing_tasks(self): self.assign_task_group_id(self.process.script.get('map', []), 0) for t in self.process.script.get('map', []): self.launch_task(t) self.assign_task_group_id(self.process.script.get('reduce', []), 0) for t in self.process.script.get('reduce', []): if 'operation' not in t: t['operation'] = 'perform_reduce' self.launch_task(t) else: raise ValueError('{} is not a valid reduce operation, reduce tasks should not have an operation'.format( t['operation'])) def launch_query_tasks(self): self.assign_task_group_id(self.process.script.get('map', [])) for t in self.process.script['map']: operation = t['operation'] arguments = t.get('arguments', {}) queue_name, operation = get_queue_name_and_operation(operation, arguments) next_task = TEvent.objects.create(parent_process=self.process, operation=operation, arguments=arguments, queue=queue_name, task_group_id=t['task_group_id']) self.task_results[next_task.pk] = app.send_task(name=operation, args=[next_task.pk, ], queue=queue_name, priority=5) def launch_process_monitor(self): monitoring_task = TEvent.objects.create(operation="perform_process_monitoring", arguments={}, parent=None, task_group_id=-1, parent_process=self.process, queue=settings.Q_REDUCER) app.send_task(name=monitoring_task.operation, args=[monitoring_task.pk, ], queue=monitoring_task.queue) def wait_query(self, timeout=60): if self.process.process_type != DVAPQL.QUERY: raise ValueError("wait query is only supported by Query processes") for _, result in self.task_results.iteritems(): try: next_task_ids = result.get(timeout=timeout) while next_task_ids: if type(next_task_ids) is list: for next_task_id in next_task_ids: next_result = AsyncResult(id=next_task_id) next_task_ids = next_result.get(timeout=timeout) except Exception, e: raise ValueError(e)
class DVAPQLProcess(object): def __init__(self, process=None, media_dir=None): self.process = process self.media_dir = media_dir self.task_results = {} self.created_objects = [] self.task_group_index = 0 def create_from_json(self, j, user=None): if self.process is None: self.process = DVAPQL() if not (user is None): self.process.user = user if j['process_type'] == DVAPQL.QUERY: image_data = None if j['image_data_b64'].strip(): image_data = base64.decodestring(j['image_data_b64']) j['image_data_b64'] = None self.process.process_type = DVAPQL.QUERY self.process.script = j self.process.save() if image_data: query_path = "{}/queries/{}.png".format( settings.MEDIA_ROOT, self.process.uuid) redis_client.set("/queries/{}.png".format(self.process.uuid), image_data, ex=1200) with open(query_path, 'w') as fh: fh.write(image_data) if settings.DISABLE_NFS: query_key = "/queries/{}.png".format(self.process.uuid) fs.upload_file_to_remote(query_key) os.remove(query_path) elif j['process_type'] == DVAPQL.PROCESS: self.process.process_type = DVAPQL.PROCESS self.process.script = j self.process.save() elif j['process_type'] == DVAPQL.SCHEDULE: raise NotImplementedError else: raise ValueError return self.process def validate(self): pass def assign_task_group_id(self, tasks): for t in tasks: t['task_group_id'] = self.task_group_index self.task_group_index += 1 if 'map' in t.get('arguments', {}): self.assign_task_group_id(t['arguments']['map']) if 'reduce' in t.get('arguments', {}): self.assign_task_group_id(t['arguments']['reduce']) def launch(self): if self.process.script['process_type'] == DVAPQL.PROCESS: for d in self.process.script.get('delete', []): if d['MODEL'] == 'Video': d_copy = copy.deepcopy(d) m = apps.get_model(app_label='dvaapp', model_name=d['MODEL']) instance = m.objects.get(**d_copy['selector']) DeletedVideo.objects.create(deleter=self.process.user, video_uuid=instance.pk) instance.delete() else: self.process.failed = True self.process.error_message = "Cannot delete {}; Only video deletion implemented.".format( d['MODEL']) self.assign_task_group_id(self.process.script.get('tasks', [])) for c in self.process.script.get('create', []): c_copy = copy.deepcopy(c) m = apps.get_model(app_label='dvaapp', model_name=c['MODEL']) for k, v in c['spec'].iteritems(): if v == '__timezone.now__': c_copy['spec'][k] = timezone.now() instance = m.objects.create(**c_copy['spec']) self.created_objects.append(instance) self.assign_task_group_id(c.get('tasks', [])) for t in copy.deepcopy(c.get('tasks', [])): self.launch_task(t, instance.pk) self.assign_task_group_id(self.process.script.get('tasks', [])) for t in self.process.script.get('tasks', []): self.launch_task(t) elif self.process.script['process_type'] == DVAPQL.QUERY: for t in self.process.script['tasks']: operation = t['operation'] arguments = t.get('arguments', {}) queue_name, operation = get_queue_name_and_operation( operation, arguments) next_task = TEvent.objects.create(parent_process=self.process, operation=operation, arguments=arguments, queue=queue_name) self.task_results[next_task.pk] = app.send_task( name=operation, args=[ next_task.pk, ], queue=queue_name, priority=5) else: raise NotImplementedError self.process.save() def wait(self, timeout=60): for _, result in self.task_results.iteritems(): try: next_task_ids = result.get(timeout=timeout) if next_task_ids: for next_task_id in next_task_ids: next_result = AsyncResult(id=next_task_id) _ = next_result.get(timeout=timeout) except Exception, e: raise ValueError(e)