def sync_projects_for_task_template(self, task_template): connection = task_template.interface_connection connector_manager = ConnectorManager(connection=connection, session=self.session) datasaur_connector = connector_manager.get_connector_instance() datasaur_connector.connect() project_map = ExternalMap.get( session = self.session, job_id = task_template.id, diffgram_class_string = 'task_template', type = 'datasaur_project', return_kind = 'first') if not project_map: logger.error('Could not find external map for task template {}'.format(task_template.id)) return # Fetch all completed projects project_data = datasaur_connector.fetch_data({ 'action_type': 'get_project', 'event_data': {}, 'project_id': project_map.external_id }) # For each project, generate a export for all files. if 'log' in project_data: if 'error' in project_data['log']: logger.error('Error fetching datasaur project {}'.format(project_data['log']['error'])) logger.error('Datasaur project ID {} not found. Maybe was deleted?'.format(project_map.external_id)) return datasaur_project = project_data['result'] logger.debug('Fetched project: {}'.format(datasaur_project)) if project_map: task_template = project_map.job if datasaur_project['status'] != 'COMPLETE': logger.debug('Datasaur project {} is not completed. Skipping...'.format(datasaur_project['id'])) return # Now get All files from the project files_map = ExternalMap.get( return_kind = 'all', session = self.session, job_id = task_template.id, diffgram_class_string = 'file', type = 'datasaur_file') for file_map in files_map: diffgram_file = file_map.file datasaur_file_id = file_map.external_id logger.debug('Syncing File from Datasaur {}'.format(diffgram_file.id)) self.fetch_instances_from_file(task_template, diffgram_file, datasaur_file_id, datasaur_connector) else: logger.error('Could not find external map for task_template {}'.format(task_template.id))
def labelbox_web_hook_manager(): """ Webhook for receiving data on Diffgram once finished on labelbox. # NOTE: Labelbox does not supportText or dropdown classifications in export for videos. :return: """ # First check if secret is correct payload = request.data secret = settings.LABEL_BOX_SECRET log = regular_log.default() computed_signature = hmac.new(bytearray(secret.encode('utf-8')), msg=payload, digestmod=hashlib.sha1).hexdigest() if request.headers['X-Hub-Signature'] != 'sha1=' + computed_signature: error = 'Error: computed_signature does not match signature provided in the headers' logger.error( 'Error: computed_signature does not match signature provided in the headers' ) return error with sessionMaker.session_scope() as session: labelbox_event = request.headers['X-Labelbox-Event'] payload = request.json logger.debug('Payload for labelbox webhooks: {}'.format(payload)) labelbox_project_id = payload['project']['id'] project_external_mapping = ExternalMap.get( session=session, external_id=labelbox_project_id, type='labelbox', diffgram_class_string='task_template') if project_external_mapping: task_template = Job.get_by_id(session, project_external_mapping.job_id) if task_template: connection = task_template.interface_connection logger.debug('Connection for labelbox: {}'.format(connection)) connector_manager = ConnectorManager(connection=connection, session=session) connector = connector_manager.get_connector_instance() connector.connect() sync_manager = LabelBoxSyncManager( session=session, task_template=task_template, labelbox_project=None, log=log, labelbox_connector=connector) sync_manager.handle_task_creation_hook(payload) return jsonify({'message': 'OK.'}) else: log['error']['task_template'] = 'Task template not found.' return jsonify(log) else: log['error'][ 'labelbox_project'] = 'Labelbox external mapping not found.' return jsonify(log)
def get_scale_ai_project(self): external_map = ExternalMap.get( session=self.session, job_id=self.task_template.id, connection_id=self.task_template.interface_connection_id, diffgram_class_string='task_template', type=self.task_template.interface_connection.integration_name, ) if external_map: return external_map.external_id return None
def test_execute_after_launch_strategy(self): file = data_mocking.create_file({'project_id': self.project.id}, self.session) attach_dir1 = data_mocking.create_directory({ 'project': self.project, 'user': self.project_data['users'][0], 'files': [file] }, self.session) connection = data_mocking.create_connection({ 'name': 'test', 'integration_name': 'scale_ai', 'project_id': self.project.id }, self.session) job = data_mocking.create_job({ 'name': 'my-test-job-{}'.format(1), 'project': self.project, 'status': 'active', 'type': "Normal", 'attached_directories': [ attach_dir1 ], 'interface_connection_id': connection.id }, self.session) strategy = ScaleAITaskTemplateAfterLaunchStrategy( task_template=job, session=self.session, log=regular_log.default() ) with patch.object(ScaleAITaskTemplateAfterLaunchStrategy, 'create_scale_ai_project', return_value={'id': '123', 'name': 'scaleaitest'}): strategy.execute_after_launch_strategy() commit_with_rollback(self.session) tasks_count = self.session.query(Task).filter( Task.job_id == job.id ).count() tasks = self.session.query(Task).filter( Task.job_id == job.id ).all() self.assertEqual(tasks_count, 1) external_maps = ExternalMap.get( session=self.session, job_id=job.id, diffgram_class_string='task_template', connection_id=connection.id, type=connection.integration_name ) self.assertNotEqual(external_maps, None)
def get_external_id_to_task(session, task, task_template): if not task_template: return if not task_template.interface_connection: return connection = task_template.interface_connection if connection.integration_name == 'labelbox': # Try to find the task external ID external_map = ExternalMap.get( session=session, task_id=task.id, diffgram_class_string="task", type="labelbox", ) if not external_map: return None return external_map.external_id
def task_completed_scaleai(): spec_list = [ { 'status': str }, { 'task': { 'task_id': str, 'completed_at': str, 'response': dict, 'created_at': str, 'callback_url': str, 'type': str, 'status': str, 'instruction': str, 'params': dict, 'metadata': dict, } }, { 'response': dict }, { 'task_id': str }, ] log, input, untrusted_input = regular_input.master(request=request, spec_list=spec_list) annotations = input['response']['annotations'] with sessionMaker.session_scope() as session: external_map_task = ExternalMap.get( session=session, external_id=input['task']['task_id'], type='{}_task'.format('scale_ai')) task = external_map_task.task task_template = task.job scale_ai_sync_manager = ScaleAISyncManager(session=session, task_template=task_template, log=log, scale_ai_connector=None) scale_ai_sync_manager.enqueue_scale_ai_annotations(task, annotations) return jsonify({})
def test_execute_after_launch_strategy(self): file = data_mocking.create_file( { 'project_id': self.project.id, 'type': 'text' }, self.session) label = data_mocking.create_label({ 'name': 'mylabel', }, self.session) label_file = data_mocking.create_label_file( { 'label': label, 'project_id': self.project.id }, self.session) attach_dir1 = data_mocking.create_directory( { 'project': self.project, 'user': self.project_data['users'][0], 'files': [file] }, self.session) connection = data_mocking.create_connection( { 'name': 'test', 'integration_name': 'datasaur', 'project_id': self.project.id }, self.session) labeldict = { "label_file_list_serialized": [{ "id": label_file.id, "hash": "083e9ebc48d64e9a8874c6b95f490b56b8c4c5b0f4dacd90bd3534085e87d9fa", "type": "label", "state": "added", "created_time": "2020-07-15T18:48:34.477333", "time_last_updated": "2020-07-15T18:48:34.705290", "ann_is_complete": None, "original_filename": None, "video_id": None, "video_parent_file_id": None, "count_instances_changed": None, "attribute_group_list": [{ "id": 2, "kind": "multiple_select", "is_root": True, "name": "carwheeltag", "prompt": "How is this car wheel", "show_prompt": True, "time_updated": "2020-08-05 19:37:07.703576", "attribute_template_list": [{ "id": 4, "name": "Is rounded", "value_type": None, "archived": False, "group_id": 2, "display_order": None }, { "id": 5, "name": "is squared", "value_type": None, "archived": False, "group_id": 2, "display_order": None }, { "id": 6, "name": "is beautiful", "value_type": None, "archived": False, "group_id": 2, "display_order": None }, { "id": 7, "name": "is crazy", "value_type": None, "archived": False, "group_id": 2, "display_order": None }] }, { "id": 3, "kind": "select", "is_root": True, "name": "selectwheel", "prompt": "Please selectt something special about this wheels", "show_prompt": True, "time_updated": "2020-08-12 16:29:54.817801", "attribute_template_list": [{ "id": 10, "name": "Silver Wheel", "value_type": None, "archived": False, "group_id": 3, "display_order": None }, { "id": 9, "name": "+Gold wheel", "value_type": None, "archived": False, "group_id": 3, "display_order": None }] }, { "id": 4, "kind": "text", "is_root": True, "name": "freewheel", "prompt": "What are your thought on this wheel?", "show_prompt": True, "time_updated": "2020-08-05 20:50:59.195249", "attribute_template_list": [] }, { "id": 5, "kind": "radio", "is_root": True, "name": "clean", "prompt": "Is this wheel clean?", "show_prompt": True, "time_updated": "2020-08-05 20:53:46.314143", "attribute_template_list": [{ "id": 11, "name": "Wheel is dirty", "value_type": None, "archived": False, "group_id": 5, "display_order": None }, { "id": 12, "name": "Wheek is clean", "value_type": None, "archived": False, "group_id": 5, "display_order": None }] }, { "id": 6, "kind": "text", "is_root": True, "name": "TEST", "prompt": "TEST28", "show_prompt": True, "time_updated": "2020-08-12 16:30:03.770141", "attribute_template_list": [] }], "colour": { "hex": "#194d33", "hsl": { "h": 150, "s": 0.5, "l": 0.2, "a": 1 }, "hsv": { "h": 150, "s": 0.66, "v": 0.3, "a": 1 }, "rgba": { "r": 25, "g": 77, "b": 51, "a": 1 }, "a": 1 }, "label": { "id": 5, "name": "Car wheel", "default_sequences_to_single_frame": False } }], "label_file_colour_map": {} } job = data_mocking.create_job( { 'name': 'my-test-job-{}'.format(1), 'project': self.project, 'status': 'active', 'type': "Normal", 'label_dict': labeldict, 'attached_directories': [attach_dir1], 'interface_connection_id': connection.id }, self.session) strategy = DatasaurTaskTemplateAfterLaunchStrategy( task_template=job, session=self.session, log=regular_log.default()) with patch.object(DatasaurTaskTemplateAfterLaunchStrategy, 'create_datasaur_labelset', return_value={ 'result': { 'createLabelSet': { 'id': 'mytestid' } } }): with patch.object(DatasaurTaskTemplateAfterLaunchStrategy, 'create_datasaur_project', return_value={'result': { 'id': 'datasaur_test' }}): with patch.object(DatasaurTaskTemplateAfterLaunchStrategy, 'get_project_files_list', return_value={ 'result': { 'id': 'datasaur_test', 'documents': [{ 'id': str(file.id), 'name': str(file.id) }] } }): strategy.execute_after_launch_strategy() commit_with_rollback(self.session) tasks_count = self.session.query(Task).filter( Task.job_id == job.id).count() tasks = self.session.query(Task).filter( Task.job_id == job.id).all() self.assertEqual(tasks_count, 1) external_map = ExternalMap.get( session=self.session, job_id=job.id, external_id='mytestid', connection_id=connection.id, diffgram_class_string='', type='{}_label_set'.format( connection.integration_name), ) self.assertNotEqual(external_map, None) project_map = ExternalMap.get( session=self.session, job_id=job.id, external_id='datasaur_test', connection_id=connection.id, diffgram_class_string='task_template', type='{}_project'.format(connection.integration_name), ) self.assertNotEqual(project_map, None) files_maps = ExternalMap.get( session=self.session, job_id=job.id, external_id=str(file.id), file_id=file.id, connection_id=connection.id, diffgram_class_string='file', type='{}_file'.format(connection.integration_name), ) self.assertNotEqual(files_maps, None)
def add_files_to_labelbox_dataset(self, diffgram_files=[], labelbox_dataset=None, force_create=False): """ Adds the files to labelbox. Important! If you call this method multiple times, multiple versions of the same file will be created at labelbox, so use only on initialization of task templates. :param diffgram_files: :param labelbox_dataset: :param force_create: Ignore existing files and always create (useful for recreating a dataset that was deleted) :return: """ if labelbox_dataset is None: return False file_urls = [] diffgram_files_by_id = {} external_ids = [] file_ids = [x.id for x in diffgram_files] datarow_external_maps = ExternalMap.get(session=self.session, file_id=file_ids, diffgram_class_string='file', type='labelbox', return_kind='all') # To avoid querying external map each time on for loop. external_map_by_id = { ext_map.file_id: ext_map for ext_map in datarow_external_maps } data_row_ids = [ external_map.external_id for external_map in datarow_external_maps if external_map.external_id ] result_datarows = self.labelbox_connector.fetch_data({ 'action_type': 'get_data_rows', 'event_data': '', 'dataset': labelbox_dataset, 'data_row_ids': data_row_ids }) labelbox_existing_data_rows = result_datarows['result']['datasets'][0][ 'dataRows'] existing_data_rows_ids = [x['id'] for x in labelbox_existing_data_rows] deleted_data_rows = [ row_id for row_id in data_row_ids if row_id not in existing_data_rows_ids ] for diffgram_file in diffgram_files: # If we have a registered ID on labelbox, we skip file creation for this file. # We have to re-create it if it was deleted for some reason. diffgram_file_external_map = external_map_by_id.get( diffgram_file.id) if diffgram_file_external_map and diffgram_file_external_map.external_id and not force_create \ and external_map_by_id.get(diffgram_file.id).external_id not in deleted_data_rows: logger.debug('File {} exists. Skipping..'.format( diffgram_file.id)) continue if diffgram_file.type == "image": logger.debug('Adding image {} in Labelbox'.format( diffgram_file.id)) if diffgram_file.image: data = diffgram_file.image.serialize_for_source_control( self.session) data_row = { labelbox.schema.data_row.DataRow.row_data: data['url_signed'], 'external_id': diffgram_file.id } # Cache in memory the file for updating labelbox ID's later diffgram_files_by_id[diffgram_file.id] = diffgram_file external_ids.append(diffgram_file.id) file_urls.append(data_row) if diffgram_file.type == "video": if diffgram_file.video: logger.debug('Adding video {} in Labelbox'.format( diffgram_file.id)) data = diffgram_file.video.serialize_list_view( self.session, self.task_template.project) data_row = { labelbox.schema.data_row.DataRow.row_data: data['file_signed_url'], 'external_id': diffgram_file.id } # Cache in memory the file for updating labelbox ID's later external_ids.append(diffgram_file.id) diffgram_files_by_id[diffgram_file.id] = diffgram_file file_urls.append(data_row) task = labelbox_dataset.create_data_rows(file_urls) # We want to wait since we're already deferring the creation process. task.wait_till_done() # Now update al Diffgram files with their labelbox data_row ID. query = """query($datasetId: ID!, $externalId: [String!]) { datasets(where:{id: $datasetId }){ name id dataRows(where:{externalId_in: $externalId}){ id, externalId } } } """ data = {'datasetId': labelbox_dataset.uid, 'externalId': external_ids} result = self.labelbox_connector.put_data({ 'action_type': 'execute', 'event_data': [], 'query': query, 'data': data }) created_datarows = result['result']['datasets'][0]['dataRows'] for datarow in created_datarows: file = diffgram_files_by_id[int(datarow['externalId'])] file.default_external_map = ExternalMap.new( session=self.session, external_id=datarow['id'], file=file, diffgram_class_string="file", type="labelbox", add_to_session=True, flush_session=True) self.session.add(file) return task
def handle_task_creation_hook(self, payload): labelbox_data_row_id = payload['dataRow']['id'] label = json.loads(payload['label']) labelbox_label_id = payload['id'] video_mode = False frames_data = None if 'frames' in label: # Fetch video objects frames_result = self.labelbox_connector.fetch_data({ 'action_type': 'get_frames', 'frames_url': label['frames'], 'event_data': {}, }) if result_has_error(frames_result): return jsonify(frames_result), 400 frames_data = frames_result['result'] video_mode = True else: label_instances = label['objects'] file_external_mapping = ExternalMap.get( session=self.session, external_id=labelbox_data_row_id, diffgram_class_string='file', type='labelbox') if file_external_mapping: diffgram_task = self.session.query(Task).filter( Task.job_id == self.task_template.id, Task.file_id == file_external_mapping.file_id).first() if diffgram_task: # Build external mapping diffgram_task.default_external_map = ExternalMap.new( session=self.session, external_id=payload['id'], task=diffgram_task, diffgram_class_string="task", type="labelbox", add_to_session=True, flush_session=True) self.session.add(diffgram_task) # Now process Labels and add them to file. if video_mode: result = self.update_instance_list_for_video( frames_data, diffgram_task) if not result: logger.error('Error updating instances') return jsonify('Error updating instances'), 400 logger.info('Updated instances succesfully enqueued.') else: result = self.update_instance_list_for_image_or_frame( label_instances, diffgram_task) if not result or not result: logger.error('Error updating instances') return jsonify('Error updating instances'), 400 else: logger.info('Updated instances succesfully enqueued.') else: logger.error('Diffgram task not found') raise Exception('Diffgram task not found') else: logger.error('file_external_mapping not found') raise Exception('file_external_mapping not found')
def update_instance_list_for_image_or_frame(self, label_instances, diffgram_task, video_data=None, frame_packet_map=None): instance_list = [] count = 1 for labelbox_instance in label_instances: # Check if instance mapping already exists, if so provide instance_id to avoid overriding data. instance_map = ExternalMap.get( session=self.session, external_id=labelbox_instance['featureId'], diffgram_class_string='instance', type='labelbox_instance', connection_id=self.task_template.interface_connection.id) if not instance_map: instance_map = ExternalMap.new( session=self.session, external_id=None, diffgram_class_string='instance', type='labelbox_instance', connection=self.task_template.interface_connection, add_to_session=True, flush_session=True) diffgram_label_file_data = self.task_template.get_label_file_by_name( labelbox_instance['title']) diffgram_label_instance = self.transform_labelbox_label_to_diffgram_instance( labelbox_instance, diffgram_label_file_data, instance_map=instance_map, sequence_num=count if video_data is not None else None) if frame_packet_map is not None: if video_data['current_frame'] not in frame_packet_map: frame_packet_map[video_data['current_frame']] = [ diffgram_label_instance ] else: frame_packet_map[video_data['current_frame']].append( diffgram_label_instance) if diffgram_label_instance: instance_list.append(diffgram_label_instance) count += 1 if instance_list and video_data is None: enqueue_packet( project_string_id=self.task_template.project.project_string_id, session=self.session, media_url=None, media_type='image', job_id=self.task_template.id, file_id=diffgram_task.file.id, instance_list=instance_list, task_id=diffgram_task.id, task_action='complete_task', commit_input=True, external_map_id=instance_map.id, external_map_action='set_instance_id', mode="update_with_existing") return True elif instance_list: return True else: return False
def fetch_instances_from_file( self, task_template, diffgram_file, file_id, datasaur_connector): file_export_data = self.trigger_export_single_datasaur_file( datasaur_connector = datasaur_connector, file_id = file_id) instance_list = [] # We get the task based on file id since assumption for datasaur is file and task will be the same concept. task = self.session.query(Task).filter( Task.job_id == task_template.id, Task.file_id == diffgram_file.id ).first() if 'log' in file_export_data and 'error' in file_export_data['log']: logger.error('Error fetching export data {}'.format(file_export_data)) label_items = file_export_data['result']['labelSet']['labelItems'] label_items_by_id = {} for label in label_items: external_map_label = ExternalMap.get( session=self.session, job_id=task_template.id, external_id=label['id'], connection_id=task_template.interface_connection.id, diffgram_class_string='label_file', type='datasaur_label' ) if external_map_label: label_items_by_id[label['id']] = label label_items_by_id[label['id']]['label_file_id'] = external_map_label.file_id else: logger.error('No label_file found for datasaur ID: {}'.format(label['id'])) return sentences = file_export_data['result']['sentences'] for sentence in sentences: instances = sentence['labels'] for instance in instances: instance_map = ExternalMap.get( session = self.session, external_id = instance['id'], diffgram_class_string = 'instance', type = 'datasaur_instance', return_kind = 'first') if not instance_map: logger.debug('Creating Instance Map...') instance_map = ExternalMap.new( session=self.session, job=task_template, external_id=instance['id'], connection=task_template.interface_connection, diffgram_class_string='instance', type='{}_instance'.format( task_template.interface_connection.integration_name), url='', add_to_session=True, flush_session=True) else: logger.debug('Instance Map exists, proceding to update.') instance_list.append({ 'start_sentence': instance['sidS'], 'end_sentence': instance['sidE'], 'start_token': instance['s'], 'end_token': instance['e'], 'start_char': instance['charS'], 'end_char': instance['charE'], 'sentence': sentence['id'], 'type': 'text_token', 'name': label_items_by_id[instance['l']]['labelName'], 'label_file_id': label_items_by_id[instance['l']]['label_file_id'] }) logger.debug('Enqueuing new instances....') # Create new packet to ensure to commit this if task and task_template and diffgram_file: enqueue_packet(project_string_id=task_template.project.project_string_id, session=self.session, media_url=None, media_type='text', job_id=task_template.id, file_id=diffgram_file.id, instance_list=instance_list, task_id=task.id, task_action='complete_task', commit_input=True, mode="update") logger.info('Updated Task {} from datasaur.'.format(task.id))
def serialize_builder_info_default(self, session, user=None): # TODO share this with trainer info function user_to_job_serialized = None if user: user_to_job = User_To_Job.get_single_by_ids(session=session, user_id=user.id, job_id=self.id) if user_to_job: user_to_job_serialized = user_to_job.serialize_trainer_info_default( ) percent_completed = 0 tasks_remaining = 0 if self.stat_count_tasks: percent_completed = (self.stat_count_complete / self.stat_count_tasks) * 100 tasks_remaining = self.stat_count_tasks - self.stat_count_complete external_mappings = ExternalMap.get( session=session, job_id=self.id, diffgram_class_string='task_template', return_kind='all') member_list_ids = None if session: member_list_ids = self.get_with_cache( cache_key='member_list_ids', cache_miss_function=self.regenerate_member_list_ids, session=session, miss_function_args={'session': session}) external_mappings_serialized = [ x.serialize() for x in external_mappings ] default_userscript = None if self.default_userscript: default_userscript = self.default_userscript.serialize() return { 'id': self.id, 'name': self.name, 'type': self.type, 'share_type': self.share_type, 'member_list_ids': member_list_ids, 'status': self.status, 'time_created': self.time_created, 'time_completed': self.time_completed, 'user_to_job': user_to_job_serialized, 'attached_directories_dict': self.get_with_cache( cache_key='attached_directories_dict', cache_miss_function=self.get_attached_dirs_serialized, session=session, miss_function_args={'session': session}), 'external_mappings': external_mappings_serialized, 'file_count_statistic': self.file_count_statistic, 'stat_count_tasks': self.stat_count_tasks, 'stat_count_complete': self.stat_count_complete, 'percent_completed': percent_completed, 'tasks_remaining': tasks_remaining, 'is_live': self.is_live, 'pending_initial_dir_sync': self.pending_initial_dir_sync, 'interface_connection': self.interface_connection.serialize() if self.interface_connection else None, # For now the SDK uses the /info path # So if we want to expose this stuff we need that there # maybe something to review in the future 'file_count': self.file_count, 'launch_datetime': self.launch_datetime, 'launch_datetime_deferred': self.launch_datetime_deferred, 'launch_attempt_log': self.launch_attempt_log, 'waiting_to_be_launched': self.waiting_to_be_launched, 'interface_connection_id': self.interface_connection_id, # Realizing we want the label dict # ie to show the label information (not just ids...) 'label_dict': self.label_dict, 'completion_directory_id': self.completion_directory_id, 'output_dir_action': self.output_dir_action, 'pro_network': self.pro_network, 'default_userscript': default_userscript }