def create_scale_ai_project_mapping(self, scale_ai_project, connection): external_map = ExternalMap.new( session=self.session, job=self.task_template, external_id=scale_ai_project['name'], connection=connection, diffgram_class_string='task_template', type=connection.integration_name, url='https://dashboard.scale.com/test/tasks?project={}'.format(scale_ai_project['name']), add_to_session=True, flush_session=True ) # Commented to bottom to avoid circular dependencies on job. self.task_template.default_external_map = external_map logger.debug('Created ScaleAI Project {}'.format(scale_ai_project['name']))
def execute_after_launch_strategy(self): """ This strategy will attach files from sync directories and creates tasks in Diffgram for each of them. :return: """ job_sync_manager = job_dir_sync_utils.JobDirectorySyncManager( session=self.session, job=self.task_template, log=self.log ) job_sync_manager.create_file_links_for_attached_dirs(create_tasks=True) # This removes the job from initial file sync queue. self.task_template.pending_initial_dir_sync = False self.session.add(self.task_template) logger.debug('StandardTaskTemplateAfterLaunchStrategy for Task Template ID: {} completed successfully.'.format( self.task_template.id))
def execute_after_launch_strategy(self): """ This strategy will create a project on scale AI for user to send tasks into it when task is launched. :return: """ scale_ai_project = None try: # TODO: ADD LABELBOX LOGIC HERE # We don't check perms here because we assume this was checked on the task template creation. # Otherwise, we would need request context here, which we don't have. connection = self.task_template.interface_connection logger.debug('Connection for ScaleAI: {}'.format(connection)) connector_manager = ConnectorManager(connection=connection, session=self.session) connector = connector_manager.get_connector_instance() connector.connect() # We create a project scale_ai_project = self.create_scale_ai_project(connector) external_map = self.create_scale_ai_project_mapping(scale_ai_project, connection) logger.info( 'ScaleAITaskTemplateAfterLaunchStrategy for Task Template ID: {} completed successfully.'.format( self.task_template.id)) logger.debug('Proceding to standard task template launch...') # Now create tasks as usual. standard_strategy = StandardTaskTemplateAfterLaunchStrategy( session=self.session, task_template=self.task_template, log=self.log ) standard_strategy.execute_after_launch_strategy() except Exception as e: logger.exception(e) raise(e)
def determine_task_template_post_launch_strategy(self): """ This function will be routing to the appropriate launching algorithm depending on the task template configurations. For example, if an external interface was selected, the launch strategy will be different than if the standard Diffgram interface. :return: """ logger.debug('Determining post launch strategy...') # We default to standard launch strategy. strategy = StandardTaskTemplateAfterLaunchStrategy( session=self.session, task_template=self.task_template, log=self.log) logger.debug('interface_connection is {}'.format( self.task_template.interface_connection)) if self.task_template.interface_connection: interface_connection = self.task_template.interface_connection logger.debug('integration name is {}'.format( interface_connection.integration_name)) # If task template has an integration with labelbox. Change the after launch strategy. if interface_connection.integration_name == 'labelbox': strategy = LabelboxTaskTemplateAfterLaunchStrategy( session=self.session, task_template=self.task_template, log=self.log) if interface_connection.integration_name == 'datasaur': strategy = DatasaurTaskTemplateAfterLaunchStrategy( session=self.session, task_template=self.task_template, log=self.log) if interface_connection.integration_name == 'scale_ai': strategy = ScaleAITaskTemplateAfterLaunchStrategy( session=self.session, task_template=self.task_template, log=self.log) return strategy
def execute_after_launch_strategy(self): """ This strategy will attach files from sync directories and creates tasks in Diffgram for each of them. :return: """ datasaur_project = None connection = self.task_template.interface_connection logger.debug('Connection for Datasaur: {}'.format(connection)) connector_manager = ConnectorManager(connection=connection, session=self.session) connector = connector_manager.get_connector_instance() connector.connect() try: label_data = [] for label_element in self.task_template.label_dict.get('label_file_list_serialized', []): element = { 'uuid': str(uuid.uuid4()), 'diffgram_label_file': label_element['id'], 'name': '{}'.format(label_element['label']['name']), 'color': label_element['colour']['hex'].upper(), } label_data.append(element) # First we need to build a label set label_set_result = self.create_datasaur_labelset(label_data, connector) label_set = label_set_result['result']['createLabelSet'] logger.debug('Created label_set {}'.format(label_set)) if label_set.get('id'): logger.info('Datasaur Labelset created succesfully ID:'.format(label_set['id'])) ExternalMap.new( session=self.session, job=self.task_template, external_id=label_set['id'], connection=connection, diffgram_class_string='', type='{}_label_set'.format(connection.integration_name), url='', add_to_session=True, flush_session=True ) # Now save mappings for created labels for label_element in label_data: ExternalMap.new( session=self.session, job=self.task_template, file_id=label_element['diffgram_label_file'], external_id=label_element['uuid'], connection=connection, diffgram_class_string='label_file', type='{}_label'.format(connection.integration_name), url='', add_to_session=True, flush_session=True ) # Now we create a project files_to_process = self.task_template.get_attached_files(self.session, type='text') files_to_process_by_id = {} if len(files_to_process) == 0: raise Exception('Task template has no files in attached folder. Stopping Datasaur launch strategy.') for file in files_to_process: files_to_process_by_id[str(file.id)] = file print('files_to_process_by_id', files_to_process_by_id) result = self.create_datasaur_project(connector, label_set, files_to_process) logger.debug('Create datasaur Project result: {}'.format(result)) if 'result' in result: datasaur_project = result['result'] ExternalMap.new( session=self.session, job=self.task_template, external_id=datasaur_project['id'], connection=connection, diffgram_class_string='task_template', type='{}_project'.format(connection.integration_name), url='https://datasaur.ai/projects/{}/'.format(datasaur_project['id']), add_to_session=True, flush_session=True, ) logger.debug('Created Datasaur Project.') # Save file ID's mappings project_files_results = self.get_project_files_list(connector, datasaur_project) print('qweqwe', project_files_results) project_files = project_files_results['result']['documents'] for file in project_files: diffgram_file = files_to_process_by_id[file['name']] ExternalMap.new( session=self.session, job=self.task_template, external_id=file['id'], file=diffgram_file, connection=connection, diffgram_class_string='file', type='{}_file'.format(connection.integration_name), url='', add_to_session=True, flush_session=True, ) # Now create tasks as usual. logger.info( 'DatasaurTaskTemplateAfterLaunchStrategy for Task Template ID: {} completed successfully.'.format( self.task_template.id)) logger.debug('Proceding to standard task template launch...') standard_strategy = StandardTaskTemplateAfterLaunchStrategy( session=self.session, task_template=self.task_template, log=self.log ) standard_strategy.execute_after_launch_strategy() else: logger.error('Error from connector: Rolling back project creation...') raise Exception(result) except Exception as e: logger.error('Error during datasaur launch strategy. {}'.format(traceback.format_exc())) if datasaur_project: logger.error('Rolling back project creation...') result = connector.put_data({ 'action_type': 'delete_project', 'project_id': datasaur_project['id'], 'event_data': {}, }) raise e