def create_scale_ai_project_mapping(self, scale_ai_project, connection):
external_map = ExternalMap.new(
session=self.session,
job=self.task_template,
external_id=scale_ai_project['name'],
connection=connection,
diffgram_class_string='task_template',
type=connection.integration_name,
url='https://dashboard.scale.com/test/tasks?project={}'.format(scale_ai_project['name']),
add_to_session=True,
flush_session=True
)
# Commented to bottom to avoid circular dependencies on job.
self.task_template.default_external_map = external_map
logger.debug('Created ScaleAI Project {}'.format(scale_ai_project['name']))
def execute_after_launch_strategy(self):
"""
This strategy will attach files from sync directories and creates tasks in
Diffgram for each of them.
:return:
"""
job_sync_manager = job_dir_sync_utils.JobDirectorySyncManager(
session=self.session,
job=self.task_template,
log=self.log
)
job_sync_manager.create_file_links_for_attached_dirs(create_tasks=True)
# This removes the job from initial file sync queue.
self.task_template.pending_initial_dir_sync = False
self.session.add(self.task_template)
logger.debug('StandardTaskTemplateAfterLaunchStrategy for Task Template ID: {} completed successfully.'.format(
self.task_template.id))
def execute_after_launch_strategy(self):
"""
This strategy will create a project on scale AI for user to send tasks into
it when task is launched.
:return:
"""
scale_ai_project = None
try:
# TODO: ADD LABELBOX LOGIC HERE
# We don't check perms here because we assume this was checked on the task template creation.
# Otherwise, we would need request context here, which we don't have.
connection = self.task_template.interface_connection
logger.debug('Connection for ScaleAI: {}'.format(connection))
connector_manager = ConnectorManager(connection=connection, session=self.session)
connector = connector_manager.get_connector_instance()
connector.connect()
# We create a project
scale_ai_project = self.create_scale_ai_project(connector)
external_map = self.create_scale_ai_project_mapping(scale_ai_project, connection)
logger.info(
'ScaleAITaskTemplateAfterLaunchStrategy for Task Template ID: {} completed successfully.'.format(
self.task_template.id))
logger.debug('Proceding to standard task template launch...')
# Now create tasks as usual.
standard_strategy = StandardTaskTemplateAfterLaunchStrategy(
session=self.session,
task_template=self.task_template,
log=self.log
)
standard_strategy.execute_after_launch_strategy()
except Exception as e:
logger.exception(e)
raise(e)
def determine_task_template_post_launch_strategy(self):
"""
This function will be routing to the appropriate launching algorithm depending on
the task template configurations. For example, if an external interface was selected,
the launch strategy will be different than if the standard Diffgram interface.
:return:
"""
logger.debug('Determining post launch strategy...')
# We default to standard launch strategy.
strategy = StandardTaskTemplateAfterLaunchStrategy(
session=self.session,
task_template=self.task_template,
log=self.log)
logger.debug('interface_connection is {}'.format(
self.task_template.interface_connection))
if self.task_template.interface_connection:
interface_connection = self.task_template.interface_connection
logger.debug('integration name is {}'.format(
interface_connection.integration_name))
# If task template has an integration with labelbox. Change the after launch strategy.
if interface_connection.integration_name == 'labelbox':
strategy = LabelboxTaskTemplateAfterLaunchStrategy(
session=self.session,
task_template=self.task_template,
log=self.log)
if interface_connection.integration_name == 'datasaur':
strategy = DatasaurTaskTemplateAfterLaunchStrategy(
session=self.session,
task_template=self.task_template,
log=self.log)
if interface_connection.integration_name == 'scale_ai':
strategy = ScaleAITaskTemplateAfterLaunchStrategy(
session=self.session,
task_template=self.task_template,
log=self.log)
return strategy
def execute_after_launch_strategy(self):
"""
This strategy will attach files from sync directories and creates tasks in
Diffgram for each of them.
:return:
"""
datasaur_project = None
connection = self.task_template.interface_connection
logger.debug('Connection for Datasaur: {}'.format(connection))
connector_manager = ConnectorManager(connection=connection, session=self.session)
connector = connector_manager.get_connector_instance()
connector.connect()
try:
label_data = []
for label_element in self.task_template.label_dict.get('label_file_list_serialized', []):
element = {
'uuid': str(uuid.uuid4()),
'diffgram_label_file': label_element['id'],
'name': '{}'.format(label_element['label']['name']),
'color': label_element['colour']['hex'].upper(),
}
label_data.append(element)
# First we need to build a label set
label_set_result = self.create_datasaur_labelset(label_data, connector)
label_set = label_set_result['result']['createLabelSet']
logger.debug('Created label_set {}'.format(label_set))
if label_set.get('id'):
logger.info('Datasaur Labelset created succesfully ID:'.format(label_set['id']))
ExternalMap.new(
session=self.session,
job=self.task_template,
external_id=label_set['id'],
connection=connection,
diffgram_class_string='',
type='{}_label_set'.format(connection.integration_name),
url='',
add_to_session=True,
flush_session=True
)
# Now save mappings for created labels
for label_element in label_data:
ExternalMap.new(
session=self.session,
job=self.task_template,
file_id=label_element['diffgram_label_file'],
external_id=label_element['uuid'],
connection=connection,
diffgram_class_string='label_file',
type='{}_label'.format(connection.integration_name),
url='',
add_to_session=True,
flush_session=True
)
# Now we create a project
files_to_process = self.task_template.get_attached_files(self.session, type='text')
files_to_process_by_id = {}
if len(files_to_process) == 0:
raise Exception('Task template has no files in attached folder. Stopping Datasaur launch strategy.')
for file in files_to_process:
files_to_process_by_id[str(file.id)] = file
print('files_to_process_by_id', files_to_process_by_id)
result = self.create_datasaur_project(connector, label_set, files_to_process)
logger.debug('Create datasaur Project result: {}'.format(result))
if 'result' in result:
datasaur_project = result['result']
ExternalMap.new(
session=self.session,
job=self.task_template,
external_id=datasaur_project['id'],
connection=connection,
diffgram_class_string='task_template',
type='{}_project'.format(connection.integration_name),
url='https://datasaur.ai/projects/{}/'.format(datasaur_project['id']),
add_to_session=True,
flush_session=True,
)
logger.debug('Created Datasaur Project.')
# Save file ID's mappings
project_files_results = self.get_project_files_list(connector, datasaur_project)
print('qweqwe', project_files_results)
project_files = project_files_results['result']['documents']
for file in project_files:
diffgram_file = files_to_process_by_id[file['name']]
ExternalMap.new(
session=self.session,
job=self.task_template,
external_id=file['id'],
file=diffgram_file,
connection=connection,
diffgram_class_string='file',
type='{}_file'.format(connection.integration_name),
url='',
add_to_session=True,
flush_session=True,
)
# Now create tasks as usual.
logger.info(
'DatasaurTaskTemplateAfterLaunchStrategy for Task Template ID: {} completed successfully.'.format(
self.task_template.id))
logger.debug('Proceding to standard task template launch...')
standard_strategy = StandardTaskTemplateAfterLaunchStrategy(
session=self.session,
task_template=self.task_template,
log=self.log
)
standard_strategy.execute_after_launch_strategy()
else:
logger.error('Error from connector: Rolling back project creation...')
raise Exception(result)
except Exception as e:
logger.error('Error during datasaur launch strategy. {}'.format(traceback.format_exc()))
if datasaur_project:
logger.error('Rolling back project creation...')
result = connector.put_data({
'action_type': 'delete_project',
'project_id': datasaur_project['id'],
'event_data': {},
})
raise e