def labelbox_web_hook_manager():
"""
Webhook for receiving data on Diffgram once finished on labelbox.
# NOTE: Labelbox does not supportText or dropdown classifications in export for videos.
:return:
"""
# First check if secret is correct
payload = request.data
secret = settings.LABEL_BOX_SECRET
log = regular_log.default()
computed_signature = hmac.new(bytearray(secret.encode('utf-8')),
msg=payload,
digestmod=hashlib.sha1).hexdigest()
if request.headers['X-Hub-Signature'] != 'sha1=' + computed_signature:
error = 'Error: computed_signature does not match signature provided in the headers'
logger.error(
'Error: computed_signature does not match signature provided in the headers'
)
return error
with sessionMaker.session_scope() as session:
labelbox_event = request.headers['X-Labelbox-Event']
payload = request.json
logger.debug('Payload for labelbox webhooks: {}'.format(payload))
labelbox_project_id = payload['project']['id']
project_external_mapping = ExternalMap.get(
session=session,
external_id=labelbox_project_id,
type='labelbox',
diffgram_class_string='task_template')
if project_external_mapping:
task_template = Job.get_by_id(session,
project_external_mapping.job_id)
if task_template:
connection = task_template.interface_connection
logger.debug('Connection for labelbox: {}'.format(connection))
connector_manager = ConnectorManager(connection=connection,
session=session)
connector = connector_manager.get_connector_instance()
connector.connect()
sync_manager = LabelBoxSyncManager(
session=session,
task_template=task_template,
labelbox_project=None,
log=log,
labelbox_connector=connector)
sync_manager.handle_task_creation_hook(payload)
return jsonify({'message': 'OK.'})
else:
log['error']['task_template'] = 'Task template not found.'
return jsonify(log)
else:
log['error'][
'labelbox_project'] = 'Labelbox external mapping not found.'
return jsonify(log)
def sync_projects_for_task_template(self, task_template):
connection = task_template.interface_connection
connector_manager = ConnectorManager(connection=connection, session=self.session)
datasaur_connector = connector_manager.get_connector_instance()
datasaur_connector.connect()
project_map = ExternalMap.get(
session = self.session,
job_id = task_template.id,
diffgram_class_string = 'task_template',
type = 'datasaur_project',
return_kind = 'first')
if not project_map:
logger.error('Could not find external map for task template {}'.format(task_template.id))
return
# Fetch all completed projects
project_data = datasaur_connector.fetch_data({
'action_type': 'get_project',
'event_data': {},
'project_id': project_map.external_id
})
# For each project, generate a export for all files.
if 'log' in project_data:
if 'error' in project_data['log']:
logger.error('Error fetching datasaur project {}'.format(project_data['log']['error']))
logger.error('Datasaur project ID {} not found. Maybe was deleted?'.format(project_map.external_id))
return
datasaur_project = project_data['result']
logger.debug('Fetched project: {}'.format(datasaur_project))
if project_map:
task_template = project_map.job
if datasaur_project['status'] != 'COMPLETE':
logger.debug('Datasaur project {} is not completed. Skipping...'.format(datasaur_project['id']))
return
# Now get All files from the project
files_map = ExternalMap.get(
return_kind = 'all',
session = self.session,
job_id = task_template.id,
diffgram_class_string = 'file',
type = 'datasaur_file')
for file_map in files_map:
diffgram_file = file_map.file
datasaur_file_id = file_map.external_id
logger.debug('Syncing File from Datasaur {}'.format(diffgram_file.id))
self.fetch_instances_from_file(task_template, diffgram_file, datasaur_file_id, datasaur_connector)
else:
logger.error('Could not find external map for task_template {}'.format(task_template.id))
def send_task_to_scale_ai():
"""
Webhook for receiving data on Diffgram once finished on labelbox.
# NOTE: Labelbox does not supportText or dropdown classifications in export for videos.
:return:
"""
# First check if secret is correct
spec_list = [{'task_id': dict}, {'project_string_id': str}]
log, input_data, untrusted_input = regular_input.master(
request=request, spec_list=spec_list)
log = regular_log.default()
with sessionMaker.session_scope() as session:
task_id = input_data['task_id']
task = Task.get_by_id(session, task_id=task_id)
if task:
task_template = task.job
connection = task_template.interface_connection
logger.debug('Connection for ScaleAI: {}'.format(connection))
connector_manager = ConnectorManager(connection=connection,
session=session)
connector = connector_manager.get_connector_instance()
connector.connect()
scale_ai_sync_manager = ScaleAISyncManager(
task_template=task_template,
scale_ai_connector=connector,
log=log,
session=session)
scale_ai_task, log = scale_ai_sync_manager.send_diffgram_task(task)
logger.debug('Scale AI create result: {} || {}'.format(
scale_ai_task, log))
if not scale_ai_task:
print('aaaaaaaaaaaaaa', log)
return jsonify(log=log), 400
return jsonify({
'message': 'OK.',
'scale_ai_task_id': scale_ai_task.id
})
else:
log['error']['task_id'] = 'Task not found.'
return jsonify(log)
def execute_after_launch_strategy(self):
"""
This strategy will create a project on scale AI for user to send tasks into
it when task is launched.
:return:
"""
scale_ai_project = None
try:
# TODO: ADD LABELBOX LOGIC HERE
# We don't check perms here because we assume this was checked on the task template creation.
# Otherwise, we would need request context here, which we don't have.
connection = self.task_template.interface_connection
logger.debug('Connection for ScaleAI: {}'.format(connection))
connector_manager = ConnectorManager(connection=connection, session=self.session)
connector = connector_manager.get_connector_instance()
connector.connect()
# We create a project
scale_ai_project = self.create_scale_ai_project(connector)
external_map = self.create_scale_ai_project_mapping(scale_ai_project, connection)
logger.info(
'ScaleAITaskTemplateAfterLaunchStrategy for Task Template ID: {} completed successfully.'.format(
self.task_template.id))
logger.debug('Proceding to standard task template launch...')
# Now create tasks as usual.
standard_strategy = StandardTaskTemplateAfterLaunchStrategy(
session=self.session,
task_template=self.task_template,
log=self.log
)
standard_strategy.execute_after_launch_strategy()
except Exception as e:
logger.exception(e)
raise(e)
def get_connector(connector_id, session, input=None, check_perms=True):
connection_operations = Connection_Operations(session=session,
member=None,
connection_id=connector_id)
connection = connection_operations.get_existing_connection(connector_id)
if check_perms:
connection_operations.validate_existing_connection_id_permissions()
if input is None:
conn_manager = ConnectorManager(connection=connection, session=session)
connector_class = conn_manager.get_connector_class()
auth_data = {
'client_email': connection.account_email,
'client_id': connection.private_id,
'client_secret': connection_operations.get_secret(),
'project_id': connection.project_id_external
}
else:
integration_name = input.get('integration_name') if input.get(
'integration_name') is not None else connection.integration_name
conn_manager = ConnectorManager(integration_name=integration_name)
connector_class = conn_manager.get_connector_class()
auth_data = {
'client_email':
input.get('account_email') if input.get('account_email')
is not None else connection.account_email,
'client_id':
input.get('private_id')
if input.get('private_id') is not None else connection.private_id,
'client_secret':
input.get('private_secret') if input.get('private_secret')
is not None else connection_operations.get_secret(),
'project_id':
input.get('project_id_external')
if input.get('project_id_external') is not None else
connection.project_id_external
}
config_data = {'project_string_id': connection.project.project_string_id}
connector = connector_class(auth_data=auth_data, config_data=config_data)
if len(connection_operations.log["error"].keys()) >= 1:
return connection_operations.log, False
return connector, True
def execute_after_launch_strategy(self):
"""
This strategy will attach files from sync directories and creates tasks in
Diffgram for each of them.
:return:
"""
labelbox_project = None
try:
# TODO: ADD LABELBOX LOGIC HERE
# We don't check perms here because we assume this was checked on the task template creation.
# Otherwise, we would need request context here, which we don't have.
connection = self.task_template.interface_connection
logger.debug('Connection for labelbox: {}'.format(connection))
connector_manager = ConnectorManager(connection=connection, session=self.session)
connector = connector_manager.get_connector_instance()
connector.connect()
# First we create a project
labelbox_project = self.create_labelbox_project(connector)
external_map = self.create_labelbox_project_mapping(labelbox_project, connection)
# Now create labelbox frontend
frontend = self.get_labelbox_frontend(connector)
# Next, we specify an ontology based on the label_templates of the task template.
# Note: for now all classifications wil be nested.
ontology = self.create_ontology(connector, frontend, labelbox_project)
self.save_label_instance_ontology_mapping(ontology, connection)
# Then we create a dataset for the project, where we'll add all the files
labelbox_sync_manager = LabelBoxSyncManager(
session=self.session,
labelbox_project=labelbox_project,
task_template=self.task_template,
log=self.log,
labelbox_connector=connector
)
# We first set the webhook for this task template.
labelbox_sync_manager.set_webhook_for_task_template()
# Then we start adding all files to labelbox.
labelbox_sync_manager.send_all_files_in_task_template()
logger.info(
'LabelboxTaskTemplateAfterLaunchStrategy for Task Template ID: {} completed successfully.'.format(
self.task_template.id))
logger.debug('Proceding to standard task template launch...')
# Now create tasks as usual.
standard_strategy = StandardTaskTemplateAfterLaunchStrategy(
session=self.session,
task_template=self.task_template,
log=self.log
)
standard_strategy.execute_after_launch_strategy()
except Exception as e:
logger.exception(e)
# Rollback Delete Labelbox Project
if labelbox_project is not None:
if self.task_template.default_external_map:
self.task_template.default_external_map = None
self.session.delete(self.task_template.default_external_map)
labelbox_project.delete()
logger.debug('ROLLBACK. Deleted Labelbox Project {}'.format(labelbox_project.uid))
# Allow time for rate limiter in case of a rate limit exception.
time.sleep(5)
raise e
def execute_after_launch_strategy(self):
"""
This strategy will attach files from sync directories and creates tasks in
Diffgram for each of them.
:return:
"""
datasaur_project = None
connection = self.task_template.interface_connection
logger.debug('Connection for Datasaur: {}'.format(connection))
connector_manager = ConnectorManager(connection=connection, session=self.session)
connector = connector_manager.get_connector_instance()
connector.connect()
try:
label_data = []
for label_element in self.task_template.label_dict.get('label_file_list_serialized', []):
element = {
'uuid': str(uuid.uuid4()),
'diffgram_label_file': label_element['id'],
'name': '{}'.format(label_element['label']['name']),
'color': label_element['colour']['hex'].upper(),
}
label_data.append(element)
# First we need to build a label set
label_set_result = self.create_datasaur_labelset(label_data, connector)
label_set = label_set_result['result']['createLabelSet']
logger.debug('Created label_set {}'.format(label_set))
if label_set.get('id'):
logger.info('Datasaur Labelset created succesfully ID:'.format(label_set['id']))
ExternalMap.new(
session=self.session,
job=self.task_template,
external_id=label_set['id'],
connection=connection,
diffgram_class_string='',
type='{}_label_set'.format(connection.integration_name),
url='',
add_to_session=True,
flush_session=True
)
# Now save mappings for created labels
for label_element in label_data:
ExternalMap.new(
session=self.session,
job=self.task_template,
file_id=label_element['diffgram_label_file'],
external_id=label_element['uuid'],
connection=connection,
diffgram_class_string='label_file',
type='{}_label'.format(connection.integration_name),
url='',
add_to_session=True,
flush_session=True
)
# Now we create a project
files_to_process = self.task_template.get_attached_files(self.session, type='text')
files_to_process_by_id = {}
if len(files_to_process) == 0:
raise Exception('Task template has no files in attached folder. Stopping Datasaur launch strategy.')
for file in files_to_process:
files_to_process_by_id[str(file.id)] = file
print('files_to_process_by_id', files_to_process_by_id)
result = self.create_datasaur_project(connector, label_set, files_to_process)
logger.debug('Create datasaur Project result: {}'.format(result))
if 'result' in result:
datasaur_project = result['result']
ExternalMap.new(
session=self.session,
job=self.task_template,
external_id=datasaur_project['id'],
connection=connection,
diffgram_class_string='task_template',
type='{}_project'.format(connection.integration_name),
url='https://datasaur.ai/projects/{}/'.format(datasaur_project['id']),
add_to_session=True,
flush_session=True,
)
logger.debug('Created Datasaur Project.')
# Save file ID's mappings
project_files_results = self.get_project_files_list(connector, datasaur_project)
print('qweqwe', project_files_results)
project_files = project_files_results['result']['documents']
for file in project_files:
diffgram_file = files_to_process_by_id[file['name']]
ExternalMap.new(
session=self.session,
job=self.task_template,
external_id=file['id'],
file=diffgram_file,
connection=connection,
diffgram_class_string='file',
type='{}_file'.format(connection.integration_name),
url='',
add_to_session=True,
flush_session=True,
)
# Now create tasks as usual.
logger.info(
'DatasaurTaskTemplateAfterLaunchStrategy for Task Template ID: {} completed successfully.'.format(
self.task_template.id))
logger.debug('Proceding to standard task template launch...')
standard_strategy = StandardTaskTemplateAfterLaunchStrategy(
session=self.session,
task_template=self.task_template,
log=self.log
)
standard_strategy.execute_after_launch_strategy()
else:
logger.error('Error from connector: Rolling back project creation...')
raise Exception(result)
except Exception as e:
logger.error('Error during datasaur launch strategy. {}'.format(traceback.format_exc()))
if datasaur_project:
logger.error('Rolling back project creation...')
result = connector.put_data({
'action_type': 'delete_project',
'project_id': datasaur_project['id'],
'event_data': {},
})
raise e