def new_external_export(session,
project,
export_id,
version=None,
working_dir=None,
use_request_context=True):
"""
Create a new export data file
This is run on first load
session, session object
project, project object
Designed for external consumptions
returns {"success" : True} if successfully
Security model
this is an internal function
export web DOES the validation
Job_permissions.check_job_after_project_already_valid()
"""
logger.info("[Export processor] Started")
result = False
start_time = time.time()
export = session.query(Export).filter(Export.id == export_id).first()
member = None
if use_request_context:
user = User.get(session)
export.user = user
if user:
member = user.member
else:
client_id = request.authorization.get('username', None)
auth = Auth_api.get(session, client_id)
member = auth.member
session.add(export)
if export.source == "task":
if export.task and export.task.file:
# Caution export.task not task
file_list = [export.task.file]
# While job could be None and still get files
# if we do have a job id we may want to get
# files not replaced in the directory yet.
if export.source == "job":
file_list = WorkingDirFileLink.file_list(
session=session,
limit=None,
root_files_only=True,
job_id=export.job_id,
ann_is_complete=export.ann_is_complete)
if export.source == "directory":
# Question, why are we declaring this here?
# Doesn't really make sense as export already has
# it when created?
export.working_dir_id = working_dir.id
file_list = WorkingDirFileLink.file_list(
session=session,
working_dir_id=working_dir.id,
limit=None,
root_files_only=True,
ann_is_complete=export.ann_is_complete)
result, annotations = annotation_export_core(session=session,
project=project,
export=export,
file_list=file_list)
if result is False or result is None:
return False
filename = generate_file_name_from_export(export, session)
if export.kind == "Annotations":
export.yaml_blob_name = settings.EXPORT_DIR + \
str(export.id) + filename + '.yaml'
export.json_blob_name = settings.EXPORT_DIR + \
str(export.id) + filename + '.json'
try:
yaml_data = yaml.dump(annotations, default_flow_style=False)
data_tools.upload_from_string(export.yaml_blob_name,
yaml_data,
content_type='text/yaml',
bucket_type='ml')
except Exception as exception:
trace_data = traceback.format_exc()
logger.error("[Export, YAML] {}".format(str(exception)))
logger.error(trace_data)
json_data = json.dumps(annotations)
data_tools.upload_from_string(export.json_blob_name,
json_data,
content_type='text/json',
bucket_type='ml')
end_time = time.time()
logger.info("[Export processor] ran in {}".format(end_time - start_time))
Event.new(kind="export_generation",
session=session,
member=member,
success=result,
project_id=project.id,
run_time=end_time - start_time)
return True, annotations
def __send_export(self, opts):
spec_list = [{'project_string_id': dict}]
log = regular_log.default()
log, input = regular_input.input_check_many(untrusted_input = self.config_data,
spec_list = spec_list,
log = log)
if len(log["error"].keys()) >= 1:
return {'log': log}
spec_list = [
{'path': str},
{"format": {
'default': 'JSON',
'kind': str,
'valid_values_list': ['JSON', 'YAML']
}},
{'export_id': str},
{'bucket_name': str},
]
log = regular_log.default()
log, input = regular_input.input_check_many(untrusted_input = opts,
spec_list = spec_list,
log = log,
string_len_not_zero = False)
if len(log["error"].keys()) >= 1:
return {'log': log}
if not opts['path'].endswith('/') and opts['path'] != '':
log['error']['path'] = 'Path on bucket must be a folder, not a filename.'
return log
with sessionMaker.session_scope() as session:
project = Project.get_by_string_id(session, self.config_data['project_string_id'])
member = session.query(Member).filter(Member.user_id == opts['event_data']['request_user']).first()
export = session.query(Export).filter(Export.id == opts['export_id']).first()
# Check perms and export status.
export_check_result = check_export_permissions_and_status(export,
self.config_data['project_string_id'],
session)
if len(export_check_result['error'].keys()) > 1:
return export_check_result
result = export_view_core(
export = export,
format = opts['format'],
return_type = 'bytes')
filename = generate_file_name_from_export(export, session)
if opts['path'] != '':
key = '{}{}.{}'.format(opts['path'], filename, opts['format'].lower())
else:
key = '{}.{}'.format(filename, opts['format'].lower())
file = io.BytesIO(result)
blob_client = self.connection_client.get_blob_client(container = opts['bucket_name'], blob = key)
content_type = mimetypes.guess_type(filename)[0]
my_content_settings = ContentSettings(content_type = content_type)
blob_client.upload_blob(file, content_settings = my_content_settings)
log = regular_log.default()
log['opts'] = opts
Event.new(
session = session,
member_id = opts['event_data']['request_user'],
kind = 'microsoft_azure_new_export_success',
description = 'New cloud export for {}{}'.format(opts['path'], filename),
error_log = opts,
member = member,
project_id = project.id,
success = True
)
return {'result': True}