def new_external_export(session, project, export_id, version=None, working_dir=None, use_request_context=True): """ Create a new export data file This is run on first load session, session object project, project object Designed for external consumptions returns {"success" : True} if successfully Security model this is an internal function export web DOES the validation Job_permissions.check_job_after_project_already_valid() """ logger.info("[Export processor] Started") result = False start_time = time.time() export = session.query(Export).filter(Export.id == export_id).first() member = None if use_request_context: user = User.get(session) export.user = user if user: member = user.member else: client_id = request.authorization.get('username', None) auth = Auth_api.get(session, client_id) member = auth.member session.add(export) if export.source == "task": if export.task and export.task.file: # Caution export.task not task file_list = [export.task.file] # While job could be None and still get files # if we do have a job id we may want to get # files not replaced in the directory yet. if export.source == "job": file_list = WorkingDirFileLink.file_list( session=session, limit=None, root_files_only=True, job_id=export.job_id, ann_is_complete=export.ann_is_complete) if export.source == "directory": # Question, why are we declaring this here? # Doesn't really make sense as export already has # it when created? export.working_dir_id = working_dir.id file_list = WorkingDirFileLink.file_list( session=session, working_dir_id=working_dir.id, limit=None, root_files_only=True, ann_is_complete=export.ann_is_complete) result, annotations = annotation_export_core(session=session, project=project, export=export, file_list=file_list) if result is False or result is None: return False filename = generate_file_name_from_export(export, session) if export.kind == "Annotations": export.yaml_blob_name = settings.EXPORT_DIR + \ str(export.id) + filename + '.yaml' export.json_blob_name = settings.EXPORT_DIR + \ str(export.id) + filename + '.json' try: yaml_data = yaml.dump(annotations, default_flow_style=False) data_tools.upload_from_string(export.yaml_blob_name, yaml_data, content_type='text/yaml', bucket_type='ml') except Exception as exception: trace_data = traceback.format_exc() logger.error("[Export, YAML] {}".format(str(exception))) logger.error(trace_data) json_data = json.dumps(annotations) data_tools.upload_from_string(export.json_blob_name, json_data, content_type='text/json', bucket_type='ml') end_time = time.time() logger.info("[Export processor] ran in {}".format(end_time - start_time)) Event.new(kind="export_generation", session=session, member=member, success=result, project_id=project.id, run_time=end_time - start_time) return True, annotations
def __send_export(self, opts): spec_list = [{'project_string_id': dict}] log = regular_log.default() log, input = regular_input.input_check_many(untrusted_input = self.config_data, spec_list = spec_list, log = log) if len(log["error"].keys()) >= 1: return {'log': log} spec_list = [ {'path': str}, {"format": { 'default': 'JSON', 'kind': str, 'valid_values_list': ['JSON', 'YAML'] }}, {'export_id': str}, {'bucket_name': str}, ] log = regular_log.default() log, input = regular_input.input_check_many(untrusted_input = opts, spec_list = spec_list, log = log, string_len_not_zero = False) if len(log["error"].keys()) >= 1: return {'log': log} if not opts['path'].endswith('/') and opts['path'] != '': log['error']['path'] = 'Path on bucket must be a folder, not a filename.' return log with sessionMaker.session_scope() as session: project = Project.get_by_string_id(session, self.config_data['project_string_id']) member = session.query(Member).filter(Member.user_id == opts['event_data']['request_user']).first() export = session.query(Export).filter(Export.id == opts['export_id']).first() # Check perms and export status. export_check_result = check_export_permissions_and_status(export, self.config_data['project_string_id'], session) if len(export_check_result['error'].keys()) > 1: return export_check_result result = export_view_core( export = export, format = opts['format'], return_type = 'bytes') filename = generate_file_name_from_export(export, session) if opts['path'] != '': key = '{}{}.{}'.format(opts['path'], filename, opts['format'].lower()) else: key = '{}.{}'.format(filename, opts['format'].lower()) file = io.BytesIO(result) blob_client = self.connection_client.get_blob_client(container = opts['bucket_name'], blob = key) content_type = mimetypes.guess_type(filename)[0] my_content_settings = ContentSettings(content_type = content_type) blob_client.upload_blob(file, content_settings = my_content_settings) log = regular_log.default() log['opts'] = opts Event.new( session = session, member_id = opts['event_data']['request_user'], kind = 'microsoft_azure_new_export_success', description = 'New cloud export for {}{}'.format(opts['path'], filename), error_log = opts, member = member, project_id = project.id, success = True ) return {'result': True}