def receive_uploaded_data(self, data_id=None, **kwargs): file_server_root = config.instance.file_server_root resource_target_path = tempfile.mktemp(dir=file_server_root) try: self._save_file_locally_and_extract_inputs( resource_target_path, self._get_data_url_key(), self._get_kind()) with self.Caravan(resource_target_path) as caravan_instance: caravan_instance.init_metadata() plugins = self._prepare_and_process_doc( file_server_root, resource_target_path, caravan_instance=caravan_instance, **kwargs) docs = [] for doc, plugin_dir in plugins: self._move_archive_to_uploaded_dir( doc.id, file_server_root, plugin_dir, ) docs.append(doc) return docs, 201 finally: remove(resource_target_path)
def receive_uploaded_data(self, data_id=None, **kwargs): file_server_root = config.instance.file_server_root resource_target_path = tempfile.mktemp(dir=file_server_root) try: additional_inputs = self._save_file_locally_and_extract_inputs( resource_target_path, self._get_data_url_key(), self._get_kind()) doc, dest_file_name = self._prepare_and_process_doc( data_id, file_server_root, resource_target_path, additional_inputs=additional_inputs, **kwargs) if not os.path.isfile(resource_target_path): # if the archive is a folder, we're copying its content, # so there is no meaning to a specific archive file name... dest_file_name = None self._move_archive_to_uploaded_dir(doc.id, file_server_root, resource_target_path, dest_file_name=dest_file_name) return doc, 201 finally: remove(resource_target_path)
def receive_uploaded_data(self, data_id=None, **kwargs): file_server_root = config.instance.file_server_root resource_target_path = tempfile.mktemp(dir=file_server_root) try: additional_inputs = self._save_file_locally_and_extract_inputs( resource_target_path, self._get_data_url_key(), self._get_kind()) self._prepare_and_process_doc(data_id, file_server_root, resource_target_path, additional_inputs=additional_inputs, **kwargs) return "", 204 finally: remove(resource_target_path)
def upload_archive_to_file_server(self, blueprint_id): file_server_root = config.instance.file_server_root archive_target_path = tempfile.mktemp() try: self._save_file_locally_and_extract_inputs(archive_target_path, None, self._get_kind()) self._move_archive_to_uploaded_dir(blueprint_id, file_server_root, archive_target_path) except Exception as e: sm = get_resource_manager().sm blueprint = sm.get(Blueprint, blueprint_id) blueprint.state = BlueprintUploadState.FAILED_UPLOADING blueprint.error = str(e) sm.update(blueprint) self.cleanup_blueprint_archive_from_file_server( blueprint_id, blueprint.tenant.name) raise finally: remove(archive_target_path)
def extract_blueprint_archive_to_file_server(self, blueprint_id, tenant): sm = get_resource_manager().sm file_server_root = config.instance.file_server_root local_path = os.path.join(config.instance.file_server_root, FILE_SERVER_UPLOADED_BLUEPRINTS_FOLDER, tenant, blueprint_id) for arc_type in SUPPORTED_ARCHIVE_TYPES: # attempting to find the archive file on the file system local_file_path = os.path.join( local_path, '{0}.{1}'.format(blueprint_id, arc_type)) if os.path.isfile(local_file_path): break else: error_msg = "Could not find blueprint's archive; " \ "Blueprint ID: {0}".format(blueprint_id) blueprint = sm.get(Blueprint, blueprint_id) blueprint.state = \ BlueprintUploadState.FAILED_EXTRACTING_TO_FILE_SERVER blueprint.error = error_msg sm.update(blueprint) raise manager_exceptions.NotFoundError(error_msg) try: app_dir = self._extract_file_to_file_server( local_file_path, file_server_root) except Exception as e: blueprint = sm.get(Blueprint, blueprint_id) blueprint.state = \ BlueprintUploadState.FAILED_EXTRACTING_TO_FILE_SERVER blueprint.error = str(e) sm.update(blueprint) remove(local_path) raise e tenant_dir = os.path.join(file_server_root, FILE_SERVER_BLUEPRINTS_FOLDER, tenant) mkdirs(tenant_dir) shutil.move(os.path.join(file_server_root, app_dir), os.path.join(tenant_dir, blueprint_id)) self._process_plugins(file_server_root, blueprint_id)
def receive_uploaded_data(self, data_id=None, **kwargs): file_server_root = config.instance.file_server_root resource_target_path = tempfile.mktemp(dir=file_server_root) try: self._save_file_locally_and_extract_inputs( resource_target_path, self._get_data_url_key(), self._get_kind()) plugins = self._prepare_and_process_doc( file_server_root, resource_target_path, **kwargs) docs = [] for doc, plugin_dir in plugins: self._move_archive_to_uploaded_dir( doc.id, file_server_root, plugin_dir, ) docs.append(doc) return docs, 201 finally: remove(resource_target_path)
def __exit__(self, *_): remove(self._tempdir)
def cleanup_blueprint_archive_from_file_server(blueprint_id, tenant): remove( os.path.join(config.instance.file_server_root, FILE_SERVER_UPLOADED_BLUEPRINTS_FOLDER, tenant, blueprint_id))
def patch(self, blueprint_id, **kwargs): """ Update a blueprint. Used for updating the blueprint's state (and error) while uploading, and updating the blueprint's other attributes upon a successful upload. This method is for internal use only. """ if not request.json: raise IllegalActionError('Update a blueprint request must include ' 'at least one parameter to update') request_schema = { 'plan': { 'type': dict, 'optional': True }, 'description': { 'type': text_type, 'optional': True }, 'main_file_name': { 'type': text_type, 'optional': True }, 'visibility': { 'type': text_type, 'optional': True }, 'state': { 'type': text_type, 'optional': True }, 'error': { 'type': text_type, 'optional': True }, 'error_traceback': { 'type': text_type, 'optional': True }, 'labels': { 'type': list, 'optional': True } } request_dict = rest_utils.get_json_and_verify_params(request_schema) invalid_params = set(request_dict.keys()) - set(request_schema.keys()) if invalid_params: raise BadParametersError("Unknown parameters: {}".format( ','.join(invalid_params))) sm = get_storage_manager() rm = get_resource_manager() blueprint = sm.get(models.Blueprint, blueprint_id) # if finished blueprint validation - cleanup DB entry # and uploaded blueprints folder if blueprint.state == BlueprintUploadState.VALIDATING: uploaded_blueprint_path = join( config.instance.file_server_root, FILE_SERVER_UPLOADED_BLUEPRINTS_FOLDER, blueprint.tenant.name, blueprint.id) remove(uploaded_blueprint_path) sm.delete(blueprint) return blueprint # set blueprint visibility visibility = request_dict.get('visibility') if visibility: if visibility not in VisibilityState.STATES: raise BadParametersError( "Invalid visibility: `{0}`. Valid visibility's values " "are: {1}".format(visibility, VisibilityState.STATES)) blueprint.visibility = visibility # set other blueprint attributes. if 'plan' in request_dict: blueprint.plan = request_dict['plan'] if 'description' in request_dict: blueprint.description = request_dict['description'] if 'main_file_name' in request_dict: blueprint.main_file_name = request_dict['main_file_name'] provided_labels = request_dict.get('labels') if request_dict.get('plan'): dsl_labels = request_dict['plan'].get('labels', {}) csys_obj_parents = dsl_labels.get('csys-obj-parent') if csys_obj_parents: dep_parents = csys_obj_parents['values'] missing_parents = rm.get_missing_deployment_parents( dep_parents) if missing_parents: raise DeploymentParentNotFound( 'Blueprint {0}: is referencing deployments' ' using label `csys-obj-parent` that does not exist, ' 'make sure that deployment(s) {1} exist before ' 'creating blueprint'.format(blueprint.id, ','.join(missing_parents))) # set blueprint state state = request_dict.get('state') if state: if state not in BlueprintUploadState.STATES: raise BadParametersError( "Invalid state: `{0}`. Valid blueprint state values are: " "{1}".format(state, BlueprintUploadState.STATES)) if (state != BlueprintUploadState.UPLOADED and provided_labels is not None): raise ConflictError( 'Blueprint labels can be created only if the provided ' 'blueprint state is {0}'.format( BlueprintUploadState.UPLOADED)) blueprint.state = state blueprint.error = request_dict.get('error') blueprint.error_traceback = request_dict.get('error_traceback') # On finalizing the blueprint upload, extract archive to file # server if state == BlueprintUploadState.UPLOADED: UploadedBlueprintsManager(). \ extract_blueprint_archive_to_file_server( blueprint_id=blueprint_id, tenant=blueprint.tenant.name) _create_blueprint_labels(blueprint, provided_labels) # If failed for any reason, cleanup the blueprint archive from # server elif state in BlueprintUploadState.FAILED_STATES: UploadedBlueprintsManager(). \ cleanup_blueprint_archive_from_file_server( blueprint_id=blueprint_id, tenant=blueprint.tenant.name) else: # Updating the blueprint not as part of the upload process if provided_labels is not None: if blueprint.state != BlueprintUploadState.UPLOADED: raise ConflictError( 'Blueprint labels can only be updated if the blueprint' ' was uploaded successfully') rm = get_resource_manager() labels_list = rest_utils.get_labels_list(provided_labels) rm.update_resource_labels(models.BlueprintLabel, blueprint, labels_list) blueprint.updated_at = get_formatted_timestamp() return sm.update(blueprint)