def unpack(ext: str, source: IO[bytes], dest_path: str): """Unpack the archive |source| to |dest_path|. Args: ext (str): Extension of the archive. source (IO[bytes]): File handle to the source. dest_path ([type]): Destination path to unpack to. """ try: if ext == '.tar.gz' or ext == '.tgz': un_tar_directory(source, dest_path, 'gz') elif ext == '.tar.bz2': un_tar_directory(source, dest_path, 'bz2') elif ext == '.bz2': un_bz2_file(source, dest_path) elif ext == '.gz': with open(dest_path, 'wb') as f: shutil.copyfileobj(un_gzip_stream(source), f) elif ext == '.zip': unzip_directory(source, dest_path) else: raise UsageError('Not an archive.') except (tarfile.TarError, IOError) as e: logging.error("Invalid archive upload: failed to unpack archive: %s", e) raise UsageError('Invalid archive upload: failed to unpack archive.')
def construct(cls, targets, command, metadata, owner_id, uuid=None, data_hash=None, state=State.CREATED): if not uuid: uuid = spec_util.generate_uuid() # Check that targets does not include both keyed and anonymous targets. if len(targets) > 1 and any(key == '' for key, value in targets): raise UsageError('Must specify keys when packaging multiple targets!') if not isinstance(command, basestring): raise UsageError('%r is not a valid command!' % (command,)) # List the dependencies of this bundle on its targets. dependencies = [] for (child_path, (parent_uuid, parent_path)) in targets: dependencies.append({ 'child_uuid': uuid, 'child_path': child_path, 'parent_uuid': parent_uuid, 'parent_path': parent_path, }) return super(RunBundle, cls).construct({ 'uuid': uuid, 'bundle_type': cls.BUNDLE_TYPE, 'command': command, 'data_hash': data_hash, 'state': state, 'metadata': metadata, 'dependencies': dependencies, 'owner_id': owner_id, })
def get(dep): # Return the key key, val = parse_key_target(dep) if key == '': # key only matches empty string if ':' present _, _, bundle, subpath = parse_target_spec(val) key = subpath if subpath is not None else bundle elif key is None: # key only returns None if ':' not present in original spec key = val2key[val] if val in val2key else 'b' + str( len(target_spec) + 1) if val not in val2key: val2key[val] = key if key in key2val: if key2val[key] != val: raise UsageError( 'key %s exists with multiple values: %s and %s' % (key, key2val[key], val)) else: if key is None: raise UsageError( 'target_spec is empty. Please provide a valid target_spec in the format of {}.' .format(RUN_TARGET_SPEC_FORMAT)) key2val[key] = val target_spec.append(key + ':' + val) return key
def parse_metadata_form(bundle_subclass, form_result): ''' Parse the result of a form template produced out request_missing_metadata. ''' metadata_specs = bundle_subclass.get_user_defined_metadata() metadata_types = {spec.key: spec.type for spec in metadata_specs} result = {} for line in form_result: line = line.strip() if line != '' and not line.startswith('//'): if ':' not in line: # TODO: don't delete everything; go back to the editor and show the error message raise UsageError('Malformatted line (no colon): %s' % (line,)) (metadata_key, remainder) = line.split(':', 1) remainder = remainder.strip() if remainder == '': remainder = None # TODO: handle multiple lines if metadata_key not in metadata_types: raise UsageError('Unexpected metadata key: %s' % (metadata_key,)) metadata_type = metadata_types[metadata_key] if metadata_type == list: result[metadata_key] = remainder.split() if remainder else [] elif metadata_type == basestring: result[metadata_key] = remainder else: try: result[metadata_key] = metadata_type(remainder) if remainder != None else None except: raise UsageError('Invalid value %s for type %s' % (remainder, metadata_type)) if 'name' not in result: raise UsageError('No name specified; aborting') return result
def unpack(ext, source, dest_path): """ Unpack the archive |source| to |dest_path|. Note: |source| can be a file handle or a path. |ext| contains the extension of the archive. """ close_source = False try: if isinstance(source, str): source = open(source, 'rb') close_source = True if ext == '.tar.gz' or ext == '.tgz': un_tar_directory(source, dest_path, 'gz') elif ext == '.tar.bz2': un_tar_directory(source, dest_path, 'bz2') elif ext == '.bz2': un_bz2_file(source, dest_path) elif ext == '.gz': with open(dest_path, 'wb') as f: shutil.copyfileobj(un_gzip_stream(source), f) elif ext == '.zip': unzip_directory(source, dest_path) else: raise UsageError('Not an archive.') except (tarfile.TarError, IOError): raise UsageError('Invalid archive upload.') finally: if close_source: source.close()
def inner(*args, **kwargs): import time time_delay = 1 if self.verbose >= 2: print 'remote_bundle_client: %s %s %s' % (command, args, kwargs) while True: try: return getattr(self.proxy, command)(*args, **kwargs) except xmlrpclib.ProtocolError, e: raise UsageError("Could not authenticate on %s: %s" % (host, e)) except xmlrpclib.Fault, e: # Transform server-side UsageErrors into client-side UsageErrors. if 'codalab.common.UsageError' in e.faultString: index = e.faultString.find(':') raise UsageError(e.faultString[index + 1:]) if 'codalab.common.NotFoundError' in e.faultString: index = e.faultString.find(':') raise NotFoundError(e.faultString[index + 1:]) elif 'codalab.common.PermissionError' in e.faultString: index = e.faultString.find(':') raise PermissionError(e.faultString[index + 1:]) elif 'codalab.common.AuthorizationError' in e.faultString: index = e.faultString.find(':') raise AuthorizationError(e.faultString[index + 1:]) else: raise
def validate(self, metadata_specs): ''' Check that this metadata has the correct metadata keys and that it has metadata values of the correct types. ''' expected_keys = set(spec.key for spec in metadata_specs) for key in self._metadata_keys: if key not in expected_keys: raise UsageError('Unexpected metadata key: %s' % (key, )) for spec in metadata_specs: if spec.key in self._metadata_keys: value = getattr(self, spec.key) if spec.type is float and isinstance(value, int): # cast int to float value = float(value) # Validate formatted string fields if issubclass(spec.type, str) and spec.formatting is not None and value: try: if spec.formatting == 'duration': formatting.parse_duration(value) elif spec.formatting == 'size': formatting.parse_size(value) elif spec.formatting == 'date': formatting.parse_datetime(value) except ValueError as e: raise UsageError(str(e)) if value is not None and not isinstance(value, spec.type): raise UsageError( 'Metadata value for %s should be of type %s, was %s (type %s)' % (spec.key, spec.type.__name__, value, type(value).__name__)) elif not spec.generated and not spec.optional: raise UsageError('Missing metadata key: %s' % (spec.key, ))
def get_single_group(model, group_spec, search_fn): ''' Helper function. Resolve a string group_spec to a unique group for the given |search_fn|. Throw an error if zero or more than one group matches. ''' if not group_spec: raise UsageError('Tried to expand empty group_spec!') if spec_util.UUID_REGEX.match(group_spec): groups = search_fn(model, uuid=group_spec) message = "uuid starting with '%s'" % (group_spec, ) elif spec_util.UUID_PREFIX_REGEX.match(group_spec): groups = search_fn(model, uuid=LikeQuery(group_spec + '%')) message = "uuid starting with '%s'" % (group_spec, ) else: spec_util.check_name(group_spec) groups = search_fn(model, name=group_spec) message = "name '%s'" % (group_spec, ) if not groups: raise NotFoundError('Found no group with %s' % (message, )) elif len(groups) > 1: raise UsageError('Found multiple groups with %s:%s' % (message, ''.join('\n uuid=%s' % (group['uuid'], ) for group in groups))) return groups[0]
def unzip(zip_path, temp_path, file_name): ''' Take an absolute path to a zip file |zip_path| and return the path to a file or directory called |file_name| in |temp_path| containing its unzipped contents. Assume the zip file contains one file/directory called |file_name|. If |file_name| is not specified, then return the temp_path itself. ''' path_util.check_isfile(zip_path, 'unzip_directory') if file_name: temp_subpath = os.path.join(temp_path, file_name) else: temp_subpath = temp_path print_util.open_line('Unzipping %s to %s' % (zip_path, temp_subpath)) if os.system("cd %s && unzip -q %s" % (temp_path, zip_path)) != 0: raise UsageError('unzip failed') print_util.clear_line() # Corner case: note that the temp_subpath might not 'exist' because it is a # symlink (which is broken until it's put in the right place). if not os.path.exists(temp_subpath) and not os.path.islink(temp_subpath): raise UsageError('Zip file %s missing %s (%s doesn\'t exist)' % (zip_path, file_name, temp_subpath)) return temp_subpath
def get_worksheet_uuid(model, base_worksheet_uuid, worksheet_spec): """ Resolve a string worksheet_spec to a unique worksheet uuid. If base_worksheet_uuid specified, then try to resolve worksheet_spec in the context of base_worksheet_uuid. """ if not worksheet_spec: raise UsageError('Tried to expand empty worksheet_spec!') if spec_util.UUID_REGEX.match(worksheet_spec): return worksheet_spec if spec_util.UUID_PREFIX_REGEX.match(worksheet_spec): worksheets = model.batch_get_worksheets(fetch_items=False, uuid=LikeQuery(worksheet_spec + '%'), base_worksheet_uuid=base_worksheet_uuid) message = "uuid starting with '%s'" % (worksheet_spec,) else: spec_util.check_name(worksheet_spec) worksheets = model.batch_get_worksheets(fetch_items=False, name=worksheet_spec, base_worksheet_uuid=base_worksheet_uuid) message = "name '%s'" % (worksheet_spec,) if not worksheets: raise NotFoundError('No worksheet found with %s' % (message,)) if len(worksheets) > 1: raise UsageError( 'Found multiple worksheets with %s:%s' % (message, ''.join('\n %s' % (worksheet,) for worksheet in worksheets)) ) return worksheets[0].uuid
def unpack_to_archive(ext: str, source: IO[bytes]) -> IO[bytes]: """Unpack the archive |source| and returns the unpacked fileobj. If |source| is an archive, unpacks to a .tar.gz archive file. If |source| is a non-archive file, unpacks to a .gz file. Args: ext (str): Extension of the source archive. source (IO[bytes]): File handle to the source. Returns: IO[bytes]: File object with the archive. """ try: if ext == '.tar.gz' or ext == '.tgz': return source elif ext == '.tar.bz2': return GzipStream(UnBz2Stream(source)) elif ext == '.bz2': return GzipStream(UnBz2Stream(source)) elif ext == '.gz': return source elif ext == '.zip': return GzipStream(ZipToTarStream(source)) else: raise UsageError('Not an archive.') except (tarfile.TarError, IOError) as e: logging.error("Invalid archive upload: failed to unpack archive: %s", e) raise UsageError('Invalid archive upload: failed to unpack archive.')
def resolve_source(source): # Resolve symlink if desired resolved_source = source if follow_symlinks: resolved_source = os.path.realpath(source) if not os.path.exists(resolved_source): raise UsageError('Broken symlink') elif os.path.islink(source): raise UsageError('Not following symlinks.') return resolved_source
def check_quota(self, need_time=False, need_disk=False): if need_time: if self.time_used >= self.time_quota: raise UsageError( 'Out of time quota: %s' % formatting.ratio_str(formatting.duration_str, self.time_used, self.time_quota)) if need_disk: if self.disk_used >= self.disk_quota: raise UsageError( 'Out of disk quota: %s' % formatting.ratio_str( formatting.size_str, self.disk_used, self.disk_quota))
def delete_worksheet(uuid, force): worksheet = local.model.get_worksheet(uuid, fetch_items=True) check_worksheet_has_all_permission(local.model, request.user, worksheet) if not force: if worksheet.frozen: raise UsageError( "Can't delete worksheet %s because it is frozen (--force to override)." % worksheet.uuid) if len(worksheet.items) > 0: raise UsageError( "Can't delete worksheet %s because it is not empty (--force to override)." % worksheet.uuid) local.model.delete_worksheet(uuid)
def validate(self, require_child_path=False): """ Validates that the dependency is well formed. :param require_child_path: If True, make sure the child path is not empty This is a needed condition for Run bundles, but not so for Make bundles """ spec_util.check_uuid(self.child_uuid) spec_util.check_uuid(self.parent_uuid) if not self.CHILD_PATH_REGEX.match(self.child_path): raise UsageError('child_path must match %s, was %s' % (self.CHILD_PATH_REGEX.pattern, self.child_path)) if require_child_path and len(self.child_path) == 0: raise UsageError('child_path empty')
def get_bundle_uuid(model, user_id, worksheet_uuid, bundle_spec): """ Resolve a string bundle_spec to a bundle uuid. Types of specifications: - uuid: should be unique. - name[^[<index>]: there might be many uuids with this name. - ^[<index>], where index is the i-th (1-based) most recent element on the current worksheet. """ if not bundle_spec: raise UsageError('Tried to expand empty bundle_spec!') if spec_util.UUID_REGEX.match(bundle_spec): return bundle_spec elif spec_util.UUID_PREFIX_REGEX.match(bundle_spec): bundle_uuids = model.get_bundle_uuids({'uuid': LikeQuery(bundle_spec + '%'), 'user_id': user_id}, max_results=2) if len(bundle_uuids) == 0: raise NotFoundError('uuid prefix %s doesn\'t match any bundles' % bundle_spec) elif len(bundle_uuids) == 1: return bundle_uuids[0] else: raise UsageError('uuid prefix %s more than one bundle' % bundle_spec) else: bundle_spec, reverse_index = _parse_relative_bundle_spec(bundle_spec) if bundle_spec: bundle_spec = bundle_spec.replace('.*', '%') # Convert regular expression syntax to SQL syntax if '%' in bundle_spec: bundle_spec_query = LikeQuery(bundle_spec) else: bundle_spec_query = bundle_spec else: bundle_spec_query = None # query results are ordered from newest to old bundle_uuids = model.get_bundle_uuids({ 'name': bundle_spec_query, 'worksheet_uuid': worksheet_uuid, 'user_id': user_id }, max_results=reverse_index) # Take the last bundle if reverse_index <= 0 or reverse_index > len(bundle_uuids): if bundle_spec is None: raise UsageError('%d bundles, index %d out of bounds' % (len(bundle_uuids), reverse_index)) elif len(bundle_uuids) == 0: raise NotFoundError('bundle spec %s doesn\'t match any bundles' % bundle_spec) else: raise UsageError('bundle spec %s matches %d bundles, index %d out of bounds' % (bundle_spec, len(bundle_uuids), reverse_index)) return bundle_uuids[reverse_index - 1]
def read_target(self, target): """ Given target (bundle uuid, path), return (stream, name, content_type). """ uuid, path = target bundle_info = self.client.get_bundle_info(uuid, False, False, False) if bundle_info is None: raise UsageError('Bundle %s does not exist' % (uuid, )) if path == '': name = bundle_info['metadata']['name'] else: name = os.path.basename(path) target_info = self.client.get_target_info(target, 0) if target_info is None: raise UsageError('Target does not exist: %s' % (target, )) target_type = target_info.get('type') if target_type == 'file': # Is a file, don't need to zip it up content_type = mimetypes.guess_type(name)[0] if not content_type: content_type = 'text/plain' source_uuid = self.client.open_target(target) elif target_type == 'directory': # Is a directory, need to zip it up content_type = 'application/x-gzip' source_uuid = self.client.open_target_archive(target) name += '.tar.gz' else: raise UsageError('Target is not file/directory: %s' % (target, )) def read_file(): """ Generates a stream of strings corresponding to the contents of this file. """ try: while True: bytes = self.client.read_file(source_uuid, BundleService.MAX_BYTES) yield bytes.data if len(bytes.data) < BundleService.MAX_BYTES: break finally: self.client.finalize_file(source_uuid) print 'Downloading bundle uuid %s => %s %s' % (uuid, name, content_type) return read_file(), name, content_type
def validate_password(password): """ Check if password meets our requirements, raising UsageError if not. Requirements: - minimum length of 8 characters :param password: string password to validate :return: None """ if not all(33 <= ord(c) <= 126 for c in password): raise UsageError("Password must consist of only printable, non-whitespace ASCII characters.") if len(password) < User.PASSWORD_MIN_LENGTH: raise UsageError("Password must contain at least %d characters." % User.PASSWORD_MIN_LENGTH)
def unpack(ext, source, dest_path): """ Unpack the archive |source| to |dest_path|. Note: |source| can be a file handle or a path. |ext| contains the extension of the archive. """ if ext != '.zip': close_source = False try: if isinstance(source, str): source = open(source, 'rb') close_source = True if ext == '.tar.gz' or ext == '.tgz': un_tar_directory(source, dest_path, 'gz') elif ext == '.tar.bz2': un_tar_directory(source, dest_path, 'bz2') elif ext == '.bz2': un_bz2_file(source, dest_path) elif ext == '.gz': with open(dest_path, 'wb') as f: shutil.copyfileobj(un_gzip_stream(source), f) else: raise UsageError('Not an archive.') except (tarfile.TarError, IOError): raise UsageError('Invalid archive upload.') finally: if close_source: source.close() else: delete_source = False try: # unzip doesn't accept input from standard input, so we have to save # to a temporary file. if not isinstance(source, str): temp_path = dest_path + '.zip' with open(temp_path, 'wb') as f: shutil.copyfileobj(source, f) source = temp_path delete_source = True exitcode = subprocess.call( ['unzip', '-q', source, '-d', dest_path]) if exitcode != 0: raise UsageError('Invalid archive upload.') finally: if delete_source: path_util.remove(source)
def collapse_dicts(cls, metadata_specs, rows): ''' Convert a list of Metadata dictionaries into a normalized metadata dict. ''' metadata_dict = {} metadata_spec_dict = {} for spec in metadata_specs: if spec.type == list or not spec.generated: metadata_dict[spec.key] = spec.get_constructor()() metadata_spec_dict[spec.key] = spec for row in rows: (maybe_unicode_key, value) = (row['metadata_key'], row['metadata_value']) # If the key is Unicode text (which is the case if it was extracted from a # database), cast it to a string. This operation encodes it with UTF-8. key = str(maybe_unicode_key) if key not in metadata_spec_dict: # print 'Warning: %s not in %s, skipping value %s!' % (key, metadata_spec_dict.keys(), value) continue # Somewhat dangerous since we might lose information spec = metadata_spec_dict[key] if spec.type == list: metadata_dict[key].append(value) else: if metadata_dict.get(key): raise UsageError( 'Got duplicate values {} and {} for key {}. metadata dict is {}, rows are {}' .format(metadata_dict[key], value, key, metadata_dict, rows)) # Convert string to the right type (e.g., string to int) metadata_dict[key] = spec.get_constructor()(value) return metadata_dict
def _get_and_check_target_path(self, uuid, path): bundle_path = self._bundle_store.get_bundle_location(uuid) target_path, error_message = get_and_check_target_path( bundle_path, uuid, path) if error_message is not None: raise UsageError(error_message) return target_path
def model(self): """ Return a model. Called by the server. """ model_class = self.config['server']['class'] model = None if model_class == 'MySQLModel': from codalab.model.mysql_model import MySQLModel model = MySQLModel( engine_url=self.config['server']['engine_url'], default_user_info=self.default_user_info(), root_user_id=self.root_user_id(), system_user_id=self.system_user_id(), ) elif model_class == 'SQLiteModel': from codalab.model.sqlite_model import SQLiteModel model = SQLiteModel( default_user_info=self.default_user_info(), root_user_id=self.root_user_id(), system_user_id=self.system_user_id(), ) else: raise UsageError( 'Unexpected model class: %s, expected MySQLModel' % (model_class, )) return model
def do_command(command): def inner(*args, **kwargs): import time time_delay = 1 if self.verbose >= 2: print 'remote_bundle_client: %s %s %s' % (command, args, kwargs) while True: try: return getattr(self.proxy, command)(*args, **kwargs) except xmlrpclib.ProtocolError, e: raise UsageError("Could not authenticate on %s: %s" % (host, e)) except xmlrpclib.Fault, e: # Transform server-side UsageErrors into client-side UsageErrors. if 'codalab.common.UsageError' in e.faultString: index = e.faultString.find(':') raise UsageError(e.faultString[index + 1:]) elif 'codalab.common.PermissionError' in e.faultString: index = e.faultString.find(':') raise PermissionError(e.faultString[index + 1:]) else: raise except socket.error, e: print >> sys.stderr, "Failed to connect to %s: %s. Trying to reconnect in %s seconds..." % ( host, e, time_delay) time.sleep(time_delay) time_delay *= 2 if time_delay > 512: raise UsageError('Failed to connect to %s: %s' % (host, e))
def _parse_relative_bundle_spec(bundle_spec): """ Parse bundle spec "BUNDLESPEC^I" into ("BUNDLE_SPEC", I). :param bundle_spec: string bundle spec :return: (base bundle spec, reverse history index) """ # run: bundle whose name starts with foo m = spec_util.NAME_PATTERN_REGEX.match(bundle_spec) if m: bundle_spec = m.group(1) reverse_index = 1 return (bundle_spec, reverse_index) # foo^3: 3rd to last bundle whose name starts with foo m = spec_util.NAME_PATTERN_HISTORY_REGEX.match(bundle_spec) if m: bundle_spec = m.group(1) reverse_index = int(m.group(2)) if m.group(2) != '' else 1 return (bundle_spec, reverse_index) # ^3: 3rd to last bundle whose name starts with foo in this worksheet m = spec_util.HISTORY_REGEX.match(bundle_spec) if m: bundle_spec = None reverse_index = int(m.group(1)) if m.group(1) != '' else 1 return (bundle_spec, reverse_index) raise UsageError('Invalid bundle_spec: %s' % bundle_spec)
def general_command(worksheet_uuid, command): """ Executes an arbitrary CLI command with |worksheet_uuid| as the current worksheet. Basically, all CLI functionality should go through this command. The method currently intercepts stdout/stderr and returns it back to the user. """ # Tokenize if isinstance(command, basestring): # shlex throws ValueError on incorrectly formatted commands try: args = shlex.split(command) except ValueError as e: raise UsageError(e.message) else: args = list(command) # Ensure command always starts with 'cl' if args[0] == 'cl': args = args[1:] cli, output_buffer = create_cli(worksheet_uuid) structured_result = None try: structured_result = cli.do_command(args) except SystemExit: # as exitcode: # argparse sometimes throws SystemExit, we don't want to exit pass output_str = output_buffer.getvalue() output_buffer.close() return { 'structured_result': structured_result, 'output': output_str, }
def model(self): """ Return a model. Called by the server. """ model_class = self.config['server']['class'] model = None if model_class == 'MySQLModel': from codalab.model.mysql_model import MySQLModel model = MySQLModel( engine_url=self.config['server']['engine_url'], default_user_info=self.default_user_info(), ) elif model_class == 'SQLiteModel': from codalab.model.sqlite_model import SQLiteModel # Patch for backwards-compatibility until we have a cleaner abstraction around config # that can update configs to newer "versions" engine_url = self.config['server'].get( 'engine_url', "sqlite:///{}".format( os.path.join(self.codalab_home, 'bundle.db'))) model = SQLiteModel(engine_url=engine_url, default_user_info=self.default_user_info()) else: raise UsageError( 'Unexpected model class: %s, expected MySQLModel or SQLiteModel' % (model_class, )) model.root_user_id = self.root_user_id() model.system_user_id = self.system_user_id() return model
def safe_uri(redirect_uri): """Check if an URI is relative, otherwise raise an error. """ absolute = bool(urllib.parse.urlparse(redirect_uri).netloc) if absolute: raise UsageError('Only relative URIs are allowed!') return redirect_uri
def create_bundle_actions(): """ Sends the message to the worker to do the bundle action, and adds the action string to the bundle metadata. """ actions = BundleActionSchema( strict=True, many=True, ).load(request.json).data check_bundles_have_all_permission(local.model, request.user, [a['uuid'] for a in actions]) for action in actions: bundle = local.model.get_bundle(action['uuid']) if bundle.state != State.RUNNING: raise UsageError('Cannot execute this action on a bundle that is not running.') worker = local.worker_model.get_bundle_worker(action['uuid']) precondition( local.worker_model.send_json_message(worker['socket_id'], action, 60), 'Unable to reach worker.') new_actions = getattr(bundle.metadata, 'actions', []) + [BundleAction.as_string(action)] db_update = {'metadata': {'actions': new_actions}} local.model.update_bundle(bundle, db_update) return BundleActionSchema(many=True).dump(actions).data
def create_bundle_actions(): """ Sends the message to the worker to do the bundle action, and adds the action string to the bundle metadata. """ actions = BundleActionSchema(strict=True, many=True).load(request.json).data check_bundles_have_all_permission(local.model, request.user, [a['uuid'] for a in actions]) for action in actions: bundle = local.model.get_bundle(action['uuid']) if bundle.state in [State.READY, State.FAILED, State.KILLED]: raise UsageError( 'Cannot execute this action on a bundle that is in the following states: ready, failed, killed. ' 'Kill action can be executed on bundles in created, uploading, staged, making, starting, ' 'running, preparing, or finalizing state.' ) worker = local.model.get_bundle_worker(action['uuid']) new_actions = getattr(bundle.metadata, 'actions', []) + [BundleAction.as_string(action)] # The state updates of bundles in PREPARING, RUNNING, or FINALIZING state will be handled on the worker side. if worker: precondition( local.worker_model.send_json_message(worker['socket_id'], action, 60), 'Unable to reach worker.', ) local.model.update_bundle(bundle, {'metadata': {'actions': new_actions}}) else: # The state updates of bundles in CREATED, UPLOADING, MAKING, STARTING or STAGED state # will be handled on the rest-server side. local.model.update_bundle( bundle, {'state': State.KILLED, 'metadata': {'actions': new_actions}} ) return BundleActionSchema(many=True).dump(actions).data
def update_metadata_and_save(self, bundle, enforce_disk_quota=False): """ Updates the metadata about the contents of the bundle, including data_size as well as the total amount of disk used by the user. If |new_bundle| is True, saves the bundle as a new bundle. Otherwise, updates it. """ bundle_path = self._bundle_store.get_bundle_location(bundle.uuid) dirs_and_files = None if os.path.isdir(bundle_path): dirs_and_files = path_util.recursive_ls(bundle_path) else: dirs_and_files = [], [bundle_path] data_hash = '0x%s' % (path_util.hash_directory(bundle_path, dirs_and_files)) data_size = path_util.get_size(bundle_path, dirs_and_files) if enforce_disk_quota: disk_left = self._bundle_model.get_user_disk_quota_left( bundle.owner_id) if data_size > disk_left: raise UsageError( "Can't save bundle, bundle size %s greater than user's disk quota left: %s" % (data_size, disk_left)) bundle_update = { 'data_hash': data_hash, 'metadata': { 'data_size': data_size, }, } self._bundle_model.update_bundle(bundle, bundle_update) self._bundle_model.update_user_disk_used(bundle.owner_id)