def unpack(ext: str, source: IO[bytes], dest_path: str):
"""Unpack the archive |source| to |dest_path|.
Args:
ext (str): Extension of the archive.
source (IO[bytes]): File handle to the source.
dest_path ([type]): Destination path to unpack to.
"""
try:
if ext == '.tar.gz' or ext == '.tgz':
un_tar_directory(source, dest_path, 'gz')
elif ext == '.tar.bz2':
un_tar_directory(source, dest_path, 'bz2')
elif ext == '.bz2':
un_bz2_file(source, dest_path)
elif ext == '.gz':
with open(dest_path, 'wb') as f:
shutil.copyfileobj(un_gzip_stream(source), f)
elif ext == '.zip':
unzip_directory(source, dest_path)
else:
raise UsageError('Not an archive.')
except (tarfile.TarError, IOError) as e:
logging.error("Invalid archive upload: failed to unpack archive: %s",
e)
raise UsageError('Invalid archive upload: failed to unpack archive.')
def construct(cls, targets, command, metadata, owner_id, uuid=None, data_hash=None, state=State.CREATED):
if not uuid: uuid = spec_util.generate_uuid()
# Check that targets does not include both keyed and anonymous targets.
if len(targets) > 1 and any(key == '' for key, value in targets):
raise UsageError('Must specify keys when packaging multiple targets!')
if not isinstance(command, basestring):
raise UsageError('%r is not a valid command!' % (command,))
# List the dependencies of this bundle on its targets.
dependencies = []
for (child_path, (parent_uuid, parent_path)) in targets:
dependencies.append({
'child_uuid': uuid,
'child_path': child_path,
'parent_uuid': parent_uuid,
'parent_path': parent_path,
})
return super(RunBundle, cls).construct({
'uuid': uuid,
'bundle_type': cls.BUNDLE_TYPE,
'command': command,
'data_hash': data_hash,
'state': state,
'metadata': metadata,
'dependencies': dependencies,
'owner_id': owner_id,
})
def get(dep): # Return the key
key, val = parse_key_target(dep)
if key == '':
# key only matches empty string if ':' present
_, _, bundle, subpath = parse_target_spec(val)
key = subpath if subpath is not None else bundle
elif key is None:
# key only returns None if ':' not present in original spec
key = val2key[val] if val in val2key else 'b' + str(
len(target_spec) + 1)
if val not in val2key:
val2key[val] = key
if key in key2val:
if key2val[key] != val:
raise UsageError(
'key %s exists with multiple values: %s and %s' %
(key, key2val[key], val))
else:
if key is None:
raise UsageError(
'target_spec is empty. Please provide a valid target_spec in the format of {}.'
.format(RUN_TARGET_SPEC_FORMAT))
key2val[key] = val
target_spec.append(key + ':' + val)
return key
def parse_metadata_form(bundle_subclass, form_result):
'''
Parse the result of a form template produced out request_missing_metadata.
'''
metadata_specs = bundle_subclass.get_user_defined_metadata()
metadata_types = {spec.key: spec.type for spec in metadata_specs}
result = {}
for line in form_result:
line = line.strip()
if line != '' and not line.startswith('//'):
if ':' not in line:
# TODO: don't delete everything; go back to the editor and show the error message
raise UsageError('Malformatted line (no colon): %s' % (line,))
(metadata_key, remainder) = line.split(':', 1)
remainder = remainder.strip()
if remainder == '':
remainder = None
# TODO: handle multiple lines
if metadata_key not in metadata_types:
raise UsageError('Unexpected metadata key: %s' % (metadata_key,))
metadata_type = metadata_types[metadata_key]
if metadata_type == list:
result[metadata_key] = remainder.split() if remainder else []
elif metadata_type == basestring:
result[metadata_key] = remainder
else:
try:
result[metadata_key] = metadata_type(remainder) if remainder != None else None
except:
raise UsageError('Invalid value %s for type %s' % (remainder, metadata_type))
if 'name' not in result:
raise UsageError('No name specified; aborting')
return result
def unpack(ext, source, dest_path):
"""
Unpack the archive |source| to |dest_path|.
Note: |source| can be a file handle or a path.
|ext| contains the extension of the archive.
"""
close_source = False
try:
if isinstance(source, str):
source = open(source, 'rb')
close_source = True
if ext == '.tar.gz' or ext == '.tgz':
un_tar_directory(source, dest_path, 'gz')
elif ext == '.tar.bz2':
un_tar_directory(source, dest_path, 'bz2')
elif ext == '.bz2':
un_bz2_file(source, dest_path)
elif ext == '.gz':
with open(dest_path, 'wb') as f:
shutil.copyfileobj(un_gzip_stream(source), f)
elif ext == '.zip':
unzip_directory(source, dest_path)
else:
raise UsageError('Not an archive.')
except (tarfile.TarError, IOError):
raise UsageError('Invalid archive upload.')
finally:
if close_source:
source.close()
def inner(*args, **kwargs):
import time
time_delay = 1
if self.verbose >= 2:
print 'remote_bundle_client: %s %s %s' % (command, args,
kwargs)
while True:
try:
return getattr(self.proxy, command)(*args, **kwargs)
except xmlrpclib.ProtocolError, e:
raise UsageError("Could not authenticate on %s: %s" %
(host, e))
except xmlrpclib.Fault, e:
# Transform server-side UsageErrors into client-side UsageErrors.
if 'codalab.common.UsageError' in e.faultString:
index = e.faultString.find(':')
raise UsageError(e.faultString[index + 1:])
if 'codalab.common.NotFoundError' in e.faultString:
index = e.faultString.find(':')
raise NotFoundError(e.faultString[index + 1:])
elif 'codalab.common.PermissionError' in e.faultString:
index = e.faultString.find(':')
raise PermissionError(e.faultString[index + 1:])
elif 'codalab.common.AuthorizationError' in e.faultString:
index = e.faultString.find(':')
raise AuthorizationError(e.faultString[index + 1:])
else:
raise
def validate(self, metadata_specs):
'''
Check that this metadata has the correct metadata keys and that it has
metadata values of the correct types.
'''
expected_keys = set(spec.key for spec in metadata_specs)
for key in self._metadata_keys:
if key not in expected_keys:
raise UsageError('Unexpected metadata key: %s' % (key, ))
for spec in metadata_specs:
if spec.key in self._metadata_keys:
value = getattr(self, spec.key)
if spec.type is float and isinstance(value, int):
# cast int to float
value = float(value)
# Validate formatted string fields
if issubclass(spec.type,
str) and spec.formatting is not None and value:
try:
if spec.formatting == 'duration':
formatting.parse_duration(value)
elif spec.formatting == 'size':
formatting.parse_size(value)
elif spec.formatting == 'date':
formatting.parse_datetime(value)
except ValueError as e:
raise UsageError(str(e))
if value is not None and not isinstance(value, spec.type):
raise UsageError(
'Metadata value for %s should be of type %s, was %s (type %s)'
% (spec.key, spec.type.__name__, value,
type(value).__name__))
elif not spec.generated and not spec.optional:
raise UsageError('Missing metadata key: %s' % (spec.key, ))
def get_single_group(model, group_spec, search_fn):
'''
Helper function.
Resolve a string group_spec to a unique group for the given |search_fn|.
Throw an error if zero or more than one group matches.
'''
if not group_spec:
raise UsageError('Tried to expand empty group_spec!')
if spec_util.UUID_REGEX.match(group_spec):
groups = search_fn(model, uuid=group_spec)
message = "uuid starting with '%s'" % (group_spec, )
elif spec_util.UUID_PREFIX_REGEX.match(group_spec):
groups = search_fn(model, uuid=LikeQuery(group_spec + '%'))
message = "uuid starting with '%s'" % (group_spec, )
else:
spec_util.check_name(group_spec)
groups = search_fn(model, name=group_spec)
message = "name '%s'" % (group_spec, )
if not groups:
raise NotFoundError('Found no group with %s' % (message, ))
elif len(groups) > 1:
raise UsageError('Found multiple groups with %s:%s' %
(message, ''.join('\n uuid=%s' % (group['uuid'], )
for group in groups)))
return groups[0]
def unzip(zip_path, temp_path, file_name):
'''
Take an absolute path to a zip file |zip_path| and return the path to a file or
directory called |file_name| in |temp_path| containing its unzipped
contents.
Assume the zip file contains one file/directory called |file_name|.
If |file_name| is not specified, then return the temp_path itself.
'''
path_util.check_isfile(zip_path, 'unzip_directory')
if file_name:
temp_subpath = os.path.join(temp_path, file_name)
else:
temp_subpath = temp_path
print_util.open_line('Unzipping %s to %s' % (zip_path, temp_subpath))
if os.system("cd %s && unzip -q %s" % (temp_path, zip_path)) != 0:
raise UsageError('unzip failed')
print_util.clear_line()
# Corner case: note that the temp_subpath might not 'exist' because it is a
# symlink (which is broken until it's put in the right place).
if not os.path.exists(temp_subpath) and not os.path.islink(temp_subpath):
raise UsageError('Zip file %s missing %s (%s doesn\'t exist)' %
(zip_path, file_name, temp_subpath))
return temp_subpath
def get_worksheet_uuid(model, base_worksheet_uuid, worksheet_spec):
"""
Resolve a string worksheet_spec to a unique worksheet uuid.
If base_worksheet_uuid specified, then try to resolve worksheet_spec in the
context of base_worksheet_uuid.
"""
if not worksheet_spec:
raise UsageError('Tried to expand empty worksheet_spec!')
if spec_util.UUID_REGEX.match(worksheet_spec):
return worksheet_spec
if spec_util.UUID_PREFIX_REGEX.match(worksheet_spec):
worksheets = model.batch_get_worksheets(fetch_items=False, uuid=LikeQuery(worksheet_spec + '%'),
base_worksheet_uuid=base_worksheet_uuid)
message = "uuid starting with '%s'" % (worksheet_spec,)
else:
spec_util.check_name(worksheet_spec)
worksheets = model.batch_get_worksheets(fetch_items=False, name=worksheet_spec,
base_worksheet_uuid=base_worksheet_uuid)
message = "name '%s'" % (worksheet_spec,)
if not worksheets:
raise NotFoundError('No worksheet found with %s' % (message,))
if len(worksheets) > 1:
raise UsageError(
'Found multiple worksheets with %s:%s' %
(message, ''.join('\n %s' % (worksheet,) for worksheet in worksheets))
)
return worksheets[0].uuid
def unpack_to_archive(ext: str, source: IO[bytes]) -> IO[bytes]:
"""Unpack the archive |source| and returns the unpacked fileobj.
If |source| is an archive, unpacks to a .tar.gz archive file.
If |source| is a non-archive file, unpacks to a .gz file.
Args:
ext (str): Extension of the source archive.
source (IO[bytes]): File handle to the source.
Returns:
IO[bytes]: File object with the archive.
"""
try:
if ext == '.tar.gz' or ext == '.tgz':
return source
elif ext == '.tar.bz2':
return GzipStream(UnBz2Stream(source))
elif ext == '.bz2':
return GzipStream(UnBz2Stream(source))
elif ext == '.gz':
return source
elif ext == '.zip':
return GzipStream(ZipToTarStream(source))
else:
raise UsageError('Not an archive.')
except (tarfile.TarError, IOError) as e:
logging.error("Invalid archive upload: failed to unpack archive: %s",
e)
raise UsageError('Invalid archive upload: failed to unpack archive.')
def resolve_source(source):
# Resolve symlink if desired
resolved_source = source
if follow_symlinks:
resolved_source = os.path.realpath(source)
if not os.path.exists(resolved_source):
raise UsageError('Broken symlink')
elif os.path.islink(source):
raise UsageError('Not following symlinks.')
return resolved_source
def check_quota(self, need_time=False, need_disk=False):
if need_time:
if self.time_used >= self.time_quota:
raise UsageError(
'Out of time quota: %s' %
formatting.ratio_str(formatting.duration_str,
self.time_used, self.time_quota))
if need_disk:
if self.disk_used >= self.disk_quota:
raise UsageError(
'Out of disk quota: %s' % formatting.ratio_str(
formatting.size_str, self.disk_used, self.disk_quota))
def delete_worksheet(uuid, force):
worksheet = local.model.get_worksheet(uuid, fetch_items=True)
check_worksheet_has_all_permission(local.model, request.user, worksheet)
if not force:
if worksheet.frozen:
raise UsageError(
"Can't delete worksheet %s because it is frozen (--force to override)."
% worksheet.uuid)
if len(worksheet.items) > 0:
raise UsageError(
"Can't delete worksheet %s because it is not empty (--force to override)."
% worksheet.uuid)
local.model.delete_worksheet(uuid)
def validate(self, require_child_path=False):
"""
Validates that the dependency is well formed.
:param require_child_path: If True, make sure the child path is not empty
This is a needed condition for Run bundles, but not so for Make bundles
"""
spec_util.check_uuid(self.child_uuid)
spec_util.check_uuid(self.parent_uuid)
if not self.CHILD_PATH_REGEX.match(self.child_path):
raise UsageError('child_path must match %s, was %s' %
(self.CHILD_PATH_REGEX.pattern, self.child_path))
if require_child_path and len(self.child_path) == 0:
raise UsageError('child_path empty')
def get_bundle_uuid(model, user_id, worksheet_uuid, bundle_spec):
"""
Resolve a string bundle_spec to a bundle uuid.
Types of specifications:
- uuid: should be unique.
- name[^[<index>]: there might be many uuids with this name.
- ^[<index>], where index is the i-th (1-based) most recent element on the current worksheet.
"""
if not bundle_spec:
raise UsageError('Tried to expand empty bundle_spec!')
if spec_util.UUID_REGEX.match(bundle_spec):
return bundle_spec
elif spec_util.UUID_PREFIX_REGEX.match(bundle_spec):
bundle_uuids = model.get_bundle_uuids({'uuid': LikeQuery(bundle_spec + '%'), 'user_id': user_id}, max_results=2)
if len(bundle_uuids) == 0:
raise NotFoundError('uuid prefix %s doesn\'t match any bundles' % bundle_spec)
elif len(bundle_uuids) == 1:
return bundle_uuids[0]
else:
raise UsageError('uuid prefix %s more than one bundle' % bundle_spec)
else:
bundle_spec, reverse_index = _parse_relative_bundle_spec(bundle_spec)
if bundle_spec:
bundle_spec = bundle_spec.replace('.*', '%') # Convert regular expression syntax to SQL syntax
if '%' in bundle_spec:
bundle_spec_query = LikeQuery(bundle_spec)
else:
bundle_spec_query = bundle_spec
else:
bundle_spec_query = None
# query results are ordered from newest to old
bundle_uuids = model.get_bundle_uuids({
'name': bundle_spec_query,
'worksheet_uuid': worksheet_uuid,
'user_id': user_id
}, max_results=reverse_index)
# Take the last bundle
if reverse_index <= 0 or reverse_index > len(bundle_uuids):
if bundle_spec is None:
raise UsageError('%d bundles, index %d out of bounds' %
(len(bundle_uuids), reverse_index))
elif len(bundle_uuids) == 0:
raise NotFoundError('bundle spec %s doesn\'t match any bundles' % bundle_spec)
else:
raise UsageError('bundle spec %s matches %d bundles, index %d out of bounds' %
(bundle_spec, len(bundle_uuids), reverse_index))
return bundle_uuids[reverse_index - 1]
def read_target(self, target):
"""
Given target (bundle uuid, path), return (stream, name, content_type).
"""
uuid, path = target
bundle_info = self.client.get_bundle_info(uuid, False, False,
False)
if bundle_info is None:
raise UsageError('Bundle %s does not exist' % (uuid, ))
if path == '':
name = bundle_info['metadata']['name']
else:
name = os.path.basename(path)
target_info = self.client.get_target_info(target, 0)
if target_info is None:
raise UsageError('Target does not exist: %s' % (target, ))
target_type = target_info.get('type')
if target_type == 'file':
# Is a file, don't need to zip it up
content_type = mimetypes.guess_type(name)[0]
if not content_type: content_type = 'text/plain'
source_uuid = self.client.open_target(target)
elif target_type == 'directory':
# Is a directory, need to zip it up
content_type = 'application/x-gzip'
source_uuid = self.client.open_target_archive(target)
name += '.tar.gz'
else:
raise UsageError('Target is not file/directory: %s' %
(target, ))
def read_file():
"""
Generates a stream of strings corresponding to the contents of this file.
"""
try:
while True:
bytes = self.client.read_file(source_uuid,
BundleService.MAX_BYTES)
yield bytes.data
if len(bytes.data) < BundleService.MAX_BYTES:
break
finally:
self.client.finalize_file(source_uuid)
print 'Downloading bundle uuid %s => %s %s' % (uuid, name,
content_type)
return read_file(), name, content_type
def validate_password(password):
"""
Check if password meets our requirements, raising UsageError if not.
Requirements:
- minimum length of 8 characters
:param password: string password to validate
:return: None
"""
if not all(33 <= ord(c) <= 126 for c in password):
raise UsageError("Password must consist of only printable, non-whitespace ASCII characters.")
if len(password) < User.PASSWORD_MIN_LENGTH:
raise UsageError("Password must contain at least %d characters." % User.PASSWORD_MIN_LENGTH)
def unpack(ext, source, dest_path):
"""
Unpack the archive |source| to |dest_path|.
Note: |source| can be a file handle or a path.
|ext| contains the extension of the archive.
"""
if ext != '.zip':
close_source = False
try:
if isinstance(source, str):
source = open(source, 'rb')
close_source = True
if ext == '.tar.gz' or ext == '.tgz':
un_tar_directory(source, dest_path, 'gz')
elif ext == '.tar.bz2':
un_tar_directory(source, dest_path, 'bz2')
elif ext == '.bz2':
un_bz2_file(source, dest_path)
elif ext == '.gz':
with open(dest_path, 'wb') as f:
shutil.copyfileobj(un_gzip_stream(source), f)
else:
raise UsageError('Not an archive.')
except (tarfile.TarError, IOError):
raise UsageError('Invalid archive upload.')
finally:
if close_source:
source.close()
else:
delete_source = False
try:
# unzip doesn't accept input from standard input, so we have to save
# to a temporary file.
if not isinstance(source, str):
temp_path = dest_path + '.zip'
with open(temp_path, 'wb') as f:
shutil.copyfileobj(source, f)
source = temp_path
delete_source = True
exitcode = subprocess.call(
['unzip', '-q', source, '-d', dest_path])
if exitcode != 0:
raise UsageError('Invalid archive upload.')
finally:
if delete_source:
path_util.remove(source)
def collapse_dicts(cls, metadata_specs, rows):
'''
Convert a list of Metadata dictionaries into a normalized metadata dict.
'''
metadata_dict = {}
metadata_spec_dict = {}
for spec in metadata_specs:
if spec.type == list or not spec.generated:
metadata_dict[spec.key] = spec.get_constructor()()
metadata_spec_dict[spec.key] = spec
for row in rows:
(maybe_unicode_key, value) = (row['metadata_key'],
row['metadata_value'])
# If the key is Unicode text (which is the case if it was extracted from a
# database), cast it to a string. This operation encodes it with UTF-8.
key = str(maybe_unicode_key)
if key not in metadata_spec_dict:
# print 'Warning: %s not in %s, skipping value %s!' % (key, metadata_spec_dict.keys(), value)
continue # Somewhat dangerous since we might lose information
spec = metadata_spec_dict[key]
if spec.type == list:
metadata_dict[key].append(value)
else:
if metadata_dict.get(key):
raise UsageError(
'Got duplicate values {} and {} for key {}. metadata dict is {}, rows are {}'
.format(metadata_dict[key], value, key, metadata_dict,
rows))
# Convert string to the right type (e.g., string to int)
metadata_dict[key] = spec.get_constructor()(value)
return metadata_dict
def _get_and_check_target_path(self, uuid, path):
bundle_path = self._bundle_store.get_bundle_location(uuid)
target_path, error_message = get_and_check_target_path(
bundle_path, uuid, path)
if error_message is not None:
raise UsageError(error_message)
return target_path
def model(self):
"""
Return a model. Called by the server.
"""
model_class = self.config['server']['class']
model = None
if model_class == 'MySQLModel':
from codalab.model.mysql_model import MySQLModel
model = MySQLModel(
engine_url=self.config['server']['engine_url'],
default_user_info=self.default_user_info(),
root_user_id=self.root_user_id(),
system_user_id=self.system_user_id(),
)
elif model_class == 'SQLiteModel':
from codalab.model.sqlite_model import SQLiteModel
model = SQLiteModel(
default_user_info=self.default_user_info(),
root_user_id=self.root_user_id(),
system_user_id=self.system_user_id(),
)
else:
raise UsageError(
'Unexpected model class: %s, expected MySQLModel' %
(model_class, ))
return model
def do_command(command):
def inner(*args, **kwargs):
import time
time_delay = 1
if self.verbose >= 2:
print 'remote_bundle_client: %s %s %s' % (command, args,
kwargs)
while True:
try:
return getattr(self.proxy, command)(*args, **kwargs)
except xmlrpclib.ProtocolError, e:
raise UsageError("Could not authenticate on %s: %s" %
(host, e))
except xmlrpclib.Fault, e:
# Transform server-side UsageErrors into client-side UsageErrors.
if 'codalab.common.UsageError' in e.faultString:
index = e.faultString.find(':')
raise UsageError(e.faultString[index + 1:])
elif 'codalab.common.PermissionError' in e.faultString:
index = e.faultString.find(':')
raise PermissionError(e.faultString[index + 1:])
else:
raise
except socket.error, e:
print >> sys.stderr, "Failed to connect to %s: %s. Trying to reconnect in %s seconds..." % (
host, e, time_delay)
time.sleep(time_delay)
time_delay *= 2
if time_delay > 512:
raise UsageError('Failed to connect to %s: %s' %
(host, e))
def _parse_relative_bundle_spec(bundle_spec):
"""
Parse bundle spec "BUNDLESPEC^I" into ("BUNDLE_SPEC", I).
:param bundle_spec: string bundle spec
:return: (base bundle spec, reverse history index)
"""
# run: bundle whose name starts with foo
m = spec_util.NAME_PATTERN_REGEX.match(bundle_spec)
if m:
bundle_spec = m.group(1)
reverse_index = 1
return (bundle_spec, reverse_index)
# foo^3: 3rd to last bundle whose name starts with foo
m = spec_util.NAME_PATTERN_HISTORY_REGEX.match(bundle_spec)
if m:
bundle_spec = m.group(1)
reverse_index = int(m.group(2)) if m.group(2) != '' else 1
return (bundle_spec, reverse_index)
# ^3: 3rd to last bundle whose name starts with foo in this worksheet
m = spec_util.HISTORY_REGEX.match(bundle_spec)
if m:
bundle_spec = None
reverse_index = int(m.group(1)) if m.group(1) != '' else 1
return (bundle_spec, reverse_index)
raise UsageError('Invalid bundle_spec: %s' % bundle_spec)
def general_command(worksheet_uuid, command):
"""
Executes an arbitrary CLI command with |worksheet_uuid| as the current worksheet.
Basically, all CLI functionality should go through this command.
The method currently intercepts stdout/stderr and returns it back to the user.
"""
# Tokenize
if isinstance(command, basestring):
# shlex throws ValueError on incorrectly formatted commands
try:
args = shlex.split(command)
except ValueError as e:
raise UsageError(e.message)
else:
args = list(command)
# Ensure command always starts with 'cl'
if args[0] == 'cl':
args = args[1:]
cli, output_buffer = create_cli(worksheet_uuid)
structured_result = None
try:
structured_result = cli.do_command(args)
except SystemExit: # as exitcode:
# argparse sometimes throws SystemExit, we don't want to exit
pass
output_str = output_buffer.getvalue()
output_buffer.close()
return {
'structured_result': structured_result,
'output': output_str,
}
def model(self):
"""
Return a model. Called by the server.
"""
model_class = self.config['server']['class']
model = None
if model_class == 'MySQLModel':
from codalab.model.mysql_model import MySQLModel
model = MySQLModel(
engine_url=self.config['server']['engine_url'],
default_user_info=self.default_user_info(),
)
elif model_class == 'SQLiteModel':
from codalab.model.sqlite_model import SQLiteModel
# Patch for backwards-compatibility until we have a cleaner abstraction around config
# that can update configs to newer "versions"
engine_url = self.config['server'].get(
'engine_url', "sqlite:///{}".format(
os.path.join(self.codalab_home, 'bundle.db')))
model = SQLiteModel(engine_url=engine_url,
default_user_info=self.default_user_info())
else:
raise UsageError(
'Unexpected model class: %s, expected MySQLModel or SQLiteModel'
% (model_class, ))
model.root_user_id = self.root_user_id()
model.system_user_id = self.system_user_id()
return model
def safe_uri(redirect_uri):
"""Check if an URI is relative, otherwise raise an error.
"""
absolute = bool(urllib.parse.urlparse(redirect_uri).netloc)
if absolute:
raise UsageError('Only relative URIs are allowed!')
return redirect_uri
def create_bundle_actions():
"""
Sends the message to the worker to do the bundle action, and adds the
action string to the bundle metadata.
"""
actions = BundleActionSchema(
strict=True, many=True,
).load(request.json).data
check_bundles_have_all_permission(local.model, request.user, [a['uuid'] for a in actions])
for action in actions:
bundle = local.model.get_bundle(action['uuid'])
if bundle.state != State.RUNNING:
raise UsageError('Cannot execute this action on a bundle that is not running.')
worker = local.worker_model.get_bundle_worker(action['uuid'])
precondition(
local.worker_model.send_json_message(worker['socket_id'], action, 60),
'Unable to reach worker.')
new_actions = getattr(bundle.metadata, 'actions', []) + [BundleAction.as_string(action)]
db_update = {'metadata': {'actions': new_actions}}
local.model.update_bundle(bundle, db_update)
return BundleActionSchema(many=True).dump(actions).data
def create_bundle_actions():
"""
Sends the message to the worker to do the bundle action, and adds the
action string to the bundle metadata.
"""
actions = BundleActionSchema(strict=True, many=True).load(request.json).data
check_bundles_have_all_permission(local.model, request.user, [a['uuid'] for a in actions])
for action in actions:
bundle = local.model.get_bundle(action['uuid'])
if bundle.state in [State.READY, State.FAILED, State.KILLED]:
raise UsageError(
'Cannot execute this action on a bundle that is in the following states: ready, failed, killed. '
'Kill action can be executed on bundles in created, uploading, staged, making, starting, '
'running, preparing, or finalizing state.'
)
worker = local.model.get_bundle_worker(action['uuid'])
new_actions = getattr(bundle.metadata, 'actions', []) + [BundleAction.as_string(action)]
# The state updates of bundles in PREPARING, RUNNING, or FINALIZING state will be handled on the worker side.
if worker:
precondition(
local.worker_model.send_json_message(worker['socket_id'], action, 60),
'Unable to reach worker.',
)
local.model.update_bundle(bundle, {'metadata': {'actions': new_actions}})
else:
# The state updates of bundles in CREATED, UPLOADING, MAKING, STARTING or STAGED state
# will be handled on the rest-server side.
local.model.update_bundle(
bundle, {'state': State.KILLED, 'metadata': {'actions': new_actions}}
)
return BundleActionSchema(many=True).dump(actions).data
def update_metadata_and_save(self, bundle, enforce_disk_quota=False):
"""
Updates the metadata about the contents of the bundle, including
data_size as well as the total amount of disk used by the user.
If |new_bundle| is True, saves the bundle as a new bundle. Otherwise,
updates it.
"""
bundle_path = self._bundle_store.get_bundle_location(bundle.uuid)
dirs_and_files = None
if os.path.isdir(bundle_path):
dirs_and_files = path_util.recursive_ls(bundle_path)
else:
dirs_and_files = [], [bundle_path]
data_hash = '0x%s' % (path_util.hash_directory(bundle_path,
dirs_and_files))
data_size = path_util.get_size(bundle_path, dirs_and_files)
if enforce_disk_quota:
disk_left = self._bundle_model.get_user_disk_quota_left(
bundle.owner_id)
if data_size > disk_left:
raise UsageError(
"Can't save bundle, bundle size %s greater than user's disk quota left: %s"
% (data_size, disk_left))
bundle_update = {
'data_hash': data_hash,
'metadata': {
'data_size': data_size,
},
}
self._bundle_model.update_bundle(bundle, bundle_update)
self._bundle_model.update_user_disk_used(bundle.owner_id)
