def _get_target_info_within_bundle(self, target, depth):
"""
Helper for get_target_info that only checks for the target info within that bundle
without considering the path might be pointing to one of the dependencies.
Raises NotFoundError if the path is not found.
"""
bundle_state = self._bundle_model.get_bundle_state(target.bundle_uuid)
bundle_link_url = self._bundle_model.get_bundle_metadata(
[target.bundle_uuid], "link_url").get(target.bundle_uuid)
if bundle_link_url:
bundle_link_url = self._transform_link_path(bundle_link_url)
# Raises NotFoundException if uuid is invalid
if bundle_state == State.PREPARING:
raise NotFoundError(
"Bundle {} hasn't started running yet, files not available".
format(target.bundle_uuid))
elif bundle_state != State.RUNNING:
bundle_path = bundle_link_url or self._bundle_store.get_bundle_location(
target.bundle_uuid)
try:
return download_util.get_target_info(bundle_path, target,
depth)
except download_util.PathException as err:
raise NotFoundError(str(err))
else:
# get_target_info calls are sent to the worker even on a shared file
# system since 1) due to NFS caching the worker has more up to date
# information on directory contents, and 2) the logic of hiding
# the dependency paths doesn't need to be re-implemented here.
worker = self._bundle_model.get_bundle_worker(target.bundle_uuid)
response_socket_id = self._worker_model.allocate_socket(
worker['user_id'], worker['worker_id'])
try:
read_args = {'type': 'get_target_info', 'depth': depth}
self._send_read_message(worker, response_socket_id, target,
read_args)
with closing(
self._worker_model.start_listening(
response_socket_id)) as sock:
result = self._worker_model.get_json_message(sock, 60)
if result is None: # dead workers are a fact of life now
logging.info(
'Unable to reach worker, bundle state {}'.format(
bundle_state))
raise NotFoundError(
'Unable to reach worker of running bundle with bundle state {}'
.format(bundle_state))
elif 'error_code' in result:
raise http_error_to_exception(result['error_code'],
result['error_message'])
target_info = result['target_info']
# Deserialize dict response sent over JSON
target_info[
'resolved_target'] = download_util.BundleTarget.from_dict(
target_info['resolved_target'])
return target_info
finally:
self._worker_model.deallocate_socket(response_socket_id)
def get_target_info(self, uuid, path, depth):
"""
Returns information about an individual target inside the bundle,
Raises NotFoundError if the bundle or the path within the bundle is not found
For information about the format of the return value, see
worker.download_util.get_target_info.
"""
bundle_state = self._bundle_model.get_bundle_state(uuid)
# Raises NotFoundException if uuid is invalid
if bundle_state == State.PREPARING:
raise NotFoundError(
"Bundle {} hasn't started running yet, files not available".
format(uuid))
elif bundle_state != State.RUNNING:
bundle_path = self._bundle_store.get_bundle_location(uuid)
try:
return download_util.get_target_info(bundle_path, uuid, path,
depth)
except download_util.PathException as e:
raise NotFoundError(str(e))
else:
# get_target_info calls are sent to the worker even on a shared file
# system since 1) due to NFS caching the worker has more up to date
# information on directory contents, and 2) the logic of hiding
# the dependency paths doesn't need to be re-implemented here.
worker = self._worker_model.get_bundle_worker(uuid)
response_socket_id = self._worker_model.allocate_socket(
worker['user_id'], worker['worker_id'])
try:
read_args = {'type': 'get_target_info', 'depth': depth}
self._send_read_message(worker, response_socket_id, uuid, path,
read_args)
with closing(
self._worker_model.start_listening(
response_socket_id)) as sock:
result = self._worker_model.get_json_message(sock, 60)
if result is None: # dead workers are a fact of life now
logging.info(
'Unable to reach worker, bundle state {}'.format(
bundle_state))
raise NotFoundError(
'Unable to reach worker of running bundle with bundle state {}'
.format(bundle_state))
elif 'error_code' in result:
raise http_error_to_exception(result['error_code'],
result['error_message'])
return result['target_info']
finally:
self._worker_model.deallocate_socket(response_socket_id)
def get_bundle_uuid(model, user_id, worksheet_uuid, bundle_spec):
"""
Resolve a string bundle_spec to a bundle uuid.
Types of specifications:
- uuid: should be unique.
- name[^[<index>]: there might be many uuids with this name.
- ^[<index>], where index is the i-th (1-based) most recent element on the current worksheet.
"""
if not bundle_spec:
raise UsageError('Tried to expand empty bundle_spec!')
if spec_util.UUID_REGEX.match(bundle_spec):
return bundle_spec
elif spec_util.UUID_PREFIX_REGEX.match(bundle_spec):
bundle_uuids = model.get_bundle_uuids({'uuid': LikeQuery(bundle_spec + '%'), 'user_id': user_id}, max_results=2)
if len(bundle_uuids) == 0:
raise NotFoundError('uuid prefix %s doesn\'t match any bundles' % bundle_spec)
elif len(bundle_uuids) == 1:
return bundle_uuids[0]
else:
raise UsageError('uuid prefix %s more than one bundle' % bundle_spec)
else:
bundle_spec, reverse_index = _parse_relative_bundle_spec(bundle_spec)
if bundle_spec:
bundle_spec = bundle_spec.replace('.*', '%') # Convert regular expression syntax to SQL syntax
if '%' in bundle_spec:
bundle_spec_query = LikeQuery(bundle_spec)
else:
bundle_spec_query = bundle_spec
else:
bundle_spec_query = None
# query results are ordered from newest to old
bundle_uuids = model.get_bundle_uuids({
'name': bundle_spec_query,
'worksheet_uuid': worksheet_uuid,
'user_id': user_id
}, max_results=reverse_index)
# Take the last bundle
if reverse_index <= 0 or reverse_index > len(bundle_uuids):
if bundle_spec is None:
raise UsageError('%d bundles, index %d out of bounds' %
(len(bundle_uuids), reverse_index))
elif len(bundle_uuids) == 0:
raise NotFoundError('bundle spec %s doesn\'t match any bundles' % bundle_spec)
else:
raise UsageError('bundle spec %s matches %d bundles, index %d out of bounds' %
(bundle_spec, len(bundle_uuids), reverse_index))
return bundle_uuids[reverse_index - 1]
def get_single_group(model, group_spec, search_fn):
'''
Helper function.
Resolve a string group_spec to a unique group for the given |search_fn|.
Throw an error if zero or more than one group matches.
'''
if not group_spec:
raise UsageError('Tried to expand empty group_spec!')
if spec_util.UUID_REGEX.match(group_spec):
groups = search_fn(model, uuid=group_spec)
message = "uuid starting with '%s'" % (group_spec, )
elif spec_util.UUID_PREFIX_REGEX.match(group_spec):
groups = search_fn(model, uuid=LikeQuery(group_spec + '%'))
message = "uuid starting with '%s'" % (group_spec, )
else:
spec_util.check_name(group_spec)
groups = search_fn(model, name=group_spec)
message = "name '%s'" % (group_spec, )
if not groups:
raise NotFoundError('Found no group with %s' % (message, ))
elif len(groups) > 1:
raise UsageError('Found multiple groups with %s:%s' %
(message, ''.join('\n uuid=%s' % (group['uuid'], )
for group in groups)))
return groups[0]
def inner(*args, **kwargs):
import time
time_delay = 1
if self.verbose >= 2:
print 'remote_bundle_client: %s %s %s' % (command, args,
kwargs)
while True:
try:
return getattr(self.proxy, command)(*args, **kwargs)
except xmlrpclib.ProtocolError, e:
raise UsageError("Could not authenticate on %s: %s" %
(host, e))
except xmlrpclib.Fault, e:
# Transform server-side UsageErrors into client-side UsageErrors.
if 'codalab.common.UsageError' in e.faultString:
index = e.faultString.find(':')
raise UsageError(e.faultString[index + 1:])
if 'codalab.common.NotFoundError' in e.faultString:
index = e.faultString.find(':')
raise NotFoundError(e.faultString[index + 1:])
elif 'codalab.common.PermissionError' in e.faultString:
index = e.faultString.find(':')
raise PermissionError(e.faultString[index + 1:])
elif 'codalab.common.AuthorizationError' in e.faultString:
index = e.faultString.find(':')
raise AuthorizationError(e.faultString[index + 1:])
else:
raise
def stream_tarred_gzipped_directory(self, target):
"""
Returns a file-like object containing a tarred and gzipped archive
of the given directory.
"""
bundle_state = self._bundle_model.get_bundle_state(target.bundle_uuid)
# Raises NotFoundException if uuid is invalid
if bundle_state == State.PREPARING:
raise NotFoundError(
"Bundle {} hasn't started running yet, files not available".
format(target.bundle_uuid))
elif bundle_state != State.RUNNING:
directory_path = self._get_target_path(target)
return file_util.tar_gzip_directory(directory_path)
else:
# stream_tarred_gzipped_directory calls are sent to the worker even
# on a shared filesystem since
# 1) due to NFS caching the worker has more up to date
# information on directory contents
# 2) the logic of hiding
# the dependency paths doesn't need to be re-implemented here.
worker = self._bundle_model.get_bundle_worker(target.bundle_uuid)
response_socket_id = self._worker_model.allocate_socket(
worker['user_id'], worker['worker_id'])
try:
read_args = {'type': 'stream_directory'}
self._send_read_message(worker, response_socket_id, target,
read_args)
fileobj = self._get_read_response_stream(response_socket_id)
return Deallocating(fileobj, self._worker_model,
response_socket_id)
except Exception:
self._worker_model.deallocate_socket(response_socket_id)
raise
def get_worksheet_uuid(model, base_worksheet_uuid, worksheet_spec):
"""
Resolve a string worksheet_spec to a unique worksheet uuid.
If base_worksheet_uuid specified, then try to resolve worksheet_spec in the
context of base_worksheet_uuid.
"""
if not worksheet_spec:
raise UsageError('Tried to expand empty worksheet_spec!')
if spec_util.UUID_REGEX.match(worksheet_spec):
return worksheet_spec
if spec_util.UUID_PREFIX_REGEX.match(worksheet_spec):
worksheets = model.batch_get_worksheets(fetch_items=False, uuid=LikeQuery(worksheet_spec + '%'),
base_worksheet_uuid=base_worksheet_uuid)
message = "uuid starting with '%s'" % (worksheet_spec,)
else:
spec_util.check_name(worksheet_spec)
worksheets = model.batch_get_worksheets(fetch_items=False, name=worksheet_spec,
base_worksheet_uuid=base_worksheet_uuid)
message = "name '%s'" % (worksheet_spec,)
if not worksheets:
raise NotFoundError('No worksheet found with %s' % (message,))
if len(worksheets) > 1:
raise UsageError(
'Found multiple worksheets with %s:%s' %
(message, ''.join('\n %s' % (worksheet,) for worksheet in worksheets))
)
return worksheets[0].uuid
def get_target_info(self, target, depth):
"""
Returns information about an individual target inside the bundle,
If the path is not found within the bundle files, checks whether the path
points to one of the dependencies of the bundle, and if so, recursively
tries to get the information for the path within that dependency bundle.
:param target: a download_util.BundleTarget containing the bundle UUID and subpath
Raises NotFoundError if the bundle or the path within the bundle is not found
For information about the format of the return value, see
worker.download_util.get_target_info.
"""
try:
return self._get_target_info_within_bundle(target, depth)
except NotFoundError as err:
# if path not in bundle, check if it matches one of its dependencies
child_path_to_dep = {
dep.child_path: dep
for dep in self._bundle_model.get_bundle_dependencies(
target.bundle_uuid)
}
matching_dep = child_path_to_dep.get(
target.subpath.split(os.path.sep)[0])
if matching_dep:
# The path actually belongs to a dependency of this bundle
# Get the subpath of the dependency, and the subpath requested in this call and join them
# ie if dependency is key:dep-bundle/dep-subpath and the requested path is bundle/key/path-subpath
# call get_target_info((dep-bundle, dep-subpath/path-subpath))
parent_path = matching_dep.parent_path
parent_subpath = target.subpath.split(os.path.sep)[1:]
if parent_path:
new_path = os.path.sep.join([parent_path] + parent_subpath)
else:
new_path = os.path.sep.join(parent_subpath)
return self.get_target_info(
download_util.BundleTarget(matching_dep.parent_uuid,
new_path), depth)
raise err
except Exception as ex:
raise NotFoundError(str(ex))
def process_dep(dep):
parent = parent_dict[dep.parent_uuid]
# Compute an absolute target and check that the dependency exists.
if not parent.uuid:
raise UsageError('Parent %s does not have uuid' % parent)
target = path_util.safe_join(
bundle_store.get_bundle_location(parent.uuid),
dep.parent_path,
)
if not os.path.exists(target):
parent_spec = getattr(parent.metadata, 'name', parent.uuid)
target_text = path_util.safe_join(parent_spec, dep.parent_path)
raise NotFoundError('Target not found: %s' % (target_text, ))
if relative_symlinks:
# Create a symlink that points to the dependency's relative target.
target = path_util.safe_join(
(os.pardir if dep.child_path else ''),
bundle_store.get_bundle_location(parent.uuid,
relative=True),
dep.parent_path,
)
link_path = path_util.safe_join(dest_path, dep.child_path)
return (target, link_path)