def get_target_info(self, run_state, path, dep_paths, args, reply_fn):
"""
Return target_info of path in bundle as a message on the reply_fn
"""
bundle_uuid = run_state.bundle['uuid']
target_info = None
# if path is a dependency raise an error
if path and os.path.normpath(path) in dep_paths:
err = (http.client.NOT_FOUND,
'{} not found in bundle {}'.format(path, bundle_uuid))
reply_fn(err, None, None)
return
else:
try:
target_info = download_util.get_target_info(
run_state.bundle_path, bundle_uuid, path, args['depth'])
except PathException as e:
err = (http.client.NOT_FOUND, str(e))
reply_fn(err, None, None)
return
if not path and args['depth'] > 0:
target_info['contents'] = [
child for child in target_info['contents']
if child['name'] not in dep_paths
]
reply_fn(None, {'target_info': target_info}, None)
def get_target_info(self, run_state, path, args, reply_fn):
"""
Return target_info of path in bundle as a message on the reply_fn
"""
target_info = None
dep_paths = set([dep.child_path for dep in run_state.bundle.dependencies])
# if path is a dependency raise an error
if path and os.path.normpath(path) in dep_paths:
err = (
http.client.NOT_FOUND,
'{} not found in bundle {}'.format(path, run_state.bundle.uuid),
)
reply_fn(err, None, None)
return
else:
try:
target_info = download_util.get_target_info(
run_state.bundle_path, BundleTarget(run_state.bundle.uuid, path), args['depth']
)
except PathException as e:
err = (http.client.NOT_FOUND, str(e))
reply_fn(err, None, None)
return
if not path and args['depth'] > 0:
target_info['contents'] = [
child for child in target_info['contents'] if child['name'] not in dep_paths
]
# Object is not JSON serializable so submit its dict in API response
# The client is responsible for deserializing it
target_info['resolved_target'] = target_info['resolved_target'].__dict__
reply_fn(None, {'target_info': target_info}, None)
def test_single_file(self):
"""Test getting target info of a single file (compressed as .gz) on Azure Blob Storage."""
bundle_uuid, bundle_path = self.create_file(b"a")
target_info = get_target_info(bundle_path, BundleTarget(bundle_uuid, None), 0)
target_info.pop("resolved_target")
self.assertEqual(
target_info, {'name': bundle_uuid, 'type': 'file', 'size': 1, 'perm': 0o755}
)
def _get_target_info_within_bundle(self, target, depth):
"""
Helper for get_target_info that only checks for the target info within that bundle
without considering the path might be pointing to one of the dependencies.
Raises NotFoundError if the path is not found.
"""
bundle_state = self._bundle_model.get_bundle_state(target.bundle_uuid)
bundle_link_url = self._bundle_model.get_bundle_metadata(
[target.bundle_uuid], "link_url").get(target.bundle_uuid)
if bundle_link_url:
bundle_link_url = self._transform_link_path(bundle_link_url)
# Raises NotFoundException if uuid is invalid
if bundle_state == State.PREPARING:
raise NotFoundError(
"Bundle {} hasn't started running yet, files not available".
format(target.bundle_uuid))
elif bundle_state != State.RUNNING:
bundle_path = bundle_link_url or self._bundle_store.get_bundle_location(
target.bundle_uuid)
try:
return download_util.get_target_info(bundle_path, target,
depth)
except download_util.PathException as err:
raise NotFoundError(str(err))
else:
# get_target_info calls are sent to the worker even on a shared file
# system since 1) due to NFS caching the worker has more up to date
# information on directory contents, and 2) the logic of hiding
# the dependency paths doesn't need to be re-implemented here.
worker = self._bundle_model.get_bundle_worker(target.bundle_uuid)
response_socket_id = self._worker_model.allocate_socket(
worker['user_id'], worker['worker_id'])
try:
read_args = {'type': 'get_target_info', 'depth': depth}
self._send_read_message(worker, response_socket_id, target,
read_args)
with closing(
self._worker_model.start_listening(
response_socket_id)) as sock:
result = self._worker_model.get_json_message(sock, 60)
if result is None: # dead workers are a fact of life now
logging.info(
'Unable to reach worker, bundle state {}'.format(
bundle_state))
raise NotFoundError(
'Unable to reach worker of running bundle with bundle state {}'
.format(bundle_state))
elif 'error_code' in result:
raise http_error_to_exception(result['error_code'],
result['error_message'])
target_info = result['target_info']
# Deserialize dict response sent over JSON
target_info[
'resolved_target'] = download_util.BundleTarget.from_dict(
target_info['resolved_target'])
return target_info
finally:
self._worker_model.deallocate_socket(response_socket_id)
def get_target_info(self, uuid, path, depth):
"""
Returns information about an individual target inside the bundle,
Raises NotFoundError if the bundle or the path within the bundle is not found
For information about the format of the return value, see
worker.download_util.get_target_info.
"""
bundle_state = self._bundle_model.get_bundle_state(uuid)
# Raises NotFoundException if uuid is invalid
if bundle_state == State.PREPARING:
raise NotFoundError(
"Bundle {} hasn't started running yet, files not available".
format(uuid))
elif bundle_state != State.RUNNING:
bundle_path = self._bundle_store.get_bundle_location(uuid)
try:
return download_util.get_target_info(bundle_path, uuid, path,
depth)
except download_util.PathException as e:
raise NotFoundError(str(e))
else:
# get_target_info calls are sent to the worker even on a shared file
# system since 1) due to NFS caching the worker has more up to date
# information on directory contents, and 2) the logic of hiding
# the dependency paths doesn't need to be re-implemented here.
worker = self._worker_model.get_bundle_worker(uuid)
response_socket_id = self._worker_model.allocate_socket(
worker['user_id'], worker['worker_id'])
try:
read_args = {'type': 'get_target_info', 'depth': depth}
self._send_read_message(worker, response_socket_id, uuid, path,
read_args)
with closing(
self._worker_model.start_listening(
response_socket_id)) as sock:
result = self._worker_model.get_json_message(sock, 60)
if result is None: # dead workers are a fact of life now
logging.info(
'Unable to reach worker, bundle state {}'.format(
bundle_state))
raise NotFoundError(
'Unable to reach worker of running bundle with bundle state {}'
.format(bundle_state))
elif 'error_code' in result:
raise http_error_to_exception(result['error_code'],
result['error_message'])
return result['target_info']
finally:
self._worker_model.deallocate_socket(response_socket_id)
def test_nested_directories(self):
"""Test getting target info of different files within a bundle that consists of nested directories, on Azure Blob Storage."""
bundle_uuid, bundle_path = self.create_directory()
target_info = get_target_info(bundle_path,
BundleTarget(bundle_uuid, None), 0)
target_info.pop("resolved_target")
self.assertEqual(target_info, {
'name': bundle_uuid,
'type': 'directory',
'size': 249,
'perm': 0o755
})
target_info = get_target_info(bundle_path,
BundleTarget(bundle_uuid, None), 1)
target_info.pop("resolved_target")
self.assertEqual(
target_info,
{
'name':
bundle_uuid,
'type':
'directory',
'size':
249,
'perm':
0o755,
'contents': [
{
'name': 'README.md',
'type': 'file',
'size': 11,
'perm': 0o644
},
{
'name': 'dist',
'type': 'directory',
'size': 0,
'perm': 0o644
},
{
'name': 'src',
'type': 'directory',
'size': 0,
'perm': 0o644
},
],
},
)
target_info = get_target_info(bundle_path,
BundleTarget(bundle_uuid, "README.md"),
1)
target_info.pop("resolved_target")
self.assertEqual(target_info, {
'name': 'README.md',
'type': 'file',
'size': 11,
'perm': 0o644
})
target_info = get_target_info(bundle_path,
BundleTarget(bundle_uuid, "src/test.sh"),
1)
target_info.pop("resolved_target")
self.assertEqual(target_info, {
'name': 'test.sh',
'type': 'file',
'size': 7,
'perm': 0o644
})
target_info = get_target_info(
bundle_path, BundleTarget(bundle_uuid, "dist/a/b/test2.sh"), 1)
target_info.pop("resolved_target")
self.assertEqual(target_info, {
'name': 'test2.sh',
'type': 'file',
'size': 8,
'perm': 0o644
})
target_info = get_target_info(bundle_path,
BundleTarget(bundle_uuid, "src"), 1)
target_info.pop("resolved_target")
self.assertEqual(
target_info,
{
'name':
'src',
'type':
'directory',
'size':
0,
'perm':
0o644,
'contents': [{
'name': 'test.sh',
'type': 'file',
'size': 7,
'perm': 0o644
}],
},
)
# Return all depths
target_info = get_target_info(bundle_path,
BundleTarget(bundle_uuid, "dist/a"), 999)
target_info.pop("resolved_target")
self.assertEqual(
target_info,
{
'name':
'a',
'size':
0,
'perm':
0o644,
'type':
'directory',
'contents': [{
'name':
'b',
'size':
0,
'perm':
0o644,
'type':
'directory',
'contents': [{
'name': 'test2.sh',
'size': 8,
'perm': 0o644,
'type': 'file'
}],
}],
},
)
def test_single_txt_file(self):
"""Test getting target info of a single txt file on Azure Blob Storage. As this isn't supported
(paths should be specified within existing .gz / .tar.gz files), this should throw an exception."""
bundle_uuid, bundle_path = self.create_txt_file(b"a")
with self.assertRaises(PathException):
get_target_info(bundle_path, BundleTarget(bundle_uuid, None), 0)