def install_dependencies(self, bundle_store, parent_dict, path, rel):
'''
Symlink this bundle's dependencies into the directory at path.
The caller is responsible for cleaning up this directory.
'''
precondition(os.path.isabs(path), '%s is a relative path!' % (path,))
for dep in self.dependencies:
parent = parent_dict[dep.parent_uuid]
# Compute an absolute target and check that the dependency exists.
target = path_util.safe_join(
bundle_store.get_location(parent.data_hash),
dep.parent_path,
)
if not os.path.exists(target):
parent_spec = getattr(parent.metadata, 'name', parent.uuid)
target_text = path_util.safe_join(parent_spec, dep.parent_path)
raise UsageError('Target not found: %s' % (target_text,))
if rel:
# Create a symlink that points to the dependency's relative target.
target = path_util.safe_join(
(os.pardir if dep.child_path else ''),
bundle_store.get_location(parent.data_hash, relative=True),
dep.parent_path,
)
link_path = path_util.safe_join(path, dep.child_path)
os.symlink(target, link_path)
def format_basic_info(self, info):
metadata = collections.defaultdict(lambda: None, info['metadata'])
# Format some simple fields of the basic info string.
fields = {
'bundle_type': info['bundle_type'],
'uuid': info['uuid'],
'data_hash': info['data_hash'] or '<no hash>',
'state': info['state'],
'name': metadata['name'] or '<no name>',
'description': metadata['description'] or '<no description>',
}
# Format statistics about this bundle - creation time, runtime, size, etc.
stats = []
if 'created' in metadata:
stats.append('Created: %s' % (self.time_str(metadata['created']),))
if 'data_size' in metadata:
stats.append('Size: %s' % (self.size_str(metadata['data_size']),))
fields['stats'] = 'Stats:\n %s\n' % ('\n '.join(stats),) if stats else ''
# Compute a nicely-formatted list of hard dependencies. Since this type of
# dependency is realized within this bundle as a symlink to another bundle,
# label these dependencies as 'references' in the UI.
fields['hard_dependencies'] = ''
if info['hard_dependencies']:
deps = info['hard_dependencies']
if len(deps) == 1 and not deps[0]['child_path']:
fields['hard_dependencies'] = 'Reference:\n %s\n' % (
path_util.safe_join(deps[0]['parent_uuid'], deps[0]['parent_path']),)
else:
fields['hard_dependencies'] = 'References:\n%s\n' % ('\n'.join(
' %s:%s' % (
dep['child_path'],
path_util.safe_join(dep['parent_uuid'], dep['parent_path']),
) for dep in sorted(deps, key=lambda dep: dep['child_path'])
))
# Compute a nicely-formatted failure message, if this bundle failed.
# It is possible for bundles that are not failed to have failure messages:
# for example, if a bundle is killed in the database after running for too
# long then succeeds afterwards, it will be in this state.
fields['failure_message'] = ''
if info['state'] == State.FAILED and metadata['failure_message']:
fields['failure_message'] = 'Failure message:\n %s\n' % ('\n '.join(
metadata['failure_message'].split('\n')
))
# Return the formatted summary of the bundle info.
return '''
{bundle_type}: {name}
{description}
UUID: {uuid}
Hash: {data_hash}
State: {state}
{stats}{hard_dependencies}{failure_message}
'''.format(**fields).strip()
def _make_bundle(self, bundle):
try:
bundle_location = self._bundle_store.get_bundle_location(
bundle.uuid)
path = os.path.normpath(bundle_location)
deps = []
for dep in bundle.dependencies:
parent_bundle_path = os.path.normpath(
self._bundle_store.get_bundle_location(dep.parent_uuid))
dependency_path = os.path.normpath(
os.path.join(parent_bundle_path, dep.parent_path))
if not dependency_path.startswith(parent_bundle_path) or (
not os.path.islink(dependency_path)
and not os.path.exists(dependency_path)):
raise Exception('Invalid dependency %s' %
(path_util.safe_join(
dep.parent_uuid, dep.parent_path)))
child_path = os.path.normpath(
os.path.join(path, dep.child_path))
if not child_path.startswith(path):
raise Exception('Invalid key for dependency: %s' %
(dep.child_path))
deps.append((dependency_path, child_path))
remove_path(path)
if len(deps) == 1 and deps[0][1] == path:
path_util.copy(deps[0][0], path, follow_symlinks=False)
else:
os.mkdir(path)
for dependency_path, child_path in deps:
path_util.copy(dependency_path,
child_path,
follow_symlinks=False)
self._model.update_disk_metadata(bundle,
bundle_location,
enforce_disk_quota=True)
logger.info('Finished making bundle %s', bundle.uuid)
self._model.update_bundle(bundle, {'state': State.READY})
except Exception as e:
logger.info('Failing bundle %s: %s', bundle.uuid, str(e))
self._model.update_bundle(bundle, {
'state': State.FAILED,
'metadata': {
'failure_message': str(e)
}
})
finally:
with self._make_uuids_lock:
self._make_uuids.remove(bundle.uuid)
def process_dep(dep):
parent = parent_dict[dep.parent_uuid]
# Compute an absolute target and check that the dependency exists.
if not parent.data_hash:
raise UsageError("Parent %s does not have data hash" % parent)
target = path_util.safe_join(bundle_store.get_location(parent.data_hash), dep.parent_path)
if not os.path.exists(target):
parent_spec = getattr(parent.metadata, "name", parent.uuid)
target_text = path_util.safe_join(parent_spec, dep.parent_path)
raise UsageError("Target not found: %s" % (target_text,))
if relative_symlinks:
# Create a symlink that points to the dependency's relative target.
target = path_util.safe_join(
(os.pardir if dep.child_path else ""),
bundle_store.get_location(parent.data_hash, relative=True),
dep.parent_path,
)
link_path = path_util.safe_join(dest_path, dep.child_path)
return (target, link_path)
def process_dep(dep):
parent = parent_dict[dep.parent_uuid]
# Compute an absolute target and check that the dependency exists.
if not parent.data_hash:
raise UsageError('Parent %s does not have data hash' % parent)
target = path_util.safe_join(
bundle_store.get_location(parent.data_hash),
dep.parent_path,
)
if not os.path.exists(target):
parent_spec = getattr(parent.metadata, 'name', parent.uuid)
target_text = path_util.safe_join(parent_spec, dep.parent_path)
raise UsageError('Target not found: %s' % (target_text, ))
if relative_symlinks:
# Create a symlink that points to the dependency's relative target.
target = path_util.safe_join(
(os.pardir if dep.child_path else ''),
bundle_store.get_location(parent.data_hash, relative=True),
dep.parent_path,
)
link_path = path_util.safe_join(dest_path, dep.child_path)
return (target, link_path)
def get_target_path(bundle_store, model, target):
"""
Return the on-disk location of the target (bundle_uuid, subpath) pair.
"""
(uuid, path) = target
bundle = model.get_bundle(uuid)
if not bundle.data_hash:
# Note that the bundle might not be ready, but return the location anyway to the temporary directory.
bundle_root = get_current_location(bundle_store, uuid)
else:
bundle_root = bundle_store.get_location(bundle.data_hash)
final_path = path_util.safe_join(bundle_root, path)
result = path_util.TargetPath(final_path, target)
return result
def _make_bundle(self, bundle):
try:
path = os.path.normpath(self._bundle_store.get_bundle_location(bundle.uuid))
deps = []
for dep in bundle.dependencies:
parent_bundle_path = os.path.normpath(
self._bundle_store.get_bundle_location(dep.parent_uuid)
)
dependency_path = os.path.normpath(
os.path.join(parent_bundle_path, dep.parent_path)
)
if not dependency_path.startswith(parent_bundle_path) or (
not os.path.islink(dependency_path) and not os.path.exists(dependency_path)
):
raise Exception(
'Invalid dependency %s'
% (path_util.safe_join(dep.parent_uuid, dep.parent_path))
)
child_path = os.path.normpath(os.path.join(path, dep.child_path))
if not child_path.startswith(path):
raise Exception('Invalid key for dependency: %s' % (dep.child_path))
deps.append((dependency_path, child_path))
remove_path(path)
if len(deps) == 1 and deps[0][1] == path:
path_util.copy(deps[0][0], path, follow_symlinks=False)
else:
os.mkdir(path)
for dependency_path, child_path in deps:
path_util.copy(dependency_path, child_path, follow_symlinks=False)
self._upload_manager.update_metadata_and_save(bundle, enforce_disk_quota=True)
logger.info('Finished making bundle %s', bundle.uuid)
self._model.update_bundle(bundle, {'state': State.READY})
except Exception as e:
logger.info('Failing bundle %s: %s', bundle.uuid, str(e))
self._model.update_bundle(
bundle, {'state': State.FAILED, 'metadata': {'failure_message': str(e)}}
)
finally:
with self._make_uuids_lock:
self._make_uuids.remove(bundle.uuid)
def get_target_path(bundle_store, model, target):
'''
Return the on-disk location of the target (bundle_spec, path) pair.
'''
(bundle_spec, path) = target
uuid = get_spec_uuid(model, bundle_spec)
bundle = model.get_bundle(uuid)
if not bundle.data_hash:
message = 'Unexpected: %s is ready but it has no data hash!' % (bundle,)
precondition(bundle.state != State.READY, message)
if bundle.state == State.FAILED:
raise UsageError('%s failed unrecoverably' % (bundle,))
else:
raise UsageError('%s has not yet been executed' % (bundle,))
bundle_root = bundle_store.get_location(bundle.data_hash)
final_path = path_util.safe_join(bundle_root, path)
result = path_util.TargetPath(final_path)
result.target = target
return result
def get_target_path(bundle_store, model, target):
'''
Return the on-disk location of the target (bundle_uuid, subpath) pair.
'''
(uuid, path) = target
bundle = model.get_bundle(uuid)
if not bundle.data_hash:
# Note that the bundle might not be done, but return the location anyway to the temporary directory
bundle_root = get_current_location(bundle_store, uuid)
else:
bundle_root = bundle_store.get_location(bundle.data_hash)
final_path = path_util.safe_join(bundle_root, path)
# This is too restrictive because it means we can't follow any of the
# components of a make bundle.
#path_util.check_under_path(final_path, bundle_root)
result = path_util.TargetPath(final_path)
result.target = target
return result
def _make_bundle(self, bundle):
try:
bundle_link_url = getattr(bundle.metadata, "link_url", None)
bundle_location = bundle_link_url or self._bundle_store.get_bundle_location(
bundle.uuid)
path = os.path.normpath(bundle_location)
deps = []
parent_bundle_link_urls = self._model.get_bundle_metadata(
[dep.parent_uuid for dep in bundle.dependencies], "link_url")
for dep in bundle.dependencies:
parent_bundle_link_url = parent_bundle_link_urls.get(
dep.parent_uuid)
try:
parent_bundle_path = parent_bundle_link_url or os.path.normpath(
self._bundle_store.get_bundle_location(
dep.parent_uuid))
except NotFoundError:
raise Exception('Invalid dependency %s' %
(path_util.safe_join(
dep.parent_uuid, dep.parent_path)))
# TODO(Ashwin): make this logic non-fs specific.
dependency_path = os.path.normpath(
os.path.join(parent_bundle_path, dep.parent_path))
if not dependency_path.startswith(parent_bundle_path) or (
not os.path.islink(dependency_path)
and not os.path.exists(dependency_path)):
raise Exception('Invalid dependency %s' %
(path_util.safe_join(
dep.parent_uuid, dep.parent_path)))
child_path = os.path.normpath(
os.path.join(path, dep.child_path))
if not child_path.startswith(path):
raise Exception('Invalid key for dependency: %s' %
(dep.child_path))
deps.append((dependency_path, child_path))
remove_path(path)
if len(deps) == 1 and deps[0][1] == path:
path_util.copy(deps[0][0], path, follow_symlinks=False)
else:
os.mkdir(path)
for dependency_path, child_path in deps:
path_util.copy(dependency_path,
child_path,
follow_symlinks=False)
# TODO(Ashwin): fix
self._model.update_disk_metadata(bundle,
bundle_location,
enforce_disk_quota=True)
logger.info('Finished making bundle %s', bundle.uuid)
self._model.update_bundle(bundle, {'state': State.READY})
except Exception as e:
logger.info('Failing bundle %s: %s', bundle.uuid, str(e))
self._model.update_bundle(
bundle,
{
'state': State.FAILED,
'metadata': {
'failure_message': str(e),
'error_traceback': traceback.format_exc(),
},
},
)
finally:
with self._make_uuids_lock:
self._make_uuids.remove(bundle.uuid)