def get_lines(name):
lines = self.head_target((uuid, name), self.HEAD_MAX_LINES)
if lines is not None:
import base64
lines = ''.join(map(base64.b64decode, lines))
return formatting.verbose_contents_str(lines)
def friendly_render_dep(dep):
key = dep['child_path'] or dep['parent_name']
friendly_parent_name = formatting.verbose_contents_str(
dep['parent_name'])
value = key + '{' + (friendly_parent_name + ':' if key != dep['parent_name'] else '') + \
dep['parent_uuid'][0:4] + '}'
return key, value
def get_lines(name):
lines = self.head_target((uuid, name), self.HEAD_MAX_LINES)
if lines is not None:
import base64
lines = ''.join(map(base64.b64decode, lines))
return formatting.verbose_contents_str(lines)
def friendly_render_dep(dep):
key = dep['child_path'] or dep['parent_name']
friendly_parent_name = formatting.verbose_contents_str(dep['parent_name'])
value = "%s{%s%s}" % (
key,
friendly_parent_name + ':' if key != dep['parent_name'] else '',
dep['parent_uuid'][0:4],
)
return key, value
def friendly_render_dep(dep):
key = dep['child_path'] or dep['parent_name']
friendly_parent_name = formatting.verbose_contents_str(dep['parent_name'])
value = (
key
+ '{'
+ (friendly_parent_name + ':' if key != dep['parent_name'] else '')
+ dep['parent_uuid'][0:4]
+ '}'
)
return key, value
def get_summary(info, name):
if info['type'] == 'file':
TRUNCATION_TEXT = (
'\n'
'... Truncated. Click link above to see full file. ...\n'
'\n')
contents = local.download_manager.summarize_file(
uuid, name,
num_head_lines=50, num_tail_lines=50, max_line_length=128,
truncation_text=TRUNCATION_TEXT, gzipped=False)
return formatting.verbose_contents_str(contents)
elif info['type'] == 'link':
return ' -> ' + info['link']
def get_summary(download_manager, info, name):
if info['type'] == 'file':
TRUNCATION_TEXT = (
'\n'
'... Truncated. Click link above to see full file. ...\n'
'\n')
contents = download_manager.summarize_file(
uuid, name,
num_head_lines=50, num_tail_lines=50, max_line_length=128,
truncation_text=TRUNCATION_TEXT, gzipped=False)
return formatting.verbose_contents_str(contents)
elif info['type'] == 'link':
return ' -> ' + info['link']
def interpret_genpath(bundle_info, genpath, db_model=None, owner_cache=None):
"""
Quickly interpret the genpaths (generalized path) that only require looking
bundle_info (e.g., 'time', 'command'). The interpretation of generalized
paths that require reading files is done by interpret_file_genpath.
If genpath is referring to a file, then just returns instructions for fetching that file rather than actually doing it.
:param bundle_info: dictionary which contains metadata of current bundle's information, e.g. uuid, bundle_type, owner_id, etc.
:param genpath: a generalized path, e.g. column names(summary, owner, etc.), args.
:param db_model (optional): database model which is used to query database
:param owner_cache (optional): a dictionary stores mappings from owner_id to owner
:return: the interpretation of genpath
"""
if is_file_genpath(genpath):
return (bundle_info['uuid'], genpath)
# Render dependencies
deps = bundle_info.get('dependencies', [])
anonymous = len(deps) == 1 and deps[0]['child_path'] == ''
def render_dep(dep, show_key=True, show_uuid=False):
if show_key and not anonymous:
if show_uuid or dep['child_path'] != dep['parent_name']:
a = dep['child_path'] + ':'
else:
a = ':'
else:
a = ''
b = dep['parent_uuid'] if show_uuid else (dep['parent_name'] or '')
c = '/' + dep['parent_path'] if dep['parent_path'] else ''
return a + b + c
# Special genpaths (dependencies, args)
if genpath == 'dependencies':
return ','.join([render_dep(dep) for dep in deps])
elif genpath.startswith('dependencies/'):
# Look up the particular dependency
_, name = genpath.split('/', 1)
for dep in deps:
if dep['child_path'] == name:
return render_dep(dep, show_key=False)
return formatting.verbose_contents_str(None)
elif genpath == 'args':
# Arguments that we would pass to 'cl'
args = []
bundle_type = bundle_info.get('bundle_type')
if bundle_type not in ('make', 'run'):
return None
args += [bundle_type]
# Dependencies
for dep in deps:
args.append(render_dep(dep, show_uuid=True))
# Command
if bundle_info['command']:
args.append(formatting.quote(bundle_info['command']))
# Add request arguments from metadata
metadata = bundle_info['metadata']
for key, value in metadata.items():
if key.startswith('request_') and value:
key = key.replace('_', '-')
if isinstance(value, bool):
args.append('--' + key)
else:
args.extend(['--' + key, formatting.quote(str(value))])
return ' '.join(args)
elif genpath == 'summary':
def friendly_render_dep(dep):
key = dep['child_path'] or dep['parent_name']
friendly_parent_name = formatting.verbose_contents_str(
dep['parent_name'])
value = (key + '{' + (friendly_parent_name +
':' if key != dep['parent_name'] else '') +
dep['parent_uuid'][0:4] + '}')
return key, value
# Nice easy-to-ready description of how this bundle got created.
bundle_type = bundle_info.get('bundle_type')
if bundle_type in ('dataset', 'program'):
return '[uploaded]'
if bundle_type == 'make':
args = []
for dep in deps:
args.append(friendly_render_dep(dep)[1])
return '= ' + ' '.join(args)
elif bundle_type == 'run':
return '! ' + bundle_info['command']
elif genpath == 'host_worksheets':
if 'host_worksheets' in bundle_info:
return ' '.join('%s(%s)' % (info['name'], info['uuid'])
for info in bundle_info['host_worksheets'])
elif genpath == 'permission':
if 'permission' in bundle_info:
return permission_str(bundle_info['permission'])
elif genpath == 'group_permissions':
if 'group_permissions' in bundle_info:
# FIXME(sckoo): we will be passing the old permissions format into this
# which has been updated to accommodate the new formatting
return group_permissions_str(bundle_info['group_permissions'])
elif genpath == 'owner':
if 'owner_id' in bundle_info:
if owner_cache is not None and bundle_info[
'owner_id'] in owner_cache:
return owner_cache[bundle_info['owner_id']]
else:
# We might batch this database operation in the future
owner = db_model.get_user(user_id=bundle_info['owner_id'])
owner_cache[bundle_info['owner_id']] = owner.user_name
return owner.user_name
# Bundle field?
value = bundle_info.get(genpath)
if value is not None:
return value
# Metadata field?
value = bundle_info.get('metadata', {}).get(genpath)
if value is not None:
return value
return None
def interpret_genpath(bundle_info, genpath):
"""
This function is called in the first server call to a BundleClient to
quickly interpret the genpaths (generalized path) that only require looking
bundle_info (e.g., 'time', 'command'). The interpretation of generalized
paths that require reading files is done by interpret_file_genpath.
"""
# If genpath is referring to a file, then just returns instructions for
# fetching that file rather than actually doing it.
if is_file_genpath(genpath):
return (bundle_info['uuid'], genpath)
# Render dependencies
deps = bundle_info.get('dependencies', [])
anonymous = len(deps) == 1 and deps[0]['child_path'] == ''
def render_dep(dep, show_key=True, show_uuid=False):
if show_key and not anonymous:
if show_uuid or dep['child_path'] != dep['parent_name']:
a = dep['child_path'] + ':'
else:
a = ':'
else:
a = ''
b = dep['parent_uuid'] if show_uuid else (dep['parent_name'] or '')
c = '/' + dep['parent_path'] if dep['parent_path'] else ''
return a + b + c
# Special genpaths (dependencies, args)
if genpath == 'dependencies':
return ','.join([render_dep(dep) for dep in deps])
elif genpath.startswith('dependencies/'):
# Look up the particular dependency
_, name = genpath.split('/', 1)
for dep in deps:
if dep['child_path'] == name:
return render_dep(dep, show_key=False)
return formatting.verbose_contents_str(None)
elif genpath == 'args':
# Arguments that we would pass to 'cl'
args = []
bundle_type = bundle_info.get('bundle_type')
if bundle_type not in ('make', 'run'): return None
args += [bundle_type]
# Dependencies
for dep in deps:
args.append(render_dep(dep, show_uuid=True))
# Command
if bundle_info['command']:
args.append(formatting.quote(bundle_info['command']))
# Add request arguments from metadata
metadata = bundle_info['metadata']
for key, value in metadata.items():
if key.startswith('request_') and value:
key = key.replace('_', '-')
if isinstance(value, bool):
args.append('--' + key)
else:
args.extend(['--' + key, formatting.quote(str(value))])
return ' '.join(args)
elif genpath == 'summary':
def friendly_render_dep(dep):
key = dep['child_path'] or dep['parent_name']
value = key + '{' + (dep['parent_name'] + ':' if key != dep['parent_name'] else '') + \
dep['parent_uuid'][0:4] + '}'
return key, value
# Nice easy-to-ready description of how this bundle got created.
bundle_type = bundle_info.get('bundle_type')
if bundle_type in ('dataset', 'program'):
return '[uploaded]'
if bundle_type == 'make':
args = []
for dep in deps:
args.append(friendly_render_dep(dep)[1])
return '= ' + ' '.join(args)
elif bundle_type == 'run':
command = bundle_info['command']
for dep in deps:
key, value = friendly_render_dep(dep)
command = command.replace(key, value)
return '! ' + command
elif genpath == 'host_worksheets':
if 'host_worksheets' in bundle_info:
return ' '.join('%s(%s)' % (info['name'], info['uuid']) for info in bundle_info['host_worksheets'])
elif genpath == 'permission':
if 'permission' in bundle_info:
return permission_str(bundle_info['permission'])
elif genpath == 'group_permissions':
if 'group_permissions' in bundle_info:
return group_permissions_str(bundle_info['group_permissions'])
# Bundle field?
value = bundle_info.get(genpath)
if value is not None: return value
# Metadata field?
value = bundle_info.get('metadata', {}).get(genpath)
if value is not None: return value
return None
def interpret_genpath(bundle_info, genpath):
"""
Quickly interpret the genpaths (generalized path) that only require looking
bundle_info (e.g., 'time', 'command'). The interpretation of generalized
paths that require reading files is done by interpret_file_genpath.
"""
# If genpath is referring to a file, then just returns instructions for
# fetching that file rather than actually doing it.
if is_file_genpath(genpath):
return (bundle_info['uuid'], genpath)
# Render dependencies
deps = bundle_info.get('dependencies', [])
anonymous = len(deps) == 1 and deps[0]['child_path'] == ''
def render_dep(dep, show_key=True, show_uuid=False):
if show_key and not anonymous:
if show_uuid or dep['child_path'] != dep['parent_name']:
a = dep['child_path'] + ':'
else:
a = ':'
else:
a = ''
b = dep['parent_uuid'] if show_uuid else (dep['parent_name'] or '')
c = '/' + dep['parent_path'] if dep['parent_path'] else ''
return a + b + c
# Special genpaths (dependencies, args)
if genpath == 'dependencies':
return ','.join([render_dep(dep) for dep in deps])
elif genpath.startswith('dependencies/'):
# Look up the particular dependency
_, name = genpath.split('/', 1)
for dep in deps:
if dep['child_path'] == name:
return render_dep(dep, show_key=False)
return formatting.verbose_contents_str(None)
elif genpath == 'args':
# Arguments that we would pass to 'cl'
args = []
bundle_type = bundle_info.get('bundle_type')
if bundle_type not in ('make', 'run'): return None
args += [bundle_type]
# Dependencies
for dep in deps:
args.append(render_dep(dep, show_uuid=True))
# Command
if bundle_info['command']:
args.append(formatting.quote(bundle_info['command']))
# Add request arguments from metadata
metadata = bundle_info['metadata']
for key, value in metadata.items():
if key.startswith('request_') and value:
key = key.replace('_', '-')
if isinstance(value, bool):
args.append('--' + key)
else:
args.extend(['--' + key, formatting.quote(str(value))])
return ' '.join(args)
elif genpath == 'summary':
def friendly_render_dep(dep):
key = dep['child_path'] or dep['parent_name']
friendly_parent_name = formatting.verbose_contents_str(dep['parent_name'])
value = key + '{' + (friendly_parent_name + ':' if key != dep['parent_name'] else '') + \
dep['parent_uuid'][0:4] + '}'
return key, value
# Nice easy-to-ready description of how this bundle got created.
bundle_type = bundle_info.get('bundle_type')
if bundle_type in ('dataset', 'program'):
return '[uploaded]'
if bundle_type == 'make':
args = []
for dep in deps:
args.append(friendly_render_dep(dep)[1])
return '= ' + ' '.join(args)
elif bundle_type == 'run':
command = bundle_info['command']
for dep in deps:
key, value = friendly_render_dep(dep)
# Replace full-word occurrences of key in the command with an indicator of the dependency.
# Of course, a string match in the command isn't necessary a semantic reference to the dependency,
# and there are some dependencies which are not explicit in the command.
# But this can be seen as a best-effort attempt.
command = re.sub(r'\b%s\b' % key, value, command)
return '! ' + command
elif genpath == 'host_worksheets':
if 'host_worksheets' in bundle_info:
return ' '.join('%s(%s)' % (info['name'], info['uuid']) for info in bundle_info['host_worksheets'])
elif genpath == 'permission':
if 'permission' in bundle_info:
return permission_str(bundle_info['permission'])
elif genpath == 'group_permissions':
if 'group_permissions' in bundle_info:
# FIXME(sckoo): we will be passing the old permissions format into this
# which has been updated to accommodate the new formatting
return group_permissions_str(bundle_info['group_permissions'])
# Bundle field?
value = bundle_info.get(genpath)
if value is not None: return value
# Metadata field?
value = bundle_info.get('metadata', {}).get(genpath)
if value is not None: return value
return None
def interpret_genpath(bundle_info, genpath):
"""
Quickly interpret the genpaths (generalized path) that only require looking
bundle_info (e.g., 'time', 'command'). The interpretation of generalized
paths that require reading files is done by interpret_file_genpath.
"""
# If genpath is referring to a file, then just returns instructions for
# fetching that file rather than actually doing it.
if is_file_genpath(genpath):
return (bundle_info['uuid'], genpath)
# Render dependencies
deps = bundle_info.get('dependencies', [])
anonymous = len(deps) == 1 and deps[0]['child_path'] == ''
def render_dep(dep, show_key=True, show_uuid=False):
if show_key and not anonymous:
if show_uuid or dep['child_path'] != dep['parent_name']:
a = dep['child_path'] + ':'
else:
a = ':'
else:
a = ''
b = dep['parent_uuid'] if show_uuid else (dep['parent_name'] or '')
c = '/' + dep['parent_path'] if dep['parent_path'] else ''
return a + b + c
# Special genpaths (dependencies, args)
if genpath == 'dependencies':
return ','.join([render_dep(dep) for dep in deps])
elif genpath.startswith('dependencies/'):
# Look up the particular dependency
_, name = genpath.split('/', 1)
for dep in deps:
if dep['child_path'] == name:
return render_dep(dep, show_key=False)
return formatting.verbose_contents_str(None)
elif genpath == 'args':
# Arguments that we would pass to 'cl'
args = []
bundle_type = bundle_info.get('bundle_type')
if bundle_type not in ('make', 'run'):
return None
args += [bundle_type]
# Dependencies
for dep in deps:
args.append(render_dep(dep, show_uuid=True))
# Command
if bundle_info['command']:
args.append(formatting.quote(bundle_info['command']))
# Add request arguments from metadata
metadata = bundle_info['metadata']
for key, value in metadata.items():
if key.startswith('request_') and value:
key = key.replace('_', '-')
if isinstance(value, bool):
args.append('--' + key)
else:
args.extend(['--' + key, formatting.quote(str(value))])
return ' '.join(args)
elif genpath == 'summary':
def friendly_render_dep(dep):
key = dep['child_path'] or dep['parent_name']
friendly_parent_name = formatting.verbose_contents_str(dep['parent_name'])
value = (
key
+ '{'
+ (friendly_parent_name + ':' if key != dep['parent_name'] else '')
+ dep['parent_uuid'][0:4]
+ '}'
)
return key, value
# Nice easy-to-ready description of how this bundle got created.
bundle_type = bundle_info.get('bundle_type')
if bundle_type in ('dataset', 'program'):
return '[uploaded]'
if bundle_type == 'make':
args = []
for dep in deps:
args.append(friendly_render_dep(dep)[1])
return '= ' + ' '.join(args)
elif bundle_type == 'run':
command = bundle_info['command']
for dep in deps:
key, value = friendly_render_dep(dep)
# Replace full-word occurrences of key in the command with an indicator of the dependency.
# Of course, a string match in the command isn't necessary a semantic reference to the dependency,
# and there are some dependencies which are not explicit in the command.
# But this can be seen as a best-effort attempt.
command = re.sub(r'\b%s\b' % key, value, command)
return '! ' + command
elif genpath == 'host_worksheets':
if 'host_worksheets' in bundle_info:
return ' '.join(
'%s(%s)' % (info['name'], info['uuid']) for info in bundle_info['host_worksheets']
)
elif genpath == 'permission':
if 'permission' in bundle_info:
return permission_str(bundle_info['permission'])
elif genpath == 'group_permissions':
if 'group_permissions' in bundle_info:
# FIXME(sckoo): we will be passing the old permissions format into this
# which has been updated to accommodate the new formatting
return group_permissions_str(bundle_info['group_permissions'])
# Bundle field?
value = bundle_info.get(genpath)
if value is not None:
return value
# Metadata field?
value = bundle_info.get('metadata', {}).get(genpath)
if value is not None:
return value
return None