def test_apply_func(self):
'''
Test apply_func for rendering values in worksheets.
'''
self.assertEqual(worksheet_util.apply_func(None, 'hello'), 'hello')
self.assertEqual(worksheet_util.apply_func('[1:2]', 'hello'), 'e')
self.assertEqual(worksheet_util.apply_func('[:2]', 'hello'), 'he')
self.assertEqual(worksheet_util.apply_func('[2:]', 'hello'), 'llo')
self.assertEqual(worksheet_util.apply_func('date', '1427467247'), '2015-03-27 07:40:47')
self.assertEqual(worksheet_util.apply_func('duration', '63'), '1m3s')
self.assertEqual(worksheet_util.apply_func('size', '1024'), '1K')
self.assertEqual(worksheet_util.apply_func('s/a/b', 'aa'), 'bb')
self.assertEqual(worksheet_util.apply_func(r's/(.+)\/(.+)/\2\/\1', '3/10'), '10/3')
self.assertEqual(worksheet_util.apply_func('%.2f', '1.2345'), '1.23')
def get_bundle_info(self, uuid):
## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True))
# format permission data
bundle_info['permission_str'] = permission_str(bundle_info['permission'])
# format each groups as well
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
# format based on specs from the cli
for spec in cls.METADATA_SPECS:
key = spec.key
if key not in metadata:
continue
if metadata[key] == '' or metadata[key] == []:
continue
value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
# if isinstance(value, list):
# value = ', '.join(value)
metadata[key] = value
bundle_info['metadata'] = metadata
return bundle_info
def test_apply_func(self):
'''
Test apply_func for rendering values in worksheets.
'''
self.assertEqual(worksheet_util.apply_func(None, 'hello'), 'hello')
self.assertEqual(worksheet_util.apply_func('[1:2]', 'hello'), 'e')
self.assertEqual(worksheet_util.apply_func('[:2]', 'hello'), 'he')
self.assertEqual(worksheet_util.apply_func('[2:]', 'hello'), 'llo')
self.assertEqual(
worksheet_util.apply_func('date', '1427467247')[:10], '2015-03-27'
) # Don't test time because of time zones
self.assertEqual(worksheet_util.apply_func('duration', '63'), '1m3s')
self.assertEqual(worksheet_util.apply_func('size', '1024'), '1k')
self.assertEqual(worksheet_util.apply_func('s/a/b', 'aa'), 'bb')
self.assertEqual(worksheet_util.apply_func(r's/(.+)\/(.+)/\2\/\1', '3/10'), '10/3')
self.assertEqual(worksheet_util.apply_func('%.2f', '1.2345'), '1.23')
def get_bundle_info(self, uuid):
## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False):
bundle_info = _call_with_retries(
lambda: self.client.get_bundle_info(uuid, True, True, True))
# format permission data
bundle_info['permission_str'] = permission_str(
bundle_info['permission'])
# format each groups as well
for group_permission in bundle_info['group_permissions']:
group_permission['permission_str'] = permission_str(
group_permission['permission'])
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
# format based on specs from the cli
for spec in cls.METADATA_SPECS:
key = spec.key
if key not in metadata:
continue
if metadata[key] == '' or metadata[key] == []:
continue
value = worksheet_util.apply_func(spec.formatting,
metadata.get(key))
# if isinstance(value, list):
# value = ', '.join(value)
metadata[key] = value
bundle_info['metadata'] = metadata
return bundle_info
def get_bundle_info(self, uuid):
bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid))
metadata = bundle_info['metadata']
cls = get_bundle_subclass(bundle_info['bundle_type'])
# format based on specs from the cli
for spec in cls.METADATA_SPECS:
key = spec.key
if key not in metadata:
continue
if metadata[key] == '' or metadata[key] == []:
continue
value = worksheet_util.apply_func(spec.formatting, metadata.get(key))
# if isinstance(value, list):
# value = ', '.join(value)
metadata[key] = value
bundle_info['metadata'] = metadata
return bundle_info
def interpret_file_genpath(target_cache, bundle_uuid, genpath, post):
"""
|cache| is a mapping from target (bundle_uuid, subpath) to the info map,
which is to be read/written to avoid reading/parsing the same file many
times.
|genpath| specifies the subpath and various fields (e.g., for
/stats:train/errorRate, subpath = 'stats', key = 'train/errorRate').
|post| function to apply to the resulting value.
Return the string value.
"""
MAX_LINES = 10000 # Maximum number of lines we need to read from a file.
# Load the file
if not is_file_genpath(genpath):
raise UsageError('Not file genpath: %s' % genpath)
genpath = genpath[1:]
if ':' in genpath: # Looking for a particular key in the file
subpath, key = genpath.split(':')
else:
subpath, key = genpath, None
target = (bundle_uuid, subpath)
if target not in target_cache:
info = None
try:
target_info = rest_util.get_target_info(target, 0)
if target_info['type'] == 'file':
contents = head_target(target, MAX_LINES)
if len(contents) == 0:
info = ''
elif all('\t' in x for x in contents):
# Tab-separated file (key\tvalue\nkey\tvalue...)
info = {}
for x in contents:
kv = x.strip().split("\t", 1)
if len(kv) == 2:
info[kv[0]] = kv[1]
else:
try:
# JSON file
info = json.loads(''.join(contents))
except (TypeError, ValueError):
try:
# YAML file
# Use safe_load because yaml.load() could execute
# arbitrary Python code
info = yaml.safe_load(''.join(contents))
except yaml.YAMLError:
# Plain text file
info = ''.join(contents)
except NotFoundError:
pass
# Try to interpret the structure of the file by looking inside it.
target_cache[target] = info
# Traverse the info object.
info = target_cache.get(target, None)
if key is not None and info is not None:
for k in key.split('/'):
if isinstance(info, dict):
info = info.get(k, None)
elif isinstance(info, list):
try:
info = info[int(k)]
except (KeyError, ValueError):
info = None
else:
info = None
if info is None:
break
return apply_func(post, info)
def short_uuid(full_uuid):
return worksheet_util.apply_func('[0:8]', full_uuid)
def interpret_file_genpath(target_cache, bundle_uuid, genpath, post):
"""
|cache| is a mapping from target (bundle_uuid, subpath) to the info map,
which is to be read/written to avoid reading/parsing the same file many
times.
|genpath| specifies the subpath and various fields (e.g., for
/stats:train/errorRate, subpath = 'stats', key = 'train/errorRate').
|post| function to apply to the resulting value.
Return the string value.
"""
MAX_LINES = 10000 # Maximum number of lines we need to read from a file.
# Load the file
if not is_file_genpath(genpath):
raise UsageError('Not file genpath: %s' % genpath)
genpath = genpath[1:]
if ':' in genpath: # Looking for a particular key in the file
subpath, key = genpath.split(':')
else:
subpath, key = genpath, None
target = (bundle_uuid, subpath)
if target not in target_cache:
target_info = rest_util.get_target_info(target, 0)
# Try to interpret the structure of the file by looking inside it.
if target_info is not None and target_info['type'] == 'file':
contents = head_target(target, MAX_LINES)
if len(contents) == 0:
info = ''
elif all('\t' in x for x in contents):
# Tab-separated file (key\tvalue\nkey\tvalue...)
info = {}
for x in contents:
kv = x.strip().split("\t", 1)
if len(kv) == 2: info[kv[0]] = kv[1]
else:
try:
# JSON file
info = json.loads(''.join(contents))
except (TypeError, ValueError):
try:
# YAML file
# Use safe_load because yaml.load() could execute
# arbitrary Python code
info = yaml.safe_load(''.join(contents))
except yaml.YAMLError:
# Plain text file
info = ''.join(contents)
else:
info = None
target_cache[target] = info
# Traverse the info object.
info = target_cache.get(target, None)
if key is not None and info is not None:
for k in key.split('/'):
if isinstance(info, dict):
info = info.get(k, None)
elif isinstance(info, list):
try:
info = info[int(k)]
except (KeyError, ValueError):
info = None
else:
info = None
if info is None: break
return apply_func(post, info)
def short_uuid(full_uuid):
return worksheet_util.apply_func('[0:8]', full_uuid)
def short_uuid(full_uuid):
from codalab.lib import worksheet_util
return worksheet_util.apply_func('[0:8]', full_uuid)