def test_apply_func(self): ''' Test apply_func for rendering values in worksheets. ''' self.assertEqual(worksheet_util.apply_func(None, 'hello'), 'hello') self.assertEqual(worksheet_util.apply_func('[1:2]', 'hello'), 'e') self.assertEqual(worksheet_util.apply_func('[:2]', 'hello'), 'he') self.assertEqual(worksheet_util.apply_func('[2:]', 'hello'), 'llo') self.assertEqual(worksheet_util.apply_func('date', '1427467247'), '2015-03-27 07:40:47') self.assertEqual(worksheet_util.apply_func('duration', '63'), '1m3s') self.assertEqual(worksheet_util.apply_func('size', '1024'), '1K') self.assertEqual(worksheet_util.apply_func('s/a/b', 'aa'), 'bb') self.assertEqual(worksheet_util.apply_func(r's/(.+)\/(.+)/\2\/\1', '3/10'), '10/3') self.assertEqual(worksheet_util.apply_func('%.2f', '1.2345'), '1.23')
def get_bundle_info(self, uuid): ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False): bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid, True, True, True)) # format permission data bundle_info['permission_str'] = permission_str(bundle_info['permission']) # format each groups as well for group_permission in bundle_info['group_permissions']: group_permission['permission_str'] = permission_str(group_permission['permission']) metadata = bundle_info['metadata'] cls = get_bundle_subclass(bundle_info['bundle_type']) # format based on specs from the cli for spec in cls.METADATA_SPECS: key = spec.key if key not in metadata: continue if metadata[key] == '' or metadata[key] == []: continue value = worksheet_util.apply_func(spec.formatting, metadata.get(key)) # if isinstance(value, list): # value = ', '.join(value) metadata[key] = value bundle_info['metadata'] = metadata return bundle_info
def test_apply_func(self): ''' Test apply_func for rendering values in worksheets. ''' self.assertEqual(worksheet_util.apply_func(None, 'hello'), 'hello') self.assertEqual(worksheet_util.apply_func('[1:2]', 'hello'), 'e') self.assertEqual(worksheet_util.apply_func('[:2]', 'hello'), 'he') self.assertEqual(worksheet_util.apply_func('[2:]', 'hello'), 'llo') self.assertEqual( worksheet_util.apply_func('date', '1427467247')[:10], '2015-03-27' ) # Don't test time because of time zones self.assertEqual(worksheet_util.apply_func('duration', '63'), '1m3s') self.assertEqual(worksheet_util.apply_func('size', '1024'), '1k') self.assertEqual(worksheet_util.apply_func('s/a/b', 'aa'), 'bb') self.assertEqual(worksheet_util.apply_func(r's/(.+)\/(.+)/\2\/\1', '3/10'), '10/3') self.assertEqual(worksheet_util.apply_func('%.2f', '1.2345'), '1.23')
def get_bundle_info(self, uuid): ## def get_bundle_infos(self, uuids, get_children=False, get_host_worksheets=False, get_permissions=False): bundle_info = _call_with_retries( lambda: self.client.get_bundle_info(uuid, True, True, True)) # format permission data bundle_info['permission_str'] = permission_str( bundle_info['permission']) # format each groups as well for group_permission in bundle_info['group_permissions']: group_permission['permission_str'] = permission_str( group_permission['permission']) metadata = bundle_info['metadata'] cls = get_bundle_subclass(bundle_info['bundle_type']) # format based on specs from the cli for spec in cls.METADATA_SPECS: key = spec.key if key not in metadata: continue if metadata[key] == '' or metadata[key] == []: continue value = worksheet_util.apply_func(spec.formatting, metadata.get(key)) # if isinstance(value, list): # value = ', '.join(value) metadata[key] = value bundle_info['metadata'] = metadata return bundle_info
def get_bundle_info(self, uuid): bundle_info = _call_with_retries(lambda: self.client.get_bundle_info(uuid)) metadata = bundle_info['metadata'] cls = get_bundle_subclass(bundle_info['bundle_type']) # format based on specs from the cli for spec in cls.METADATA_SPECS: key = spec.key if key not in metadata: continue if metadata[key] == '' or metadata[key] == []: continue value = worksheet_util.apply_func(spec.formatting, metadata.get(key)) # if isinstance(value, list): # value = ', '.join(value) metadata[key] = value bundle_info['metadata'] = metadata return bundle_info
def interpret_file_genpath(target_cache, bundle_uuid, genpath, post): """ |cache| is a mapping from target (bundle_uuid, subpath) to the info map, which is to be read/written to avoid reading/parsing the same file many times. |genpath| specifies the subpath and various fields (e.g., for /stats:train/errorRate, subpath = 'stats', key = 'train/errorRate'). |post| function to apply to the resulting value. Return the string value. """ MAX_LINES = 10000 # Maximum number of lines we need to read from a file. # Load the file if not is_file_genpath(genpath): raise UsageError('Not file genpath: %s' % genpath) genpath = genpath[1:] if ':' in genpath: # Looking for a particular key in the file subpath, key = genpath.split(':') else: subpath, key = genpath, None target = (bundle_uuid, subpath) if target not in target_cache: info = None try: target_info = rest_util.get_target_info(target, 0) if target_info['type'] == 'file': contents = head_target(target, MAX_LINES) if len(contents) == 0: info = '' elif all('\t' in x for x in contents): # Tab-separated file (key\tvalue\nkey\tvalue...) info = {} for x in contents: kv = x.strip().split("\t", 1) if len(kv) == 2: info[kv[0]] = kv[1] else: try: # JSON file info = json.loads(''.join(contents)) except (TypeError, ValueError): try: # YAML file # Use safe_load because yaml.load() could execute # arbitrary Python code info = yaml.safe_load(''.join(contents)) except yaml.YAMLError: # Plain text file info = ''.join(contents) except NotFoundError: pass # Try to interpret the structure of the file by looking inside it. target_cache[target] = info # Traverse the info object. info = target_cache.get(target, None) if key is not None and info is not None: for k in key.split('/'): if isinstance(info, dict): info = info.get(k, None) elif isinstance(info, list): try: info = info[int(k)] except (KeyError, ValueError): info = None else: info = None if info is None: break return apply_func(post, info)
def short_uuid(full_uuid): return worksheet_util.apply_func('[0:8]', full_uuid)
def interpret_file_genpath(target_cache, bundle_uuid, genpath, post): """ |cache| is a mapping from target (bundle_uuid, subpath) to the info map, which is to be read/written to avoid reading/parsing the same file many times. |genpath| specifies the subpath and various fields (e.g., for /stats:train/errorRate, subpath = 'stats', key = 'train/errorRate'). |post| function to apply to the resulting value. Return the string value. """ MAX_LINES = 10000 # Maximum number of lines we need to read from a file. # Load the file if not is_file_genpath(genpath): raise UsageError('Not file genpath: %s' % genpath) genpath = genpath[1:] if ':' in genpath: # Looking for a particular key in the file subpath, key = genpath.split(':') else: subpath, key = genpath, None target = (bundle_uuid, subpath) if target not in target_cache: target_info = rest_util.get_target_info(target, 0) # Try to interpret the structure of the file by looking inside it. if target_info is not None and target_info['type'] == 'file': contents = head_target(target, MAX_LINES) if len(contents) == 0: info = '' elif all('\t' in x for x in contents): # Tab-separated file (key\tvalue\nkey\tvalue...) info = {} for x in contents: kv = x.strip().split("\t", 1) if len(kv) == 2: info[kv[0]] = kv[1] else: try: # JSON file info = json.loads(''.join(contents)) except (TypeError, ValueError): try: # YAML file # Use safe_load because yaml.load() could execute # arbitrary Python code info = yaml.safe_load(''.join(contents)) except yaml.YAMLError: # Plain text file info = ''.join(contents) else: info = None target_cache[target] = info # Traverse the info object. info = target_cache.get(target, None) if key is not None and info is not None: for k in key.split('/'): if isinstance(info, dict): info = info.get(k, None) elif isinstance(info, list): try: info = info[int(k)] except (KeyError, ValueError): info = None else: info = None if info is None: break return apply_func(post, info)
def short_uuid(full_uuid): from codalab.lib import worksheet_util return worksheet_util.apply_func('[0:8]', full_uuid)