def _build_blocks(self, map_key2fm_list, map_key2name, map_key2metadata_dict): # Return named dataset for key in sorted(map_key2fm_list): result = { DataProvider.Dataset: map_key2name[key[:1]], DataProvider.BlockName: map_key2name[key[:2]], } fm_list = map_key2fm_list[key] # Determine location_list location_list = None for file_location_list in ifilter(lambda s: s is not None, imap(itemgetter(3), fm_list)): location_list = location_list or [] location_list.extend(file_location_list) if location_list is not None: result[DataProvider.Locations] = list(UniqueList(location_list)) # use first file [0] to get the initial metadata_dict [1] metadata_name_list = list(fm_list[0][1].keys()) result[DataProvider.Metadata] = metadata_name_list # translate file metadata into data provider file info entries def _translate_fm2fi(url, metadata_dict, entries, location_list, obj_dict): if entries is None: entries = -1 return {DataProvider.URL: url, DataProvider.NEntries: entries, DataProvider.Metadata: lmap(metadata_dict.get, metadata_name_list)} result[DataProvider.FileList] = lsmap(_translate_fm2fi, fm_list) yield result
def makeEnum(members = None, cls = None, useHash = True): members = members or [] if cls: enumID = md5_hex(str(members) + '!' + cls.__name__)[:4] else: enumID = md5_hex(str(members))[:4] cls = type('Enum_%s_%s' % (enumID, str.join('_', members)), (), {}) def getValue(idx, name): if useHash: return idx + int(enumID, 16) else: return idx values = lsmap(getValue, enumerate(members)) cls.enumNames = members cls.enumValues = values enumMapNV = dict(izip(imap(str.lower, cls.enumNames), cls.enumValues)) enumMapVN = dict(izip(cls.enumValues, cls.enumNames)) if len(enumMapNV) != len(enumMapVN): raise APIError('Invalid enum definition!') def str2enum(cls, value, *args): return enumMapNV.get(value.lower(), *args) cls.enum2str = enumMapVN.get cls.str2enum = classmethod(str2enum) for name, value in izip(cls.enumNames, cls.enumValues): setattr(cls, name, value) return cls
def _buildBlocks(self, protoBlocks, hashNameDictDS, hashNameDictB): # Return named dataset for hashDS in sorted(protoBlocks): for hashB in sorted(protoBlocks[hashDS]): blockSEList = None for seList in ifilter( lambda s: s is not None, imap(lambda x: x[3], protoBlocks[hashDS][hashB])): blockSEList = blockSEList or [] blockSEList.extend(seList) if blockSEList is not None: blockSEList = list(UniqueList(blockSEList)) metaKeys = protoBlocks[hashDS][hashB][0][1].keys() def fnProps(path, metadata, events, seList, objStore): if events is None: events = -1 return { DataProvider.URL: path, DataProvider.NEntries: events, DataProvider.Metadata: lmap(metadata.get, metaKeys) } yield { DataProvider.Dataset: hashNameDictDS[hashDS], DataProvider.BlockName: hashNameDictB[hashB][1], DataProvider.Locations: blockSEList, DataProvider.Metadata: list(metaKeys), DataProvider.FileList: lsmap(fnProps, protoBlocks[hashDS][hashB]) }
def _buildBlocks(self, protoBlocks, hashNameDictDS, hashNameDictB): # Return named dataset for hashDS in protoBlocks: for hashB in protoBlocks[hashDS]: blockSEList = None for seList in ifilter(lambda s: s is not None, imap(lambda x: x[3], protoBlocks[hashDS][hashB])): blockSEList = blockSEList or [] blockSEList.extend(seList) if blockSEList is not None: blockSEList = list(set(blockSEList)) metaKeys = protoBlocks[hashDS][hashB][0][1].keys() def fnProps(path, metadata, events, seList, objStore): return {DataProvider.URL: path, DataProvider.NEntries: events, DataProvider.Metadata: lmap(metadata.get, metaKeys)} yield { DataProvider.Dataset: hashNameDictDS[hashDS], DataProvider.BlockName: hashNameDictB[hashB][1], DataProvider.Locations: blockSEList, DataProvider.Metadata: list(metaKeys), DataProvider.FileList: lsmap(fnProps, protoBlocks[hashDS][hashB]) }