def _build_blocks(self, map_key2fm_list, map_key2name, map_key2metadata_dict):
# Return named dataset
for key in sorted(map_key2fm_list):
result = {
DataProvider.Dataset: map_key2name[key[:1]],
DataProvider.BlockName: map_key2name[key[:2]],
}
fm_list = map_key2fm_list[key]
# Determine location_list
location_list = None
for file_location_list in ifilter(lambda s: s is not None, imap(itemgetter(3), fm_list)):
location_list = location_list or []
location_list.extend(file_location_list)
if location_list is not None:
result[DataProvider.Locations] = list(UniqueList(location_list))
# use first file [0] to get the initial metadata_dict [1]
metadata_name_list = list(fm_list[0][1].keys())
result[DataProvider.Metadata] = metadata_name_list
# translate file metadata into data provider file info entries
def _translate_fm2fi(url, metadata_dict, entries, location_list, obj_dict):
if entries is None:
entries = -1
return {DataProvider.URL: url, DataProvider.NEntries: entries,
DataProvider.Metadata: lmap(metadata_dict.get, metadata_name_list)}
result[DataProvider.FileList] = lsmap(_translate_fm2fi, fm_list)
yield result
def makeEnum(members = None, cls = None, useHash = True):
members = members or []
if cls:
enumID = md5_hex(str(members) + '!' + cls.__name__)[:4]
else:
enumID = md5_hex(str(members))[:4]
cls = type('Enum_%s_%s' % (enumID, str.join('_', members)), (), {})
def getValue(idx, name):
if useHash:
return idx + int(enumID, 16)
else:
return idx
values = lsmap(getValue, enumerate(members))
cls.enumNames = members
cls.enumValues = values
enumMapNV = dict(izip(imap(str.lower, cls.enumNames), cls.enumValues))
enumMapVN = dict(izip(cls.enumValues, cls.enumNames))
if len(enumMapNV) != len(enumMapVN):
raise APIError('Invalid enum definition!')
def str2enum(cls, value, *args):
return enumMapNV.get(value.lower(), *args)
cls.enum2str = enumMapVN.get
cls.str2enum = classmethod(str2enum)
for name, value in izip(cls.enumNames, cls.enumValues):
setattr(cls, name, value)
return cls
def _buildBlocks(self, protoBlocks, hashNameDictDS, hashNameDictB):
# Return named dataset
for hashDS in sorted(protoBlocks):
for hashB in sorted(protoBlocks[hashDS]):
blockSEList = None
for seList in ifilter(
lambda s: s is not None,
imap(lambda x: x[3], protoBlocks[hashDS][hashB])):
blockSEList = blockSEList or []
blockSEList.extend(seList)
if blockSEList is not None:
blockSEList = list(UniqueList(blockSEList))
metaKeys = protoBlocks[hashDS][hashB][0][1].keys()
def fnProps(path, metadata, events, seList, objStore):
if events is None:
events = -1
return {
DataProvider.URL: path,
DataProvider.NEntries: events,
DataProvider.Metadata: lmap(metadata.get, metaKeys)
}
yield {
DataProvider.Dataset:
hashNameDictDS[hashDS],
DataProvider.BlockName:
hashNameDictB[hashB][1],
DataProvider.Locations:
blockSEList,
DataProvider.Metadata:
list(metaKeys),
DataProvider.FileList:
lsmap(fnProps, protoBlocks[hashDS][hashB])
}
def _buildBlocks(self, protoBlocks, hashNameDictDS, hashNameDictB):
# Return named dataset
for hashDS in protoBlocks:
for hashB in protoBlocks[hashDS]:
blockSEList = None
for seList in ifilter(lambda s: s is not None, imap(lambda x: x[3], protoBlocks[hashDS][hashB])):
blockSEList = blockSEList or []
blockSEList.extend(seList)
if blockSEList is not None:
blockSEList = list(set(blockSEList))
metaKeys = protoBlocks[hashDS][hashB][0][1].keys()
def fnProps(path, metadata, events, seList, objStore):
return {DataProvider.URL: path, DataProvider.NEntries: events,
DataProvider.Metadata: lmap(metadata.get, metaKeys)}
yield {
DataProvider.Dataset: hashNameDictDS[hashDS],
DataProvider.BlockName: hashNameDictB[hashB][1],
DataProvider.Locations: blockSEList,
DataProvider.Metadata: list(metaKeys),
DataProvider.FileList: lsmap(fnProps, protoBlocks[hashDS][hashB])
}