def classifyMetadataKeys(block): def metadataHash(fi, idx): return md5_hex(repr(fi[DataProvider.Metadata][idx])) cMetadataIdx = lrange(len(block[DataProvider.Metadata])) cMetadataHash = lmap(lambda idx: metadataHash(block[DataProvider.FileList][0], idx), cMetadataIdx) for fi in block[DataProvider.FileList]: # Identify common metadata for idx in cMetadataIdx: if metadataHash(fi, idx) != cMetadataHash[idx]: cMetadataIdx.remove(idx) def filterC(common): idxList = ifilter(lambda idx: (idx in cMetadataIdx) == common, irange(len(block[DataProvider.Metadata]))) return sorted(idxList, key = lambda idx: block[DataProvider.Metadata][idx]) return (filterC(True), filterC(False))
def classifyMetadataKeys(block): def metadataHash(fi, idx): return md5_hex(repr(fi[DataProvider.Metadata][idx])) cMetadataIdx = lrange(len(block[DataProvider.Metadata])) cMetadataHash = lmap(lambda idx: metadataHash(block[DataProvider.FileList][0], idx), cMetadataIdx) for fi in block[DataProvider.FileList]: # Identify common metadata for idx in cMetadataIdx: if metadataHash(fi, idx) != cMetadataHash[idx]: cMetadataIdx.remove(idx) def filterC(common): idxList = ifilter(lambda idx: (idx in cMetadataIdx) == common, irange(len(block[DataProvider.Metadata]))) return sorted(idxList, key = lambda idx: block[DataProvider.Metadata][idx]) return (filterC(True), filterC(False))
def _split_metadata_idx_list(block):
def _get_metadata_hash(fi, idx):
if idx < len(fi[DataProvider.Metadata]):
return md5_hex(repr(fi[DataProvider.Metadata][idx]))
fi_list = block[DataProvider.FileList]
common_metadata_idx_list = lrange(len(block[DataProvider.Metadata]))
if fi_list:
common_metadata_hash_list = lmap(
lambda idx: _get_metadata_hash(fi_list[0], idx),
common_metadata_idx_list)
for fi in fi_list: # Identify common metadata
for idx in common_metadata_idx_list:
if _get_metadata_hash(fi, idx) != common_metadata_hash_list[idx]:
common_metadata_idx_list.remove(idx)
return split_list(irange(len(block[DataProvider.Metadata])),
fun=common_metadata_idx_list.__contains__,
sort_key=lambda idx: block[DataProvider.Metadata][idx])
def _sort_deps(psrc_list):
metadata_dict = {}
metadata_set_all = set()
for psrc in psrc_list:
tmp = []
psrc.fill_parameter_metadata(tmp)
metadata_dict[psrc] = lmap(lambda meta: meta.value, tmp)
metadata_set_all.update(metadata_dict[psrc])
psrc_idx_list = lrange(len(psrc_list))
metadata_list_known = []
while psrc_idx_list:
psrc_idx = psrc_idx_list.pop(0)
psrc = psrc_list[psrc_idx]
metadata_list_known_to_psrc = metadata_list_known + metadata_dict[psrc]
for dep in psrc.get_parameter_deps():
if (dep in metadata_set_all) and (dep not in metadata_list_known_to_psrc):
# this psrc depends on a variable that is available not not yet seen
psrc_idx_list.append(psrc_idx) # back to the queue
break
if psrc_idx not in psrc_idx_list:
yield psrc
metadata_list_known = metadata_list_known_to_psrc
def _sort_deps(psrc_list):
metadata_dict = {}
metadata_set_all = set()
for psrc in psrc_list:
tmp = []
psrc.fill_parameter_metadata(tmp)
metadata_dict[psrc] = lmap(lambda meta: meta.value, tmp)
metadata_set_all.update(metadata_dict[psrc])
psrc_idx_list = lrange(len(psrc_list))
metadata_list_known = []
while psrc_idx_list:
psrc_idx = psrc_idx_list.pop(0)
psrc = psrc_list[psrc_idx]
metadata_list_known_to_psrc = metadata_list_known + metadata_dict[psrc]
for dep in psrc.get_parameter_deps():
if (dep in metadata_set_all) and (
dep not in metadata_list_known_to_psrc):
# this psrc depends on a variable that is available not not yet seen
psrc_idx_list.append(psrc_idx) # back to the queue
break
if psrc_idx not in psrc_idx_list:
yield psrc
metadata_list_known = metadata_list_known_to_psrc
except ImportError:
c_make_encoder = None
ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
HAS_UTF8 = re.compile(r'[\x80-\xff]')
ESCAPE_DCT = {
'\\': '\\\\',
'"': '\\"',
'\b': '\\b',
'\f': '\\f',
'\n': '\\n',
'\r': '\\r',
'\t': '\\t',
}
for i in lrange(0x20):
#ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i, ))
INFINITY = float('inf')
FLOAT_REPR = repr
def encode_basestring(s):
"""Return a JSON representation of a Python string
"""
def replace(match):
return ESCAPE_DCT[match.group(0)]
return '"' + ESCAPE.sub(replace, s) + '"'