def test_msgpack_iterfile(self):
t0 = ('5678', {'key': 1})
t1 = ('1234', {'key': 'haha'})
with self.getTestDir() as fdir:
fd = s_common.genfile(fdir, 'test.mpk')
for obj in (t0, t1):
fd.write(s_msgpack.en(obj))
fd.close()
gen = s_msgpack.iterfile(s_common.genpath(fdir, 'test.mpk'))
items = [obj for obj in gen]
self.len(2, items)
self.sorteq(items, [t0, t1])
fd.close()
def test_msgpack_iterfile(self):
t0 = ('5678', {'key': 1})
t1 = ('1234', {'key': 'haha'})
with self.getTestDir() as fdir:
fd = s_common.genfile(fdir, 'test.mpk')
for obj in (t0, t1):
fd.write(s_msgpack.en(obj))
fd.close()
gen = s_msgpack.iterfile(s_common.genpath(fdir, 'test.mpk'))
items = [obj for obj in gen]
self.len(2, items)
self.sorteq(items, [t0, t1])
fd.close()
def getItems(*paths):
items = []
for path in paths:
if path.endswith('.json'):
item = s_common.jsload(path)
if not isinstance(item, list):
item = [item]
items.append((path, item))
elif path.endswith(('.yaml', '.yml')):
item = s_common.yamlload(path)
if not isinstance(item, list):
item = [item]
items.append((path, item))
elif path.endswith('.mpk'):
genr = s_msgpack.iterfile(path)
items.append((path, genr))
else: # pragma: no cover
logger.warning('Unsupported file path: [%s]', path)
return items
def getItems(*paths):
items = []
for path in paths:
if path.endswith('.json'):
item = s_common.jsload(path)
if not isinstance(item, list):
item = [item]
items.append((path, item))
elif path.endswith(('.yaml', '.yml')):
item = s_common.yamlload(path)
if not isinstance(item, list):
item = [item]
items.append((path, item))
elif path.endswith('.mpk'):
genr = s_msgpack.iterfile(path)
items.append((path, genr))
else: # pragma: no cover
logger.warning('Unsupported file path: [%s]', path)
return items
def getItems(*paths):
items = []
for path in paths:
if path.endswith('.json'):
item = s_common.jsload(path)
if not isinstance(item, list):
item = [item]
items.append((path, item))
elif path.endswith('.jsonl'):
with s_common.genfile(path) as fd:
item = list(s_encoding.iterdata(fd, False, format='jsonl'))
items.append((path, item))
elif path.endswith(('.yaml', '.yml')):
item = s_common.yamlload(path)
if not isinstance(item, list):
item = [item]
items.append((path, item))
elif path.endswith('.mpk') or path.endswith('.nodes'):
genr = s_msgpack.iterfile(path)
items.append((path, genr))
else: # pragma: no cover
logger.warning('Unsupported file path: [%s]', path)
return items
