def ingest(self, core, data=None):
'''
Ingest the data from this definition into the specified cortex.
'''
scope = s_scope.Scope()
if data != None:
root = s_datapath.initelem(data)
gest = self._i_info.get('ingest')
self._ingDataInfo(core, root, gest, scope)
return
for path, info in self.get('sources'):
scope.enter()
scope.add('tags', *info.get('tags', ()))
gest = info.get('ingest')
if gest == None:
gest = self._i_info.get('ingest')
if gest == None:
raise Exception('Ingest Info Not Found: %s' % (path, ))
for datasorc in self._iterDataSorc(path, info):
for data in datasorc:
root = s_datapath.initelem(data)
self._ingDataInfo(core, root, gest, scope)
def test_datapath_iter(self):
data = s_datapath.initelem(item0)
vals = tuple(data.vals('results/*/foo'))
self.eq(vals, (10, 20))
dp = s_datapath.DataPath('results/*/foo')
vals = tuple(data.vals(dp))
self.eq(vals, (10, 20))
def _normalize_records(self, raw_records: Iterable[Tuple[int, Dict[int, str]]]) -> \
Iterable[Tuple[int, int, Union[str, int]]]:
'''
Yield stream of normalized fields
Args:
raw_records: generator of tuples of offset/decoded raw cryotank records
Returns:
generator of tuples of offset, index ID, normalized property value
'''
for offset, record in raw_records:
self._next_offset = offset + 1
dp = s_datapath.initelem(s_msgpack.un(record))
for iid, idx in ((k, v) for k, v in self._meta.indices.items()
if not self._meta.asleep[k]):
if self._meta.progresses[iid]['nextoffset'] > offset:
continue
try:
self._meta.progresses[iid]['nextoffset'] = offset + 1
for datapath in idx.datapaths:
field = dp.valu(datapath)
if field is None:
continue
# TODO : what to do with subprops?
break
else:
# logger.debug('Datapaths %s yield nothing for offset %d',
# [d.path for d in idx.datapaths], offset)
continue
normval, _ = idx.syntype.norm(field)
except (s_exc.NoSuchType, s_exc.BadTypeValu, ValueError):
# logger.debug('Norm fail', exc_info=True)
self._meta.progresses[iid]['nnormfail'] += 1
continue
self._meta.progresses[iid]['ngood'] += 1
yield offset, iid, normval
def test_datapath_self(self):
data = s_datapath.initelem(item0)
self.eq(data.valu('results/0/././foo'), 10)
def test_datapath_quoted(self):
data = s_datapath.initelem(item0)
self.eq(data.valu('".."'), 'hurr')
self.eq(data.valu('"20"'), 'durr')
def test_datapath_parent(self):
data = s_datapath.initelem(item0)
self.eq(data.valu('results/../woot'), 'hehe')
def test_datapath_valu(self):
data = s_datapath.initelem(item0)
self.eq(data.valu('results/0/foo'), 10)
self.eq(data.valu('results/1/foo'), 20)
dp = s_datapath.DataPath('results/1/foo')
self.eq(data.valu(dp), 20)
