def ingest(self, core, data=None): ''' Ingest the data from this definition into the specified cortex. ''' scope = s_scope.Scope() if data != None: root = s_datapath.initelem(data) gest = self._i_info.get('ingest') self._ingDataInfo(core, root, gest, scope) return for path, info in self.get('sources'): scope.enter() scope.add('tags', *info.get('tags', ())) gest = info.get('ingest') if gest == None: gest = self._i_info.get('ingest') if gest == None: raise Exception('Ingest Info Not Found: %s' % (path, )) for datasorc in self._iterDataSorc(path, info): for data in datasorc: root = s_datapath.initelem(data) self._ingDataInfo(core, root, gest, scope)
def test_datapath_iter(self): data = s_datapath.initelem(item0) vals = tuple(data.vals('results/*/foo')) self.eq(vals, (10, 20)) dp = s_datapath.DataPath('results/*/foo') vals = tuple(data.vals(dp)) self.eq(vals, (10, 20))
def _normalize_records(self, raw_records: Iterable[Tuple[int, Dict[int, str]]]) -> \ Iterable[Tuple[int, int, Union[str, int]]]: ''' Yield stream of normalized fields Args: raw_records: generator of tuples of offset/decoded raw cryotank records Returns: generator of tuples of offset, index ID, normalized property value ''' for offset, record in raw_records: self._next_offset = offset + 1 dp = s_datapath.initelem(s_msgpack.un(record)) for iid, idx in ((k, v) for k, v in self._meta.indices.items() if not self._meta.asleep[k]): if self._meta.progresses[iid]['nextoffset'] > offset: continue try: self._meta.progresses[iid]['nextoffset'] = offset + 1 for datapath in idx.datapaths: field = dp.valu(datapath) if field is None: continue # TODO : what to do with subprops? break else: # logger.debug('Datapaths %s yield nothing for offset %d', # [d.path for d in idx.datapaths], offset) continue normval, _ = idx.syntype.norm(field) except (s_exc.NoSuchType, s_exc.BadTypeValu, ValueError): # logger.debug('Norm fail', exc_info=True) self._meta.progresses[iid]['nnormfail'] += 1 continue self._meta.progresses[iid]['ngood'] += 1 yield offset, iid, normval
def test_datapath_self(self): data = s_datapath.initelem(item0) self.eq(data.valu('results/0/././foo'), 10)
def test_datapath_quoted(self): data = s_datapath.initelem(item0) self.eq(data.valu('".."'), 'hurr') self.eq(data.valu('"20"'), 'durr')
def test_datapath_parent(self): data = s_datapath.initelem(item0) self.eq(data.valu('results/../woot'), 'hehe')
def test_datapath_valu(self): data = s_datapath.initelem(item0) self.eq(data.valu('results/0/foo'), 10) self.eq(data.valu('results/1/foo'), 20) dp = s_datapath.DataPath('results/1/foo') self.eq(data.valu(dp), 20)