def _iter_prop(self, core, data, info, scope):
cond = info.get('cond')
if cond is not None and not self._isCondTrue(cond, scope):
return
path = info.get('iter')
if path is None:
valu = self._get_prop(core, data, info, scope)
if valu is not None:
yield valu
return
for base in data.iter(s_datapath.DataPath(path)):
with scope:
self._ingMergScope(core, base, info, scope)
valu = self._get_prop(core, base, info, scope)
if valu is None:
continue
yield valu
def test_datapath_iter(self):
data = s_datapath.initelem(item0)
vals = tuple(data.vals('results/*/foo'))
self.eq(vals, (10, 20))
dp = s_datapath.DataPath('results/*/foo')
vals = tuple(data.vals(dp))
self.eq(vals, (10, 20))
def _ingDataInfo(self, core, data, info, scope):
with scope:
self._ingMergScope(core, data, info, scope)
cond = info.get('cond')
if cond is not None and not self._isCondTrue(cond, scope):
return
self.fire('gest:prog', act='data')
# extract files embedded within the data structure
for flfo in info.get('files', ()):
self._ingFileInfo(core, data, flfo, scope)
for cond, cnfo in info.get('conds', ()):
if not self._isCondTrue(cond, scope):
continue
self._ingDataInfo(core, data, cnfo, scope)
# iterate and create any forms at our level
for form, fnfo in info.get('forms', ()):
fnfo.setdefault('form', form)
self._ingFormInfo(core, data, fnfo, scope)
# handle explicit nested iterators
for path, tifo in info.get('iters', ()):
for base in data.iter(s_datapath.DataPath(path)):
self._ingDataInfo(core, base, tifo, scope)
def _ingFileInfo(self, core, data, info, scope):
with scope:
self._ingMergScope(core, data, info, scope)
cond = info.get('cond')
if cond is not None and not self._isCondTrue(cond, scope):
return
path = info.get('path')
byts = data.valu(s_datapath.DataPath(path))
dcod = info.get('decode')
if dcod is not None:
byts = s_encoding.decode(dcod, byts)
hset = s_hashset.HashSet()
hset.update(byts)
iden, props = hset.guid()
mime = info.get('mime')
if mime is not None:
props['mime'] = mime
tufo = core.formTufoByProp('file:bytes', iden, **props)
self.fire('gest:prog', act='file')
for tag in scope.iter('tags'):
core.addTufoTag(tufo, tag)
self.fire('gest:prog', act='tag')
def __init__(self, model: s_datamodel.Model, propname: str, syntype: str,
datapaths: Iterable[str]) -> None:
'''
Makes a MetaEntry
Args:
propname: The name of the key in the normalized dictionary
syntype: The synapse type name against which the data will be normalized
datapath (Iterable[str]) One or more datapath strings that will be used to find the field in a raw record
'''
self.propname = propname
self.syntype = model.type(syntype)
self.datapaths = tuple(s_datapath.DataPath(d) for d in datapaths)
def test_datapath_valu(self):
data = s_datapath.initelem(item0)
self.eq(data.valu('results/0/foo'), 10)
self.eq(data.valu('results/1/foo'), 20)
dp = s_datapath.DataPath('results/1/foo')
self.eq(data.valu(dp), 20)
def _get_prop(self, core, base, info, scope):
cond = info.get('cond')
if cond is not None and not self._isCondTrue(cond, scope):
return
valu = info.get('value')
if valu is not None:
return valu
if valu is None:
varn = info.get('var')
if varn is not None:
valu = scope.get(varn)
gvar = info.get('guid')
if gvar is not None:
valu = []
for varn in gvar:
valu.append((varn, scope.get(varn)))
return valu
template = info.get('template')
if template is not None:
valu = template
for tvar in self._getTmplVars(template):
tval = scope.get(tvar)
if tval is None:
return None
# FIXME optimize away the following format string
valu = valu.replace('{{%s}}' % tvar, str(tval))
if valu is None:
path = info.get('path')
valu = base.valu(s_datapath.DataPath(path))
if valu is None:
return None
# If we have a regex field, use it to extract valu from the
# first grouping
rexs = info.get('regex')
if rexs is not None:
rexo = self._re_compile(rexs)
match = rexo.search(valu)
if match is None:
return None
groups = match.groups()
if groups:
valu = groups[0]
# allow type based normalization here
cast = info.get('cast')
if cast is not None:
valu = core.getTypeCast(cast, valu)
if valu is None:
return None
# FIXME make a mechanism here for field translation based
# on an included translation table within the ingest def
pivot = info.get('pivot')
if pivot is not None:
pivf, pivt = pivot
pivo = core.getTufoByProp(pivf, valu)
if pivo is None:
return None
valu = pivo[1].get(pivt)
return valu