def test_named_props(self):
'''
Ensure that provided prefixes are used.
'''
r = s_tufo.props(self.t2, pref='namespace')
self.eq(r, {'sound': 'quack'})
r = s_tufo.props(self.t3, pref='animal')
self.eq(r, {'sound': 'quack', 'stype': 'duck'})
r = s_tufo.props(self.t3, pref='geo')
self.eq(r, {})
def test_default_props(self):
'''
Ensure that when no prefix is provided, the properties are taken from the form.
'''
r = s_tufo.props(self.t1)
self.eq(r, {})
r = s_tufo.props(self.t2)
self.eq(r, {})
r = s_tufo.props(self.t3)
self.eq(r, {'sound': 'quack', 'stype': 'duck'})
def analyze(core, fdir):
d = {}
for inode in core.eval('ibutton'):
_, pprop = s_tufo.ndef(inode)
log.info('Collecting data for {}'.format(pprop))
ibutton = button.ButtonData()
props = s_tufo.props(inode)
ibutton.metadata[TMNT] = props.get(TMNT)
ibutton.metadata[BLOCK] = props.get(BLOCK)
for node in core.eval('idata:button=%s' % pprop):
props = s_tufo.props(node)
tick = props.get('rtime')
# Taken from synapse.lib.time
dt = datetime.datetime(1970, 1,
1) + datetime.timedelta(milliseconds=tick)
# from dC (int) to C (float)
v = float(props.get('temp')) / 10
ibutton.append((dt, v))
d[pprop] = ibutton
summer_rows = []
for pprop, ibutton in d.items():
log.info('Analyzing data for {}'.format(pprop))
ibutton.analyze_button()
ibutton.write_computed_data_to_files(fdir)
for k, v in ibutton.summer_gdd.items():
d = {
'year': k,
'gdd': v,
TMNT: ibutton.metadata.get(TMNT),
BLOCK: ibutton.metadata.get(BLOCK)
}
summer_rows.append(d)
fn = 'aggregate_summer_gdd.csv'
fp = os.path.join(fdir, fn)
with open(fp, 'w') as f:
w = csv.DictWriter(f, [TMNT, BLOCK, 'year', 'gdd'])
w.writeheader()
for d in summer_rows:
w.writerow(d)
total_summer_gdd = sum([d.get('gdd') for d in summer_rows])
log.info('Total summer gdd rows: [%s] [%s]', total_summer_gdd,
(total_summer_gdd / len(summer_rows)))
def _revModl201709191412(self):
'''
Migrate the XREF types to use the propvalu syntax.
'''
tick = s_common.now()
adds = []
dels = set()
nforms = set()
for form in self.core.getModelDict().get('forms'):
sforms = self.core.getTypeOfs(form)
if 'xref' in sforms:
nforms.add(form)
for ntyp in nforms:
nodes = self.core.getTufosByProp(ntyp)
xtyp = '{}:xtype'.format(ntyp)
xrefp = '{}:xref'.format(ntyp)
xrefpint = '{}:xref:intval'.format(ntyp)
xrefpstr = '{}:xref:strval'.format(ntyp)
xrefprop = '{}:xref:prop'.format(ntyp)
for node in nodes:
iden = node[0]
srcvtype = node[1].get(xtyp)
if srcvtype is None:
# This is expensive node level introspection :(
for prop, valu in s_tufo.props(node).items():
if prop.startswith('xref:'):
form = prop.split('xref:', 1)[1]
if self.core.isTufoForm(form):
srcvtype = form
break
if not srcvtype:
raise s_common.NoSuchProp(
iden=node[0],
type=ntyp,
mesg=
'Unable to find a xref prop which is a form for migrating a '
'XREF node.')
srcprp = '{}:xref:{}'.format(ntyp, srcvtype)
srcv = node[1].get(srcprp)
valu, subs = self.core.getPropNorm(xrefp, [srcvtype, srcv])
adds.append((iden, xrefp, valu, tick))
adds.append((iden, xrefprop, srcvtype, tick))
if 'intval' in subs:
adds.append((iden, xrefpint, subs.get('intval'), tick))
else:
adds.append((iden, xrefpstr, subs.get('strval'), tick))
dels.add(srcprp)
dels.add(xtyp)
with self.core.getCoreXact():
self.core.addRows(adds)
for prop in dels:
self.core.delRowsByProp(prop)
def test_model_ibutton(self):
td = {'serial': '5E00000038013B21', 'block': '3', 'tmnt': 'Control'}
ed = {'serial': '5e00000038013b21', 'block': 3, 'tmnt': 'control'}
eprop = 'a14c37660532af1e02e91a2c0cda75f0'
with self.getRamCore() as core: # s_cortex_common.Cortex
node = core.formTufoByProp('ibutton', td)
_, pprop = s_tufo.ndef(node)
props = s_tufo.props(node)
self.eq(pprop, eprop)
self.eq(props, ed)
with self.getRamCore() as core: # s_cortex_common.Cortex
new_td = td.copy()
new_td['serial'] = new_td.get('serial').lower()
node = core.formTufoByProp('ibutton', new_td)
_, pprop = s_tufo.ndef(node)
props = s_tufo.props(node)
self.eq(pprop, eprop)
self.eq(props, ed)
def _getSessByIden(self, iden):
# If we have no cortex, we have no session storage
if self.core is None:
return None
# look up the tufo and construct a Sess()
sefo = self.core.getTufoByProp('syn:sess', iden)
if sefo is None:
return None
props = s_tufo.props(sefo)
return Sess(self, iden, **props)
def test_model_orgs_oumember(self):
with self.getRamCore() as core:
pnode = core.formTufoByProp('ps:person', '*', name='grey, robert')
_, pprop = s_tufo.ndef(pnode)
onode = core.formTufoByProp('ou:org:name', 'derry sanitation corp')
_, oprop = s_tufo.ndef(onode)
mnode = core.formTufoByProp(
'ou:member', {
'org': oprop,
'person': pprop
}, **{
'start': '2017',
'end': '2018',
'title': 'Dancing Clown'
})
self.nn(mnode)
_, mpprop = s_tufo.ndef(mnode)
props = s_tufo.props(mnode)
self.eq(props.get('org'), oprop)
self.eq(props.get('person'), pprop)
self.eq(props.get('end'), core.getTypeNorm('time', '2018')[0])
self.eq(props.get('start'), core.getTypeNorm('time', '2017')[0])
self.eq(props.get('title'), 'dancing clown')
# We can traverse across the ou:member node
nodes = core.eval(
'ps:person=%s -> ou:member:person :org -> ou:org' % pprop)
self.len(1, nodes)
self.eq(oprop, nodes[0][1].get('ou:org'))
nodes = core.eval(
'ou:org=%s -> ou:member:org :person -> ps:person' % oprop)
self.len(1, nodes)
self.eq(pprop, nodes[0][1].get('ps:person'))
def _print_tufo_add(mesg):
tufo = mesg[1].get('node')
form = tufo[1].get('tufo:form')
outp.printf('add: %s=%s' % (form, tufo[1].get(form)))
for prop, valu in sorted(s_tufo.props(tufo).items()):
outp.printf(' :%s = %s' % (prop, valu))
def main(argv, outp=None):
if outp is None:
outp = s_output.OutPut()
pars = argparse.ArgumentParser(prog='autodoc', description=descr)
#pars.add_argument('--format', default='rst')
pars.add_argument('--cortex',
default='ram://',
help='Cortex URL for model inspection')
pars.add_argument(
'--doc-model',
action='store_true',
default=False,
help='Generate RST docs for the DataModel within a cortex')
pars.add_argument(
'--configable-opts',
action='store_true',
default=False,
help='Generate RST docs of the Configable classes in Synapse.')
pars.add_argument('--savefile',
default=None,
help='Save output to the given file')
opts = pars.parse_args(argv)
fd = None
if opts.savefile:
fd = open(opts.savefile, 'wb')
outp = s_output.OutPutFd(fd)
core = s_cortex.openurl(opts.cortex)
if opts.doc_model:
forms = []
types = []
props = collections.defaultdict(list)
for tufo in core.getTufosByProp('syn:type'):
name = tufo[1].get('syn:type')
info = s_tufo.props(tufo)
types.append((name, info))
for tufo in core.getTufosByProp('syn:form'):
name = tufo[1].get('syn:form')
info = s_tufo.props(tufo)
forms.append((name, info))
for tufo in core.getTufosByProp('syn:prop'):
prop = tufo[1].get('syn:prop')
form = tufo[1].get('syn:prop:form')
info = s_tufo.props(tufo)
props[form].append((prop, info))
types.sort()
forms.sort()
[v.sort() for v in props.values()]
rst = RstHelp()
rst.addHead('Synapse Data Model', lvl=0)
rst.addHead('Types', lvl=1)
for name, info in types:
rst.addHead(name, lvl=2)
inst = core.getTypeInst(name)
ex = inst.get('ex')
doc = inst.get('doc')
if doc is not None:
rst.addLines(doc)
bases = core.getTypeBases(name)
rst.addLines('', 'Type Hierarchy: %s' % (' -> '.join(bases), ), '')
if ex is not None:
#valu = core.getTypeParse(name,ex)
#vrep = reprvalu(valu)
rst.addLines('', 'Examples:', '')
rst.addLines('- repr mode: %s' % (repr(ex), ))
#rst.addLines('- system mode: %s' % (vrep,))
rst.addLines('')
cons = []
xforms = []
if core.isSubType(name, 'str'):
regex = inst.get('regex')
if regex is not None:
cons.append('- regex: %s' % (regex, ))
lower = inst.get('lower')
if lower:
xforms.append('- case: lower')
restrip = inst.get('restrip')
if restrip is not None:
xforms.append('- regex strip: %s' % (restrip, ))
nullval = inst.get('nullval')
if nullval is not None:
cons.append('- null value: %s' % (nullval, ))
if core.isSubType(name, 'int'):
minval = inst.get('min')
if minval is not None:
cons.append('- min value: %d (0x%x)' % (minval, minval))
maxval = inst.get('max')
if maxval is not None:
cons.append('- max value: %d (0x%x)' % (maxval, maxval))
ismin = inst.get('ismin')
if ismin is not None:
xforms.append('- is minimum: True')
ismax = inst.get('ismax')
if ismax is not None:
xforms.append('- is maximum: True')
if core.isSubType(name, 'sepr'):
sep = inst.get('sep')
fields = inst.get('fields')
parts = []
for part in fields.split('|'):
name, stype = part.split(',')
parts.append(stype)
seprs = sep.join(['<%s>' % p for p in parts])
rst.addLines('', 'Sepr Fields: %s' % (seprs, ))
if cons:
cons.append('')
rst.addLines('', 'Type Constraints:', '', *cons)
if xforms:
xforms.append('')
rst.addLines('', 'Type Transforms:', '', *xforms)
rst.addHead('Forms', lvl=1)
for name, info in forms:
ftype = info.get('ptype', 'str')
rst.addHead('%s = <%s>' % (name, ftype), lvl=2)
doc = core.getPropInfo(name, 'doc')
if doc is not None:
rst.addLines(doc)
rst.addLines('', 'Properties:', '')
for prop, pnfo in props.get(name, ()):
# use the resolver funcs that will recurse upward
pex = core.getPropInfo(prop, 'ex')
pdoc = core.getPropInfo(prop, 'doc')
ptype = pnfo.get('ptype')
pline = '\t- %s = <%s>' % (prop, ptype)
defval = pnfo.get('defval')
if defval is not None:
pline += ' (default: %r)' % (defval, )
rst.addLines(pline)
if pdoc:
rst.addLines('\t\t- %s' % (pdoc, ))
outp.printf(rst.getRstText())
return 0
if opts.configable_opts:
rst = RstHelp()
rst.addHead('Synapse Configable Classes', lvl=0)
rst.addLines(
'The following objects are Configable objects. They have'
' settings which may be provided at runtime or during object'
' initialization which may change their behavior.')
basename = 'synapse'
confdetails = collections.defaultdict(list)
for root, dirs, files in os.walk(base_synaspe_dir):
if any([v for v in dir_skips if v in root]):
continue
for fn in files:
if fn in fn_skips:
continue
if not fn.endswith('.py'):
continue
modname = fn.rsplit('.', 1)[0]
_modpath = root[len(base_synaspe_dir) + 1:].replace(
os.sep, '.')
modpath = '.'.join(
[v for v in [basename, _modpath, modname] if v])
mod = importlib.import_module(modpath)
for modattr, valu, name, results in inspect_mod(
mod, cls=s_config.Configable):
confdetails[modpath].append((modattr, name, results))
# Collapse details into a modpath -> Details struct
detaildict = collections.defaultdict(list)
for modpath, details in confdetails.items():
for detail in details:
modattr, name, results = detail
obj_path = '.'.join([modpath, modattr])
if obj_path in obj_path_skips:
continue
for rslt in results:
if not rslt:
continue
detaildict[obj_path].append(rslt)
# Now make the RST proper like
keys = list(detaildict.keys())
keys.sort()
for obj_path in keys:
details = detaildict.get(obj_path, [])
details.sort(key=lambda x: x[0])
rst.addHead(name=obj_path, lvl=1)
for detail in details:
confvalu, confdict = detail[0], detail[1]
rst.addHead(confvalu, lvl=2)
_keys = list(confdict.keys())
_keys.sort()
for _key in _keys:
v = confdict.get(_key)
line = ' - {}: {}'.format(_key, v)
rst.addLines(line)
outp.printf(rst.getRstText())
if fd:
fd.close()
return 0
def main(argv, outp=None):
if outp == None:
outp = s_output.OutPut()
pars = argparse.ArgumentParser(prog='autodoc', description=descr)
#pars.add_argument('--format', default='rst')
pars.add_argument('--cortex',
default='ram://',
help='Cortex URL for model inspection')
pars.add_argument(
'--doc-model',
action='store_true',
default=False,
help='Generate RST docs for the DataModel within a cortex')
pars.add_argument('--savefile',
default=None,
help='Save output to the given file')
opts = pars.parse_args(argv)
if opts.savefile:
fd = open(opts.savefile, 'wb')
outp = s_output.OutPutFd(fd)
core = s_cortex.openurl(opts.cortex)
if opts.doc_model:
forms = []
types = []
props = collections.defaultdict(list)
for tufo in core.getTufosByProp('syn:type'):
name = tufo[1].get('syn:type')
info = s_tufo.props(tufo)
types.append((name, info))
for tufo in core.getTufosByProp('syn:form'):
name = tufo[1].get('syn:form')
info = s_tufo.props(tufo)
forms.append((name, info))
for tufo in core.getTufosByProp('syn:prop'):
prop = tufo[1].get('syn:prop')
form = tufo[1].get('syn:prop:form')
info = s_tufo.props(tufo)
props[form].append((prop, info))
types.sort()
forms.sort()
[v.sort() for v in props.values()]
rst = RstHelp()
rst.addHead('Synapse Data Model', lvl=0)
rst.addHead('Types', lvl=1)
for name, info in types:
rst.addHead(name, lvl=2)
inst = core.getTypeInst(name)
ex = inst.get('ex')
doc = inst.get('doc')
if doc != None:
rst.addLines(doc)
bases = core.getTypeBases(name)
rst.addLines('', 'Type Hierarchy: %s' % (' -> '.join(bases), ), '')
if ex != None:
#valu = core.getTypeParse(name,ex)
#vrep = reprvalu(valu)
rst.addLines('', 'Examples:', '')
rst.addLines('- repr mode: %s' % (repr(ex), ))
#rst.addLines('- system mode: %s' % (vrep,))
rst.addLines('')
cons = []
xforms = []
if core.isSubType(name, 'str'):
regex = inst.get('regex')
if regex != None:
cons.append('- regex: %s' % (regex, ))
lower = inst.get('lower')
if lower:
xforms.append('- case: lower')
restrip = inst.get('restrip')
if restrip != None:
xforms.append('- regex strip: %s' % (restrip, ))
nullval = inst.get('nullval')
if nullval != None:
cons.append('- null value: %s' % (nullval, ))
if core.isSubType(name, 'int'):
minval = inst.get('min')
if minval != None:
cons.append('- min value: %d (0x%x)' % (minval, minval))
maxval = inst.get('max')
if maxval != None:
cons.append('- max value: %d (0x%x)' % (maxval, maxval))
ismin = inst.get('ismin')
if ismin != None:
xforms.append('- is minimum: True')
ismax = inst.get('ismax')
if ismax != None:
xforms.append('- is maximum: True')
if core.isSubType(name, 'sepr'):
sep = inst.get('sep')
fields = inst.get('fields')
parts = []
for part in fields.split('|'):
name, stype = part.split(',')
parts.append(stype)
seprs = sep.join(['<%s>' % p for p in parts])
rst.addLines('', 'Sepr Fields: %s' % (seprs, ))
if cons:
cons.append('')
rst.addLines('', 'Type Constraints:', '', *cons)
if xforms:
xforms.append('')
rst.addLines('', 'Type Transforms:', '', *xforms)
rst.addHead('Forms', lvl=1)
for name, info in forms:
ftype = info.get('ptype', 'str')
rst.addHead('%s = <%s>' % (name, ftype), lvl=2)
doc = core.getPropInfo(name, 'doc')
if doc != None:
rst.addLines(doc)
rst.addLines('', 'Properties:', '')
for prop, pnfo in props.get(name, ()):
# use the resolver funcs that will recurse upward
pex = core.getPropInfo(prop, 'ex')
pdoc = core.getPropInfo(prop, 'doc')
ptype = pnfo.get('ptype')
pline = '\t- %s = <%s>' % (prop, ptype)
defval = pnfo.get('defval')
if defval != None:
pline += ' (default: %r)' % (defval, )
rst.addLines(pline)
if pdoc:
rst.addLines('\t\t- %s' % (pdoc, ))
outp.printf(rst.getRstText())
return 0
def docModel(outp, fd, core):
forms = []
types = []
props = collections.defaultdict(list)
for tufo in core.getTufosByProp('syn:type'):
name = tufo[1].get('syn:type')
info = s_tufo.props(tufo)
types.append((name, info))
for tufo in core.getTufosByProp('syn:form'):
name = tufo[1].get('syn:form')
info = s_tufo.props(tufo)
forms.append((name, info))
for tufo in core.getTufosByProp('syn:prop'):
prop = tufo[1].get('syn:prop')
form = tufo[1].get('syn:prop:form')
info = s_tufo.props(tufo)
props[form].append((prop, info))
types.sort()
forms.sort()
[v.sort() for v in props.values()]
rst = RstHelp()
rst.addHead('Synapse Data Model', lvl=0)
rst.addHead('Types', lvl=1)
for name, info in types:
rst.addHead(name, lvl=2)
inst = core.getTypeInst(name)
ex = inst.get('ex')
doc = inst.get('doc')
if doc is not None:
rst.addLines(doc)
bases = core.getTypeBases(name)
rst.addLines('', 'Type Hierarchy: %s' % (' -> '.join(bases),), '')
if ex is not None:
#valu = core.getTypeParse(name,ex)
#vrep = reprvalu(valu)
rst.addLines('', 'Examples:', '')
rst.addLines('- repr mode: %s' % (repr(ex),))
#rst.addLines('- system mode: %s' % (vrep,))
rst.addLines('')
cons = []
xforms = []
if core.isSubType(name, 'str'):
regex = inst.get('regex')
if regex is not None:
cons.append('- regex: ``%s``' % (regex,))
lower = inst.get('lower')
if lower:
xforms.append('- case: lower')
restrip = inst.get('restrip')
if restrip is not None:
xforms.append('- regex strip: %s' % (restrip,))
nullval = inst.get('nullval')
if nullval is not None:
cons.append('- null value: %s' % (nullval,))
if core.isSubType(name, 'int'):
minval = inst.get('min')
if minval is not None:
cons.append('- min value: %d (0x%x)' % (minval, minval))
maxval = inst.get('max')
if maxval is not None:
cons.append('- max value: %d (0x%x)' % (maxval, maxval))
ismin = inst.get('ismin')
if ismin is not None:
xforms.append('- is minimum: True')
ismax = inst.get('ismax')
if ismax is not None:
xforms.append('- is maximum: True')
if core.isSubType(name, 'sepr'):
sep = inst.get('sep')
fields = inst.get('fields')
parts = []
for part in fields.split('|'):
name, stype = part.split(',')
parts.append(stype)
seprs = sep.join(['<%s>' % p for p in parts])
rst.addLines('', 'Sepr Fields: %s' % (seprs,))
if cons:
cons.append('')
rst.addLines('', 'Type Constraints:', '', *cons)
if xforms:
xforms.append('')
rst.addLines('', 'Type Transforms:', '', *xforms)
rst.addHead('Forms', lvl=1)
for name, info in forms:
ftype = info.get('ptype', 'str')
rst.addHead('%s = <%s>' % (name, ftype), lvl=2)
doc = core.getPropInfo(name, 'doc')
if doc is not None:
rst.addLines(doc)
rst.addLines('', 'Properties:', '')
for prop, pnfo in props.get(name, ()):
# use the resolver funcs that will recurse upward
pex = core.getPropInfo(prop, 'ex')
pdoc = core.getPropInfo(prop, 'doc')
ptype = pnfo.get('ptype')
pline = '\t- %s = <%s>' % (prop, ptype)
defval = pnfo.get('defval')
if defval is not None:
pline += ' (default: %r)' % (defval,)
rst.addLines(pline)
if pdoc:
rst.addLines('\t\t- %s' % (pdoc,))
# Add universal props last - they are not associated with a form.
uniprops = props.get(None)
if uniprops:
rst.addHead('Universal Props', lvl=1)
rst.addLines('', 'Universal props are system level properties which are generally present on every node.',
'', 'These properties are not specific to a particular form and exist outside of a particular'
' namespace.', '')
plist = sorted(uniprops, key=lambda x: x[0])
for prop, pnfo in plist:
rst.addHead(prop, lvl=2)
pdoc = core.getPropInfo(prop, 'doc')
ptype = pnfo.get('ptype')
pline = '\t- %s = <%s>' % (prop, ptype)
rst.addLines(pline)
if pdoc:
rst.addLines('\t\t- %s' % (pdoc,))
outp.printf(rst.getRstText())
if fd:
fd.close()
return 0