def test_named_props(self): ''' Ensure that provided prefixes are used. ''' r = s_tufo.props(self.t2, pref='namespace') self.eq(r, {'sound': 'quack'}) r = s_tufo.props(self.t3, pref='animal') self.eq(r, {'sound': 'quack', 'stype': 'duck'}) r = s_tufo.props(self.t3, pref='geo') self.eq(r, {})
def test_default_props(self): ''' Ensure that when no prefix is provided, the properties are taken from the form. ''' r = s_tufo.props(self.t1) self.eq(r, {}) r = s_tufo.props(self.t2) self.eq(r, {}) r = s_tufo.props(self.t3) self.eq(r, {'sound': 'quack', 'stype': 'duck'})
def analyze(core, fdir): d = {} for inode in core.eval('ibutton'): _, pprop = s_tufo.ndef(inode) log.info('Collecting data for {}'.format(pprop)) ibutton = button.ButtonData() props = s_tufo.props(inode) ibutton.metadata[TMNT] = props.get(TMNT) ibutton.metadata[BLOCK] = props.get(BLOCK) for node in core.eval('idata:button=%s' % pprop): props = s_tufo.props(node) tick = props.get('rtime') # Taken from synapse.lib.time dt = datetime.datetime(1970, 1, 1) + datetime.timedelta(milliseconds=tick) # from dC (int) to C (float) v = float(props.get('temp')) / 10 ibutton.append((dt, v)) d[pprop] = ibutton summer_rows = [] for pprop, ibutton in d.items(): log.info('Analyzing data for {}'.format(pprop)) ibutton.analyze_button() ibutton.write_computed_data_to_files(fdir) for k, v in ibutton.summer_gdd.items(): d = { 'year': k, 'gdd': v, TMNT: ibutton.metadata.get(TMNT), BLOCK: ibutton.metadata.get(BLOCK) } summer_rows.append(d) fn = 'aggregate_summer_gdd.csv' fp = os.path.join(fdir, fn) with open(fp, 'w') as f: w = csv.DictWriter(f, [TMNT, BLOCK, 'year', 'gdd']) w.writeheader() for d in summer_rows: w.writerow(d) total_summer_gdd = sum([d.get('gdd') for d in summer_rows]) log.info('Total summer gdd rows: [%s] [%s]', total_summer_gdd, (total_summer_gdd / len(summer_rows)))
def _revModl201709191412(self): ''' Migrate the XREF types to use the propvalu syntax. ''' tick = s_common.now() adds = [] dels = set() nforms = set() for form in self.core.getModelDict().get('forms'): sforms = self.core.getTypeOfs(form) if 'xref' in sforms: nforms.add(form) for ntyp in nforms: nodes = self.core.getTufosByProp(ntyp) xtyp = '{}:xtype'.format(ntyp) xrefp = '{}:xref'.format(ntyp) xrefpint = '{}:xref:intval'.format(ntyp) xrefpstr = '{}:xref:strval'.format(ntyp) xrefprop = '{}:xref:prop'.format(ntyp) for node in nodes: iden = node[0] srcvtype = node[1].get(xtyp) if srcvtype is None: # This is expensive node level introspection :( for prop, valu in s_tufo.props(node).items(): if prop.startswith('xref:'): form = prop.split('xref:', 1)[1] if self.core.isTufoForm(form): srcvtype = form break if not srcvtype: raise s_common.NoSuchProp( iden=node[0], type=ntyp, mesg= 'Unable to find a xref prop which is a form for migrating a ' 'XREF node.') srcprp = '{}:xref:{}'.format(ntyp, srcvtype) srcv = node[1].get(srcprp) valu, subs = self.core.getPropNorm(xrefp, [srcvtype, srcv]) adds.append((iden, xrefp, valu, tick)) adds.append((iden, xrefprop, srcvtype, tick)) if 'intval' in subs: adds.append((iden, xrefpint, subs.get('intval'), tick)) else: adds.append((iden, xrefpstr, subs.get('strval'), tick)) dels.add(srcprp) dels.add(xtyp) with self.core.getCoreXact(): self.core.addRows(adds) for prop in dels: self.core.delRowsByProp(prop)
def test_model_ibutton(self): td = {'serial': '5E00000038013B21', 'block': '3', 'tmnt': 'Control'} ed = {'serial': '5e00000038013b21', 'block': 3, 'tmnt': 'control'} eprop = 'a14c37660532af1e02e91a2c0cda75f0' with self.getRamCore() as core: # s_cortex_common.Cortex node = core.formTufoByProp('ibutton', td) _, pprop = s_tufo.ndef(node) props = s_tufo.props(node) self.eq(pprop, eprop) self.eq(props, ed) with self.getRamCore() as core: # s_cortex_common.Cortex new_td = td.copy() new_td['serial'] = new_td.get('serial').lower() node = core.formTufoByProp('ibutton', new_td) _, pprop = s_tufo.ndef(node) props = s_tufo.props(node) self.eq(pprop, eprop) self.eq(props, ed)
def _getSessByIden(self, iden): # If we have no cortex, we have no session storage if self.core is None: return None # look up the tufo and construct a Sess() sefo = self.core.getTufoByProp('syn:sess', iden) if sefo is None: return None props = s_tufo.props(sefo) return Sess(self, iden, **props)
def test_model_orgs_oumember(self): with self.getRamCore() as core: pnode = core.formTufoByProp('ps:person', '*', name='grey, robert') _, pprop = s_tufo.ndef(pnode) onode = core.formTufoByProp('ou:org:name', 'derry sanitation corp') _, oprop = s_tufo.ndef(onode) mnode = core.formTufoByProp( 'ou:member', { 'org': oprop, 'person': pprop }, **{ 'start': '2017', 'end': '2018', 'title': 'Dancing Clown' }) self.nn(mnode) _, mpprop = s_tufo.ndef(mnode) props = s_tufo.props(mnode) self.eq(props.get('org'), oprop) self.eq(props.get('person'), pprop) self.eq(props.get('end'), core.getTypeNorm('time', '2018')[0]) self.eq(props.get('start'), core.getTypeNorm('time', '2017')[0]) self.eq(props.get('title'), 'dancing clown') # We can traverse across the ou:member node nodes = core.eval( 'ps:person=%s -> ou:member:person :org -> ou:org' % pprop) self.len(1, nodes) self.eq(oprop, nodes[0][1].get('ou:org')) nodes = core.eval( 'ou:org=%s -> ou:member:org :person -> ps:person' % oprop) self.len(1, nodes) self.eq(pprop, nodes[0][1].get('ps:person'))
def _print_tufo_add(mesg): tufo = mesg[1].get('node') form = tufo[1].get('tufo:form') outp.printf('add: %s=%s' % (form, tufo[1].get(form))) for prop, valu in sorted(s_tufo.props(tufo).items()): outp.printf(' :%s = %s' % (prop, valu))
def main(argv, outp=None): if outp is None: outp = s_output.OutPut() pars = argparse.ArgumentParser(prog='autodoc', description=descr) #pars.add_argument('--format', default='rst') pars.add_argument('--cortex', default='ram://', help='Cortex URL for model inspection') pars.add_argument( '--doc-model', action='store_true', default=False, help='Generate RST docs for the DataModel within a cortex') pars.add_argument( '--configable-opts', action='store_true', default=False, help='Generate RST docs of the Configable classes in Synapse.') pars.add_argument('--savefile', default=None, help='Save output to the given file') opts = pars.parse_args(argv) fd = None if opts.savefile: fd = open(opts.savefile, 'wb') outp = s_output.OutPutFd(fd) core = s_cortex.openurl(opts.cortex) if opts.doc_model: forms = [] types = [] props = collections.defaultdict(list) for tufo in core.getTufosByProp('syn:type'): name = tufo[1].get('syn:type') info = s_tufo.props(tufo) types.append((name, info)) for tufo in core.getTufosByProp('syn:form'): name = tufo[1].get('syn:form') info = s_tufo.props(tufo) forms.append((name, info)) for tufo in core.getTufosByProp('syn:prop'): prop = tufo[1].get('syn:prop') form = tufo[1].get('syn:prop:form') info = s_tufo.props(tufo) props[form].append((prop, info)) types.sort() forms.sort() [v.sort() for v in props.values()] rst = RstHelp() rst.addHead('Synapse Data Model', lvl=0) rst.addHead('Types', lvl=1) for name, info in types: rst.addHead(name, lvl=2) inst = core.getTypeInst(name) ex = inst.get('ex') doc = inst.get('doc') if doc is not None: rst.addLines(doc) bases = core.getTypeBases(name) rst.addLines('', 'Type Hierarchy: %s' % (' -> '.join(bases), ), '') if ex is not None: #valu = core.getTypeParse(name,ex) #vrep = reprvalu(valu) rst.addLines('', 'Examples:', '') rst.addLines('- repr mode: %s' % (repr(ex), )) #rst.addLines('- system mode: %s' % (vrep,)) rst.addLines('') cons = [] xforms = [] if core.isSubType(name, 'str'): regex = inst.get('regex') if regex is not None: cons.append('- regex: %s' % (regex, )) lower = inst.get('lower') if lower: xforms.append('- case: lower') restrip = inst.get('restrip') if restrip is not None: xforms.append('- regex strip: %s' % (restrip, )) nullval = inst.get('nullval') if nullval is not None: cons.append('- null value: %s' % (nullval, )) if core.isSubType(name, 'int'): minval = inst.get('min') if minval is not None: cons.append('- min value: %d (0x%x)' % (minval, minval)) maxval = inst.get('max') if maxval is not None: cons.append('- max value: %d (0x%x)' % (maxval, maxval)) ismin = inst.get('ismin') if ismin is not None: xforms.append('- is minimum: True') ismax = inst.get('ismax') if ismax is not None: xforms.append('- is maximum: True') if core.isSubType(name, 'sepr'): sep = inst.get('sep') fields = inst.get('fields') parts = [] for part in fields.split('|'): name, stype = part.split(',') parts.append(stype) seprs = sep.join(['<%s>' % p for p in parts]) rst.addLines('', 'Sepr Fields: %s' % (seprs, )) if cons: cons.append('') rst.addLines('', 'Type Constraints:', '', *cons) if xforms: xforms.append('') rst.addLines('', 'Type Transforms:', '', *xforms) rst.addHead('Forms', lvl=1) for name, info in forms: ftype = info.get('ptype', 'str') rst.addHead('%s = <%s>' % (name, ftype), lvl=2) doc = core.getPropInfo(name, 'doc') if doc is not None: rst.addLines(doc) rst.addLines('', 'Properties:', '') for prop, pnfo in props.get(name, ()): # use the resolver funcs that will recurse upward pex = core.getPropInfo(prop, 'ex') pdoc = core.getPropInfo(prop, 'doc') ptype = pnfo.get('ptype') pline = '\t- %s = <%s>' % (prop, ptype) defval = pnfo.get('defval') if defval is not None: pline += ' (default: %r)' % (defval, ) rst.addLines(pline) if pdoc: rst.addLines('\t\t- %s' % (pdoc, )) outp.printf(rst.getRstText()) return 0 if opts.configable_opts: rst = RstHelp() rst.addHead('Synapse Configable Classes', lvl=0) rst.addLines( 'The following objects are Configable objects. They have' ' settings which may be provided at runtime or during object' ' initialization which may change their behavior.') basename = 'synapse' confdetails = collections.defaultdict(list) for root, dirs, files in os.walk(base_synaspe_dir): if any([v for v in dir_skips if v in root]): continue for fn in files: if fn in fn_skips: continue if not fn.endswith('.py'): continue modname = fn.rsplit('.', 1)[0] _modpath = root[len(base_synaspe_dir) + 1:].replace( os.sep, '.') modpath = '.'.join( [v for v in [basename, _modpath, modname] if v]) mod = importlib.import_module(modpath) for modattr, valu, name, results in inspect_mod( mod, cls=s_config.Configable): confdetails[modpath].append((modattr, name, results)) # Collapse details into a modpath -> Details struct detaildict = collections.defaultdict(list) for modpath, details in confdetails.items(): for detail in details: modattr, name, results = detail obj_path = '.'.join([modpath, modattr]) if obj_path in obj_path_skips: continue for rslt in results: if not rslt: continue detaildict[obj_path].append(rslt) # Now make the RST proper like keys = list(detaildict.keys()) keys.sort() for obj_path in keys: details = detaildict.get(obj_path, []) details.sort(key=lambda x: x[0]) rst.addHead(name=obj_path, lvl=1) for detail in details: confvalu, confdict = detail[0], detail[1] rst.addHead(confvalu, lvl=2) _keys = list(confdict.keys()) _keys.sort() for _key in _keys: v = confdict.get(_key) line = ' - {}: {}'.format(_key, v) rst.addLines(line) outp.printf(rst.getRstText()) if fd: fd.close() return 0
def main(argv, outp=None): if outp == None: outp = s_output.OutPut() pars = argparse.ArgumentParser(prog='autodoc', description=descr) #pars.add_argument('--format', default='rst') pars.add_argument('--cortex', default='ram://', help='Cortex URL for model inspection') pars.add_argument( '--doc-model', action='store_true', default=False, help='Generate RST docs for the DataModel within a cortex') pars.add_argument('--savefile', default=None, help='Save output to the given file') opts = pars.parse_args(argv) if opts.savefile: fd = open(opts.savefile, 'wb') outp = s_output.OutPutFd(fd) core = s_cortex.openurl(opts.cortex) if opts.doc_model: forms = [] types = [] props = collections.defaultdict(list) for tufo in core.getTufosByProp('syn:type'): name = tufo[1].get('syn:type') info = s_tufo.props(tufo) types.append((name, info)) for tufo in core.getTufosByProp('syn:form'): name = tufo[1].get('syn:form') info = s_tufo.props(tufo) forms.append((name, info)) for tufo in core.getTufosByProp('syn:prop'): prop = tufo[1].get('syn:prop') form = tufo[1].get('syn:prop:form') info = s_tufo.props(tufo) props[form].append((prop, info)) types.sort() forms.sort() [v.sort() for v in props.values()] rst = RstHelp() rst.addHead('Synapse Data Model', lvl=0) rst.addHead('Types', lvl=1) for name, info in types: rst.addHead(name, lvl=2) inst = core.getTypeInst(name) ex = inst.get('ex') doc = inst.get('doc') if doc != None: rst.addLines(doc) bases = core.getTypeBases(name) rst.addLines('', 'Type Hierarchy: %s' % (' -> '.join(bases), ), '') if ex != None: #valu = core.getTypeParse(name,ex) #vrep = reprvalu(valu) rst.addLines('', 'Examples:', '') rst.addLines('- repr mode: %s' % (repr(ex), )) #rst.addLines('- system mode: %s' % (vrep,)) rst.addLines('') cons = [] xforms = [] if core.isSubType(name, 'str'): regex = inst.get('regex') if regex != None: cons.append('- regex: %s' % (regex, )) lower = inst.get('lower') if lower: xforms.append('- case: lower') restrip = inst.get('restrip') if restrip != None: xforms.append('- regex strip: %s' % (restrip, )) nullval = inst.get('nullval') if nullval != None: cons.append('- null value: %s' % (nullval, )) if core.isSubType(name, 'int'): minval = inst.get('min') if minval != None: cons.append('- min value: %d (0x%x)' % (minval, minval)) maxval = inst.get('max') if maxval != None: cons.append('- max value: %d (0x%x)' % (maxval, maxval)) ismin = inst.get('ismin') if ismin != None: xforms.append('- is minimum: True') ismax = inst.get('ismax') if ismax != None: xforms.append('- is maximum: True') if core.isSubType(name, 'sepr'): sep = inst.get('sep') fields = inst.get('fields') parts = [] for part in fields.split('|'): name, stype = part.split(',') parts.append(stype) seprs = sep.join(['<%s>' % p for p in parts]) rst.addLines('', 'Sepr Fields: %s' % (seprs, )) if cons: cons.append('') rst.addLines('', 'Type Constraints:', '', *cons) if xforms: xforms.append('') rst.addLines('', 'Type Transforms:', '', *xforms) rst.addHead('Forms', lvl=1) for name, info in forms: ftype = info.get('ptype', 'str') rst.addHead('%s = <%s>' % (name, ftype), lvl=2) doc = core.getPropInfo(name, 'doc') if doc != None: rst.addLines(doc) rst.addLines('', 'Properties:', '') for prop, pnfo in props.get(name, ()): # use the resolver funcs that will recurse upward pex = core.getPropInfo(prop, 'ex') pdoc = core.getPropInfo(prop, 'doc') ptype = pnfo.get('ptype') pline = '\t- %s = <%s>' % (prop, ptype) defval = pnfo.get('defval') if defval != None: pline += ' (default: %r)' % (defval, ) rst.addLines(pline) if pdoc: rst.addLines('\t\t- %s' % (pdoc, )) outp.printf(rst.getRstText()) return 0
def docModel(outp, fd, core): forms = [] types = [] props = collections.defaultdict(list) for tufo in core.getTufosByProp('syn:type'): name = tufo[1].get('syn:type') info = s_tufo.props(tufo) types.append((name, info)) for tufo in core.getTufosByProp('syn:form'): name = tufo[1].get('syn:form') info = s_tufo.props(tufo) forms.append((name, info)) for tufo in core.getTufosByProp('syn:prop'): prop = tufo[1].get('syn:prop') form = tufo[1].get('syn:prop:form') info = s_tufo.props(tufo) props[form].append((prop, info)) types.sort() forms.sort() [v.sort() for v in props.values()] rst = RstHelp() rst.addHead('Synapse Data Model', lvl=0) rst.addHead('Types', lvl=1) for name, info in types: rst.addHead(name, lvl=2) inst = core.getTypeInst(name) ex = inst.get('ex') doc = inst.get('doc') if doc is not None: rst.addLines(doc) bases = core.getTypeBases(name) rst.addLines('', 'Type Hierarchy: %s' % (' -> '.join(bases),), '') if ex is not None: #valu = core.getTypeParse(name,ex) #vrep = reprvalu(valu) rst.addLines('', 'Examples:', '') rst.addLines('- repr mode: %s' % (repr(ex),)) #rst.addLines('- system mode: %s' % (vrep,)) rst.addLines('') cons = [] xforms = [] if core.isSubType(name, 'str'): regex = inst.get('regex') if regex is not None: cons.append('- regex: ``%s``' % (regex,)) lower = inst.get('lower') if lower: xforms.append('- case: lower') restrip = inst.get('restrip') if restrip is not None: xforms.append('- regex strip: %s' % (restrip,)) nullval = inst.get('nullval') if nullval is not None: cons.append('- null value: %s' % (nullval,)) if core.isSubType(name, 'int'): minval = inst.get('min') if minval is not None: cons.append('- min value: %d (0x%x)' % (minval, minval)) maxval = inst.get('max') if maxval is not None: cons.append('- max value: %d (0x%x)' % (maxval, maxval)) ismin = inst.get('ismin') if ismin is not None: xforms.append('- is minimum: True') ismax = inst.get('ismax') if ismax is not None: xforms.append('- is maximum: True') if core.isSubType(name, 'sepr'): sep = inst.get('sep') fields = inst.get('fields') parts = [] for part in fields.split('|'): name, stype = part.split(',') parts.append(stype) seprs = sep.join(['<%s>' % p for p in parts]) rst.addLines('', 'Sepr Fields: %s' % (seprs,)) if cons: cons.append('') rst.addLines('', 'Type Constraints:', '', *cons) if xforms: xforms.append('') rst.addLines('', 'Type Transforms:', '', *xforms) rst.addHead('Forms', lvl=1) for name, info in forms: ftype = info.get('ptype', 'str') rst.addHead('%s = <%s>' % (name, ftype), lvl=2) doc = core.getPropInfo(name, 'doc') if doc is not None: rst.addLines(doc) rst.addLines('', 'Properties:', '') for prop, pnfo in props.get(name, ()): # use the resolver funcs that will recurse upward pex = core.getPropInfo(prop, 'ex') pdoc = core.getPropInfo(prop, 'doc') ptype = pnfo.get('ptype') pline = '\t- %s = <%s>' % (prop, ptype) defval = pnfo.get('defval') if defval is not None: pline += ' (default: %r)' % (defval,) rst.addLines(pline) if pdoc: rst.addLines('\t\t- %s' % (pdoc,)) # Add universal props last - they are not associated with a form. uniprops = props.get(None) if uniprops: rst.addHead('Universal Props', lvl=1) rst.addLines('', 'Universal props are system level properties which are generally present on every node.', '', 'These properties are not specific to a particular form and exist outside of a particular' ' namespace.', '') plist = sorted(uniprops, key=lambda x: x[0]) for prop, pnfo in plist: rst.addHead(prop, lvl=2) pdoc = core.getPropInfo(prop, 'doc') ptype = pnfo.get('ptype') pline = '\t- %s = <%s>' % (prop, ptype) rst.addLines(pline) if pdoc: rst.addLines('\t\t- %s' % (pdoc,)) outp.printf(rst.getRstText()) if fd: fd.close() return 0