def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() parser = makeargpaser() opts = parser.parse_args(argv) if not opts.verbose: logging.disable(logging.DEBUG) if os.path.isfile(opts.output) and not opts.force: outp.printf('Cannot overwrite a backup.') return 1 genrows_kwargs = {} if opts.extra_args: with open(opts.extra_args, 'rb') as fd: genrows_kwargs = json.loads(fd.read().decode()) storconf = {'rev:storage': False} if opts.revstorage: storconf['rev:storage'] = True backup_tufo = gen_backup_tufo(opts) with open(opts.output, 'wb') as fd: fd.write(s_msgpack.en(backup_tufo)) with s_cortex.openstore(opts.store, storconf=storconf) as store: dump_store(outp, fd, store, compress=opts.compress, dump_blobstore=opts.dump_blobstore, genrows_kwargs=genrows_kwargs) outp.printf('Fin') return 0
def test_model_crypto_201708231712(self): byts = self.getRev0DbByts() with self.getTestDir() as temp: finl = os.path.join(temp, 'test.db') with open(finl, 'wb') as fd: fd.write(byts) url = 'sqlite:///%s' % finl with s_cortex.openstore(url) as store: prop = '.:modl:vers:crypto' valu = 0 rows = store.getRowsByProp(prop) if rows: iden = rows[0][0] else: iden = guid() store.setRowsByIdProp(iden, prop, valu) # Open the cortex, applying the data model updates # Validate our nodes now have the correct data with s_cortex.openurl(url) as core: modlrev = core.getModlVers('crypto') self.ge(modlrev, 201708231712) pdef = core.getTufoByProp('syn:prop', 'rsa:key:mod') self.eq(pdef[1].get('syn:prop:ptype'), 'str:hex') self.notin('syn:prop:pytpe', pdef[1])
def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() parser = makeargpaser() opts = parser.parse_args(argv) if not opts.verbose: logging.disable(logging.DEBUG) # Check to see if we're working with a savefile or a dumprows file decompress = False discard_first_event = False with open(opts.input, 'rb') as fd: gen = s_msgpack.iterfd(fd) tufo0 = next(gen) if tufo0[0] == 'syn:cortex:rowdump:info': outp.printf('Restoring from a dumprows file.') discard_first_event = True decompress = tufo0[1].get('rows:compress') if decompress: outp.printf('Gzip row compression enabled.') else: outp.printf('Restoring from a savefile') # No longer need that generator around with the dangler to fd del gen storconf = {'rev:storage': False} if opts.revstorage: # pragma: no cover storconf['rev:storage'] = True with open(opts.input, 'rb') as fd: gen = s_msgpack.iterfd(fd) if discard_first_event: next(gen) with s_cortex.openstore(opts.store, storconf=storconf) as store: outp.printf('Starting row level restore') tick = time.time() i = 0 nrows = 0 for event in gen: if decompress and 'rows' in event[1]: event[1]['rows'] = s_msgpack.un( gzip.decompress(event[1].get('rows'))) i += 1 if i % 250 == 0: outp.printf('Loaded {} events'.format(i)) store.loadbus.dist(event) _nrows = len(event[1].get('rows', ())) nrows += _nrows if _nrows and i % 10 == 0: logger.debug('Loaded %s rows', nrows) tock = time.time() outp.printf('Done loading events - took {} seconds.'.format(tock - tick)) outp.printf('Fin') return 0
def test_model_crypto_201708231712(self): with s_cortex.openstore('ram:///') as stor: # force model migration callbacks stor.setModlVers('crypto', 0) with s_cortex.fromstore(stor) as core: modlrev = core.getModlVers('crypto') self.ge(modlrev, 201708231712)
def test_model_infotech_201801041154(self): tick = now() regv = guid() iden0 = guid() iden1 = guid() ndef0 = guid(('it:dev:regkey', 'FooBar')) ndef1 = guid(('it:dev:regval', regv)) rows = [ (iden0, 'tufo:form', 'it:dev:regkey', tick), (iden0, 'it:dev:regkey', 'FooBar', tick), (iden0, 'node:ndef', ndef0, tick), (iden1, 'tufo:form', 'it:dev:regval', tick), (iden1, 'it:dev:regval', regv, tick), (iden1, 'it:dev:regval:key', 'FooBar', tick), (iden1, 'it:dev:regval:int', 20, tick), (iden1, 'node:ndef', ndef1, tick), ] with s_cortex.openstore('ram:///') as stor: # force model migration callbacks stor.setModlVers('it', 0) def addrows(mesg): stor.addRows(rows) stor.on('modl:vers:rev', addrows, name='it', vers=201801041154) with s_cortex.fromstore(stor) as core: node = core.getTufoByProp('it:dev:regkey', 'foobar') self.nn(node) self.eq(node[0], iden0) self.eq(node[1].get('it:dev:regkey'), 'foobar') self.eq(node[1].get('node:ndef'), guid(('it:dev:regkey', 'foobar'))) node = core.getTufoByProp('it:dev:regval:key', 'foobar') self.nn(node) self.eq(node[0], iden1) self.eq(node[1].get('it:dev:regval'), regv) self.eq(node[1].get('it:dev:regval:key'), 'foobar') self.eq(node[1].get('node:ndef'), ndef1) nodes = core.getTufosByDark('syn:modl:rev', 'it:201801041154') self.len(2, nodes)