def main(argv, outp=None):
if outp is None: # pragma: no cover
outp = s_output.OutPut()
parser = makeargpaser()
opts = parser.parse_args(argv)
if not opts.verbose:
logging.disable(logging.DEBUG)
if os.path.isfile(opts.output) and not opts.force:
outp.printf('Cannot overwrite a backup.')
return 1
genrows_kwargs = {}
if opts.extra_args:
with open(opts.extra_args, 'rb') as fd:
genrows_kwargs = json.loads(fd.read().decode())
storconf = {'rev:storage': False}
if opts.revstorage:
storconf['rev:storage'] = True
backup_tufo = gen_backup_tufo(opts)
with open(opts.output, 'wb') as fd:
fd.write(s_msgpack.en(backup_tufo))
with s_cortex.openstore(opts.store, storconf=storconf) as store:
dump_store(outp, fd, store,
compress=opts.compress,
dump_blobstore=opts.dump_blobstore,
genrows_kwargs=genrows_kwargs)
outp.printf('Fin')
return 0
def test_model_crypto_201708231712(self):
byts = self.getRev0DbByts()
with self.getTestDir() as temp:
finl = os.path.join(temp, 'test.db')
with open(finl, 'wb') as fd:
fd.write(byts)
url = 'sqlite:///%s' % finl
with s_cortex.openstore(url) as store:
prop = '.:modl:vers:crypto'
valu = 0
rows = store.getRowsByProp(prop)
if rows:
iden = rows[0][0]
else:
iden = guid()
store.setRowsByIdProp(iden, prop, valu)
# Open the cortex, applying the data model updates
# Validate our nodes now have the correct data
with s_cortex.openurl(url) as core:
modlrev = core.getModlVers('crypto')
self.ge(modlrev, 201708231712)
pdef = core.getTufoByProp('syn:prop', 'rsa:key:mod')
self.eq(pdef[1].get('syn:prop:ptype'), 'str:hex')
self.notin('syn:prop:pytpe', pdef[1])
def main(argv, outp=None):
if outp is None: # pragma: no cover
outp = s_output.OutPut()
parser = makeargpaser()
opts = parser.parse_args(argv)
if not opts.verbose:
logging.disable(logging.DEBUG)
# Check to see if we're working with a savefile or a dumprows file
decompress = False
discard_first_event = False
with open(opts.input, 'rb') as fd:
gen = s_msgpack.iterfd(fd)
tufo0 = next(gen)
if tufo0[0] == 'syn:cortex:rowdump:info':
outp.printf('Restoring from a dumprows file.')
discard_first_event = True
decompress = tufo0[1].get('rows:compress')
if decompress:
outp.printf('Gzip row compression enabled.')
else:
outp.printf('Restoring from a savefile')
# No longer need that generator around with the dangler to fd
del gen
storconf = {'rev:storage': False}
if opts.revstorage: # pragma: no cover
storconf['rev:storage'] = True
with open(opts.input, 'rb') as fd:
gen = s_msgpack.iterfd(fd)
if discard_first_event:
next(gen)
with s_cortex.openstore(opts.store, storconf=storconf) as store:
outp.printf('Starting row level restore')
tick = time.time()
i = 0
nrows = 0
for event in gen:
if decompress and 'rows' in event[1]:
event[1]['rows'] = s_msgpack.un(
gzip.decompress(event[1].get('rows')))
i += 1
if i % 250 == 0:
outp.printf('Loaded {} events'.format(i))
store.loadbus.dist(event)
_nrows = len(event[1].get('rows', ()))
nrows += _nrows
if _nrows and i % 10 == 0:
logger.debug('Loaded %s rows', nrows)
tock = time.time()
outp.printf('Done loading events - took {} seconds.'.format(tock -
tick))
outp.printf('Fin')
return 0
def test_model_crypto_201708231712(self):
with s_cortex.openstore('ram:///') as stor:
# force model migration callbacks
stor.setModlVers('crypto', 0)
with s_cortex.fromstore(stor) as core:
modlrev = core.getModlVers('crypto')
self.ge(modlrev, 201708231712)
def test_model_infotech_201801041154(self):
tick = now()
regv = guid()
iden0 = guid()
iden1 = guid()
ndef0 = guid(('it:dev:regkey', 'FooBar'))
ndef1 = guid(('it:dev:regval', regv))
rows = [
(iden0, 'tufo:form', 'it:dev:regkey', tick),
(iden0, 'it:dev:regkey', 'FooBar', tick),
(iden0, 'node:ndef', ndef0, tick),
(iden1, 'tufo:form', 'it:dev:regval', tick),
(iden1, 'it:dev:regval', regv, tick),
(iden1, 'it:dev:regval:key', 'FooBar', tick),
(iden1, 'it:dev:regval:int', 20, tick),
(iden1, 'node:ndef', ndef1, tick),
]
with s_cortex.openstore('ram:///') as stor:
# force model migration callbacks
stor.setModlVers('it', 0)
def addrows(mesg):
stor.addRows(rows)
stor.on('modl:vers:rev', addrows, name='it', vers=201801041154)
with s_cortex.fromstore(stor) as core:
node = core.getTufoByProp('it:dev:regkey', 'foobar')
self.nn(node)
self.eq(node[0], iden0)
self.eq(node[1].get('it:dev:regkey'), 'foobar')
self.eq(node[1].get('node:ndef'),
guid(('it:dev:regkey', 'foobar')))
node = core.getTufoByProp('it:dev:regval:key', 'foobar')
self.nn(node)
self.eq(node[0], iden1)
self.eq(node[1].get('it:dev:regval'), regv)
self.eq(node[1].get('it:dev:regval:key'), 'foobar')
self.eq(node[1].get('node:ndef'), ndef1)
nodes = core.getTufosByDark('syn:modl:rev', 'it:201801041154')
self.len(2, nodes)