def test_dump_force(self):
outp = self.getTestOutp()
with self.getTestDir() as temp:
fp = os.path.join(temp, 'dumpfile.mpk')
new_db = os.path.join(temp, 'test.db')
sqlite_url = 'sqlite:///{}'.format(new_db)
with s_cortex.openurl(sqlite_url) as core:
self.true(core.isnew)
core.setBlobValu('syn:test:tel', 8675309)
core.formTufoByProp('inet:ipv4', 0x01020304)
# Now dump that sqlite core
argv = ['-s', sqlite_url, '-o', fp]
ret = s_dumprows.main(argv, outp)
self.eq(ret, 0)
outp = self.getTestOutp()
argv = ['-s', sqlite_url, '-o', fp]
ret = s_dumprows.main(argv, outp)
self.eq(ret, 1)
self.true('Cannot overwrite a backup.' in str(outp))
outp = self.getTestOutp()
# Now dump that sqlite core
argv = ['-s', sqlite_url, '-o', fp, '-f']
ret = s_dumprows.main(argv, outp)
self.eq(ret, 0)
def test_revstore_use(self):
outp = self.getTestOutp()
byts = self.getRev0DbByts()
with self.getTestDir() as temp:
new_db = os.path.join(temp, 'rev0.db')
with open(new_db, 'wb') as fd:
fd.write(byts)
# First ensure that we are dumping the DB as it is on disk!
sqlite_url = 'sqlite:///{}'.format(new_db)
# Now dump that sqlite core
fp = os.path.join(temp, 'dumpfile.mpk')
argv = ['-s', sqlite_url, '-o', fp]
ret = s_dumprows.main(argv, outp)
self.eq(ret, 0)
revkey = 'syn:core:sqlite:version'
# Now ensure our .mpk file is correct
with open(fp, 'rb') as fd:
gen = msgpackfd(fd)
evt = next(gen)
self.eq(evt[0], 'syn:cortex:rowdump:info')
self.eq(evt[1].get('synapse:cortex:revstore'), False)
# Open the storage object and ensure the rev0 function did not run
with s_cortex.openstore(sqlite_url,
storconf={'rev:storage': False}) as store:
self.false(store.hasBlobValu(revkey))
os.unlink(fp)
# Now ensure when we set the rev:storage flag to true
# that it auto-updates DBs first
fp = os.path.join(temp, 'dumpfile.mpk')
argv = ['-s', sqlite_url, '-o', fp, '--enable-revstorage']
ret = s_dumprows.main(argv, outp)
self.eq(ret, 0)
# Now ensure our .mpk file is correct
with open(fp, 'rb') as fd:
gen = msgpackfd(fd)
evt = next(gen)
self.eq(evt[0], 'syn:cortex:rowdump:info')
self.eq(evt[1].get('synapse:cortex:revstore'), True)
# Open the storage object and ensure the revisioning did run
with s_cortex.openstore(sqlite_url,
storconf={'rev:storage': False}) as store:
self.true(store.hasBlobValu(revkey))
def test_simple_use(self):
self.thisHostMustNot(platform='darwin')
outp = self.getTestOutp()
with self.getTestDir() as temp:
fp = os.path.join(temp, 'dumpfile.mpk')
new_db = os.path.join(temp, 'test.db')
sqlite_url = 'sqlite:///{}'.format(new_db)
with s_cortex.openurl(sqlite_url) as core:
self.true(core.isnew)
core.setBlobValu('syn:test:tel', 8675309)
with core.getCoreXact():
core.formTufoByProp('inet:ipv4', 0x01020304)
for i in range(1000):
core.formTufoByProp('inet:ipv4', i)
# Now dump that sqlite core
argv = ['-s', sqlite_url, '-o', fp]
ret = s_dumprows.main(argv, outp)
self.eq(ret, 0)
# Now ensure our .mpk file is correct
with open(fp, 'rb') as fd:
gen = s_msgpack.iterfd(fd)
evt = next(gen)
self.eq(evt[0], 'syn:cortex:rowdump:info')
self.eq(evt[1].get('rows:compress'), False)
self.eq(evt[1].get('synapse:rows:output'), fp)
self.eq(evt[1].get('synapse:cortex:input'), sqlite_url)
self.eq(evt[1].get('synapse:cortex:blob_store'), False)
self.eq(evt[1].get('synapse:cortex:revstore'), False)
self.eq(evt[1].get('python:version'), version)
self.isin('synapse:version', evt[1])
evt = next(gen)
self.eq(evt[0], 'core:save:add:rows')
self.isin('rows', evt[1])
rows = evt[1].get('rows')
self.isinstance(rows, tuple)
self.isinstance(rows[0], tuple)
self.eq(len(rows[0]), 4)
# Expensive but worth checking
event_types = set()
event_types.add(evt[0])
total_rows = 0
for evt in gen:
event_types.add(evt[0])
if 'rows' in evt[1]:
total_rows = total_rows + len(evt[1].get('rows'))
self.gt(total_rows, 1000)
self.eq(event_types, {'core:save:add:rows'})
def test_dump_largecore(self):
self.skipLongTest()
self.thisHostMustNot(platform='darwin')
# This ensure we're executing the "dump rows
# when we have N number of bytes cached codepath.
# Unfortunately this is a bit slow (2-4 seconds).
ntufos = 40000
outp = self.getTestOutp()
with self.getTestDir() as temp:
fp = os.path.join(temp, 'dumpfile.mpk')
new_db = os.path.join(temp, 'test.db')
sqlite_url = 'sqlite:///{}'.format(new_db)
with s_cortex.openurl(sqlite_url) as core:
self.true(core.isnew)
rows = []
tick = now()
for i in range(1, ntufos):
iden = guid()
rows.append((iden, 'tufo:form', 'inet:asn', tick))
rows.append((iden, 'inet:asn', i, tick))
rows.append((iden, 'inet:asn:name', '??', tick))
core.addRows(rows)
q = 'SELECT count(1) from {}'.format(
core.store._getTableName())
num_core_rows = core.store.select(q)[0][0]
# Now dump that sqlite core
argv = ['-s', sqlite_url, '-o', fp]
ret = s_dumprows.main(argv, outp)
self.eq(ret, 0)
stat = os.stat(fp)
self.gt(stat.st_size, s_const.mebibyte * 4)
# Now ensure our .mpk file is correct
with open(fp, 'rb') as fd:
msgpk_rows = 0
for evt in s_msgpack.iterfd(fd):
if 'rows' in evt[1]:
msgpk_rows = msgpk_rows + len(evt[1].get('rows'))
self.eq(num_core_rows, msgpk_rows)
def test_dumprows_load_compressed(self):
self.thisHostMustNot(platform='darwin')
outp = self.getTestOutp()
with self.getTestDir() as temp:
# Make a sqlite cortex and the associated dupmfile for it
fp = os.path.join(temp, 'dumpfile.mpk')
genrows_json_fp = os.path.join(temp, 'genrows.json')
self.make_sql_genrows_json(genrows_json_fp)
old_db = os.path.join(temp, 'old.db')
new_db = os.path.join(temp, 'new.db')
sqlite_url_old = 'sqlite:///{}'.format(old_db)
sqlite_url_new = 'sqlite:///{}'.format(new_db)
with s_cortex.openurl(sqlite_url_old) as core:
self.true(core.isnew)
node = core.formTufoByProp('inet:ipv4', 0x01020304)
self.true('.new' in node[1])
core.setBlobValu('foo:bar', ('tufo', {'test': 'value'}))
# Dump that core and its blobstore to a dumpfile
dump_argv = [
'-s', sqlite_url_old, '-o', fp, '--dump-blobstore',
'--compress', '-e', genrows_json_fp
]
ret = s_dumprows.main(dump_argv, self.getTestOutp())
self.eq(ret, 0)
# Execute loadrows tool to create the sqlite store and load the rows
argv = ['-s', sqlite_url_new, '-i', fp]
ret = s_loadrows.main(argv, outp)
self.eq(ret, 0)
self.true('Gzip row compression enabled' in str(outp))
# Make sure the output is valid
with s_cortex.openurl(sqlite_url_new) as core:
self.false(core.isnew)
node = core.formTufoByProp('inet:ipv4', 0x01020304)
self.true('.new' not in node[1])
self.eq(core.getBlobValu('foo:bar'), ('tufo', {
'test': 'value'
}))
def test_simple_compress(self):
self.thisHostMustNot(platform='darwin')
outp = self.getTestOutp()
with self.getTestDir() as temp:
fp = os.path.join(temp, 'dumpfile.mpk')
new_db = os.path.join(temp, 'test.db')
sqlite_url = 'sqlite:///{}'.format(new_db)
with s_cortex.openurl(sqlite_url) as core:
self.true(core.isnew)
core.setBlobValu('syn:test:tel', 8675309)
core.formTufoByProp('inet:ipv4', 0x01020304)
# Now dump that sqlite core
argv = ['-s', sqlite_url, '-o', fp, '--compress']
ret = s_dumprows.main(argv, outp)
self.eq(ret, 0)
# Now ensure our .mpk file is correct
with open(fp, 'rb') as fd:
gen = s_msgpack.iterfd(fd)
evt = next(gen)
self.eq(evt[0], 'syn:cortex:rowdump:info')
self.eq(evt[1].get('rows:compress'), True)
evt = next(gen)
self.eq(evt[0], 'core:save:add:rows')
self.isin('rows', evt[1])
rows = evt[1].get('rows')
# we decode the rows blob not in place but separately here
rows = s_msgpack.un(gzip.decompress(rows))
self.isinstance(rows, tuple)
self.isinstance(rows[0], tuple)
self.eq(len(rows[0]), 4)
# Expensive but worth checking
event_types = set()
event_types.add(evt[0])
for evt in gen:
event_types.add(evt[0])
self.eq(event_types, {'core:save:add:rows'})
def test_blob_dump(self):
outp = self.getTestOutp()
with self.getTestDir() as temp:
fp = os.path.join(temp, 'dumpfile.mpk')
new_db = os.path.join(temp, 'test.db')
sqlite_url = 'sqlite:///{}'.format(new_db)
with s_cortex.openurl(sqlite_url) as core:
self.true(core.isnew)
core.setBlobValu('syn:test:tel', 8675309)
core.formTufoByProp('inet:ipv4', 0x01020304)
# Now dump that sqlite core
argv = ['-s', sqlite_url, '-o', fp, '--dump-blobstore']
ret = s_dumprows.main(argv, outp)
self.eq(ret, 0)
# Now ensure our .mpk file is correct
with open(fp, 'rb') as fd:
gen = s_msgpack.iterfd(fd)
evt = next(gen)
self.eq(evt[0], 'syn:cortex:rowdump:info')
self.eq(evt[1].get('synapse:cortex:blob_store'), True)
evt = next(gen)
self.eq(evt[0], 'core:save:add:rows')
self.isin('rows', evt[1])
rows = evt[1].get('rows')
self.isinstance(rows, tuple)
self.isinstance(rows[0], tuple)
self.eq(len(rows[0]), 4)
# Expensive but worth checking
event_types = set()
event_types.add(evt[0])
for evt in gen:
event_types.add(evt[0])
self.eq(event_types,
{'core:save:add:rows', 'syn:core:blob:set'})
