def test_axon_basics(self):
with self.getTestDir() as axondir:
axon = s_axon.Axon(axondir)
self.assertFalse( axon.has('md5',craphash) )
self.assertFalse( axon.has('md5',asdfhash) )
iden0 = axon.alloc(8)
self.assertIsNotNone( axon.chunk(iden0,b'asdfasdf') )
self.assertTrue( axon.has('md5',asdfhash) )
self.assertFalse( axon.has('md5',craphash) )
byts = b''.join( axon.bytes('md5',asdfhash) )
self.assertEqual(byts,b'asdfasdf')
axon.fini()
axon = s_axon.Axon(axondir)
self.assertTrue( axon.has('md5',asdfhash) )
self.assertFalse( axon.has('md5',craphash) )
byts = b''.join( axon.bytes('md5',asdfhash) )
self.assertEqual(byts,b'asdfasdf')
axon.fini()
def test_axon_basics(self):
with self.getTestDir() as axondir:
axon = s_axon.Axon(axondir)
self.false(axon.has('md5', craphash))
self.false(axon.has('md5', asdfhash))
iden0 = axon.alloc(8)
self.nn(axon.chunk(iden0, b'asdfasdf'))
self.true(axon.has('md5', asdfhash))
self.false(axon.has('md5', craphash))
byts = b''.join(axon.bytes('md5', asdfhash))
self.eq(byts, b'asdfasdf')
axon.fini()
axon = s_axon.Axon(axondir)
self.true(axon.has('md5', asdfhash))
self.false(axon.has('md5', craphash))
byts = b''.join(axon.bytes('md5', asdfhash))
self.eq(byts, b'asdfasdf')
self.none(axon.wants('md5', asdfhash, 8))
self.nn(axon.wants('md5', craphash, 8))
axon.fini()
def test_axon_base(self):
with self.getTestDir() as dirn:
with s_axon.Axon(dirn) as axon:
self.false(axon.has('md5', asdfmd5))
iden = axon.alloc(8, sha256=asdfsha256)
self.nn(iden)
self.true(axon.chunk(iden, b'asdfasdf'))
self.len(1, list(axon.files(0, 100)))
self.none(axon.alloc(8, sha256=asdfsha256))
self.false(axon.eat(b'asdfasdf'))
self.true(axon.has('md5', asdfmd5))
self.true(axon.has('sha256', asdfsha256))
answ = list(axon.find('sha256', asdfsha256))
self.len(1, answ)
byts = b''.join(axon.bytes('sha256', asdfsha256))
self.eq(b'asdfasdf', byts)
def test_axon_fs_read(self, *args, **kwargs):
with self.getTestDir() as dirname:
with s_axon.Axon(dirname) as axon:
data = b'haha\n'
fd = tempfile.SpooledTemporaryFile()
fd.write(data)
axon.fs_create('/haha', 33204)
_, blobprops = axon.eatfd(fd)
blobsize = blobprops.get('axon:blob:size')
blob = blobprops.get('axon:blob')
axon._fs_update_blob('/haha', blobsize, blob)
fd.close()
sz = axon.fs_getattr('/haha').get('st_size')
actual = axon.fs_read('/haha', sz, 0)
self.eq(actual, data)
self.raises(NoSuchEntity, axon.fs_read, '/hoho2', 100, 0)
axon.fs_create('/haha2', 33204)
actual = axon.fs_read('/haha2', 1000000000, 0)
self.eq(actual, b'')
def test_axonpath(self):
with self.getTestDir() as axondir:
axon = s_axon.Axon(axondir)
defmode = axon.core.getConfOpt('axon:dirmode')
mode = (stat.S_IFDIR | defmode)
nlinks = 1
axon.core.formTufoByProp('axon:path', '/foo/bar/baz/faz/',
st_mode=mode,
st_nlink=nlinks,
)
self.nn(axon.core.getTufoByProp('axon:path', '/foo/bar/baz/faz'))
self.nn(axon.core.getTufoByProp('file:base', 'faz'))
self.nn(axon.core.getTufoByProp('axon:path', '/foo/bar/baz'))
self.nn(axon.core.getTufoByProp('file:base', 'baz'))
self.nn(axon.core.getTufoByProp('axon:path', '/foo/bar'))
self.nn(axon.core.getTufoByProp('file:base', 'bar'))
self.nn(axon.core.getTufoByProp('axon:path', '/foo'))
self.nn(axon.core.getTufoByProp('file:base', 'foo'))
self.nn(axon.core.getTufoByProp('axon:path', '/'))
self.nn(axon.core.getTufoByProp('file:base', ''))
self.raises(BadTypeValu, axon.core.getTufoByProp, 'axon:path', '')
self.raises(BadTypeValu, axon.core.getTufoByProp, 'axon:path', 3)
self.raises(BadTypeValu, axon.core.getTufoByProp, 'file:base', '/')
def test_axon_fs_readdir(self, *args, **kwargs):
with self.getTestDir() as dirname:
with s_axon.Axon(dirname) as axon:
axon.fs_create('/foofile', 33204)
self.eq(sorted(axon.fs_readdir('/')), sorted(['.', '..', 'foofile']))
self.raises(NoSuchEntity, axon.fs_readdir, '/haha')
self.raises(NotSupported, axon.fs_readdir, '/foofile')
def test_axon_fs_getattr(self):
with self.getTestDir() as dirname:
with s_axon.Axon(dirname) as axon:
actual = axon.fs_getattr('/')
self.eq(actual['st_nlink'], 2)
self.eq(actual['st_mode'], 16893)
self.gt(actual['st_atime'], 1000000000)
self.gt(actual['st_ctime'], 1000000000)
self.gt(actual['st_mtime'], 1000000000)
def test_axon_bytesize(self):
opts = {'axon:bytemax': s_axon.megabyte}
with self.getTestDir() as axondir:
with s_axon.Axon(axondir, **opts) as axon:
iden = axon.alloc(1024)
self.nn(iden)
# Exceed bytemax by 1 byte - we should fail to allocate that space.
self.raises(NotEnoughFree, axon.alloc, s_axon.megabyte - 1023)
# But we still have some space left to alloc more blobs
iden2 = axon.alloc(1024)
self.nn(iden2)
def test_axon_basics(self):
with self.getTestDir() as axondir:
with s_axon.Axon(axondir) as axon:
self.false(axon.has('md5', craphash))
self.false(axon.has('md5', asdfhash))
iden0 = axon.alloc(8)
self.nn(axon.chunk(iden0, b'asdfasdf'))
self.raises(NoSuchIden, axon.chunk, guid(), b'asdfasdf')
self.true(axon.has('md5', asdfhash))
self.false(axon.has('md5', craphash))
byts = b''.join(axon.bytes('md5', asdfhash))
self.eq(byts, b'asdfasdf')
byts = b''.join(axon.bytes('guid', asdfhash_iden))
self.eq(byts, b'asdfasdf')
axon.fini()
axon = s_axon.Axon(axondir)
self.true(axon.has('md5', asdfhash))
self.true(axon.has('guid', asdfhash_iden))
self.false(axon.has('md5', craphash))
byts = b''.join(axon.bytes('md5', asdfhash))
self.eq(byts, b'asdfasdf')
self.none(axon.wants('md5', asdfhash, 8))
self.nn(axon.wants('md5', craphash, 8))
with self.assertRaises(NoSuchFile):
_ = [byts for byts in axon.bytes('md5', craphash)]
def test_axon_fs_rmdir(self, *args, **kwargs):
with self.getTestDir() as dirname:
with s_axon.Axon(dirname) as axon:
axon.fs_mkdir('/foo', 0x1FD)
axon.fs_create('/foo/haha', 33204)
self.raises(NotEmpty, axon.fs_rmdir, '/foo')
axon.fs_unlink('/foo/haha')
self.nn(axon.fs_getattr('/foo'))
axon.fs_rmdir('/foo')
self.raises(NoSuchEntity, axon.fs_getattr, '/foo')
self.raises(NoSuchEntity, axon.fs_rmdir, '/foo')
def test_axon_fs_create(self):
with self.getTestDir() as dirname:
with s_axon.Axon(dirname) as axon:
axon.fs_create('/foofile', 33204)
actual = axon.fs_getattr('/foofile')
self.eq(actual['st_nlink'], 1)
self.eq(actual['st_mode'], 33204)
self.gt(actual['st_atime'], 1000000000)
self.gt(actual['st_ctime'], 1000000000)
self.gt(actual['st_mtime'], 1000000000)
self.raises(NoSuchDir, axon.fs_create, '/foodir/foo2', 33204)
def test_axon_telepath(self):
with self.getTestDir() as dirname:
with s_daemon.Daemon() as dmon:
link = dmon.listen('tcp://127.0.0.1:0/')
port = link[1].get('port')
with s_axon.Axon(dirname) as axon:
dmon.share('axon', axon, fini=True)
with s_telepath.openurl('tcp://127.0.0.1/axon', port=port) as prox:
with io.BytesIO(b'vertex') as fd:
blob = prox.eatfd(fd)
self.eq(blob[1]['axon:blob:sha256'],
'e1b683e26a3aad218df6aa63afe9cf57fdb5dfaf5eb20cddac14305d67f48a02')
def test_axon_fs_getxattr(self):
with self.getTestDir() as dirname:
with s_axon.Axon(dirname) as axon:
actual = axon.fs_getattr('/')
self.eq(actual['st_nlink'], 2)
self.eq(actual['st_mode'], 16893)
self.gt(actual['st_atime'], 1000000000)
self.gt(actual['st_ctime'], 1000000000)
self.gt(actual['st_mtime'], 1000000000)
nlink = axon.fs_getxattr('/', 'st_nlink')
self.eq(nlink, actual['st_nlink'])
self.raises(NoSuchData, axon.fs_getxattr, '/', 'zzz')
self.raises(NoSuchData, axon.fs_getxattr, '/haha', 'st_nlink')
def test_axon_fs_unlink(self, *args, **kwargs):
with self.getTestDir() as dirname:
with s_axon.Axon(dirname) as axon:
axon.fs_create('/foofile', 33204)
actual = axon.fs_getattr('/foofile')
self.eq(actual['st_nlink'], 1)
self.eq(actual['st_mode'], 33204)
self.gt(actual['st_atime'], 1000000000)
self.gt(actual['st_ctime'], 1000000000)
self.gt(actual['st_mtime'], 1000000000)
axon.fs_unlink('/foofile')
self.raises(NoSuchEntity, axon.fs_getattr, '/foofile')
self.raises(NoSuchFile, axon.fs_unlink, '/foofile')
def test_axon_fs_truncate(self, *args, **kwargs):
with self.getTestDir() as dirname:
with s_axon.Axon(dirname) as axon:
axon.fs_create('/foofile', 33204)
tufo = axon.core.getTufoByProp('axon:path', '/foofile')
axon.core.setTufoProps(tufo, st_size=100, blob=32 * 'a')
self.eq(tufo[1].get('axon:path:st_size'), 100)
self.eq(tufo[1].get('axon:path:blob'), 32 * 'a')
axon.fs_truncate('/foofile')
tufo = axon.core.getTufoByProp('axon:path', '/foofile')
self.eq(tufo[1].get('axon:path:st_size'), 0)
self.eq(tufo[1].get('axon:path:blob'), None)
self.raises(NoSuchEntity, axon.fs_truncate, '/notthere')
def test_axon_fs_mkdir(self):
with self.getTestDir() as dirname:
with s_axon.Axon(dirname) as axon:
actual = axon.fs_mkdir('/foo', 0x1FD)
self.eq(actual, None)
actual = axon.fs_getattr('/foo')
self.eq(actual['st_nlink'], 2)
self.eq(actual['st_mode'], 16893)
self.gt(actual['st_atime'], 1000000000)
self.gt(actual['st_ctime'], 1000000000)
self.gt(actual['st_mtime'], 1000000000)
self.raises(NoSuchDir, axon.fs_mkdir, '/foodir/foo2', 16893)
# self.raises(NoSuchDir, axon.fs_mkdir, 'noparent', 16893)
self.raises(FileExists, axon.fs_mkdir, '/foo', 0x1FD)
def test_axon_sync(self):
with self.getTestDir() as axondir:
byts = os.urandom(128)
bytsmd5 = hashlib.md5(byts).hexdigest()
axon = s_axon.Axon(axondir,syncsize=64)
iden = axon.alloc(128)
for chnk in chunks(byts,10):
blob = axon.chunk(iden,chnk)
self.assertIsNotNone(blob)
self.assertTrue( axon.has('md5',bytsmd5) )
axon.fini()
def test_tools_pushfile(self):
with self.getTestDir() as path:
visipath = os.path.join(path, 'visi.txt')
with open(visipath, 'wb') as fd:
fd.write(b'visi')
outp = self.getTestOutp()
with s_daemon.Daemon() as dmon:
link = dmon.listen('tcp://127.0.0.1:0/')
port = link[1].get('port')
core = s_cortex.openurl('ram:///')
dmon.onfini(core.fini)
axon = s_axon.Axon(os.path.join(path, 'axon00'))
dmon.onfini(axon.fini)
dmon.share('axon00', axon)
dmon.share('core00', core)
core.setConfOpt('axon:url', 'tcp://127.0.0.1:%d/axon00' % port)
s_pushfile.main([
'--tags', 'foo.bar,baz.faz',
'tcp://127.0.0.1:%d/core00' % port, visipath
],
outp=outp)
node = core.getTufoByProp('file:bytes')
self.eq(node[1].get('file:bytes'),
'442f602ecf8230b2a59a44b4f845be27')
self.eq(node[1].get('file:bytes:size'), 4)
self.nn(node[1].get('#foo'))
self.nn(node[1].get('#foo.bar'))
self.nn(node[1].get('#baz'))
self.nn(node[1].get('#baz.faz'))
def test_axonblob(self):
with self.getTestDir() as axondir:
axon = s_axon.Axon(axondir)
core = axon.core
hset = s_axon.HashSet()
hset.update(b'visi')
valu, props = hset.guid()
props.update({'off': 0})
t0 = core.formTufoByProp('axon:blob', valu, **props)
self.eq(t0[1].get('axon:blob:off'), 0)
self.eq(t0[1].get('axon:blob:size'), 4)
self.eq(t0[1].get('axon:blob'), '442f602ecf8230b2a59a44b4f845be27')
self.eq(t0[1].get('axon:blob:md5'), '1b2e93225959e3722efed95e1731b764')
self.eq(t0[1].get('axon:blob:sha1'), '93de0c7d579384feb3561aa504acd8f23f388040')
self.eq(t0[1].get('axon:blob:sha256'), 'e45bbb7e03acacf4d1cca4c16af1ec0c51d777d10e53ed3155bd3d8deb398f3f')
self.eq(t0[1].get('axon:blob:sha512'), '8238be12bcc3c10da7e07dbea528e9970dc809c07c5aef545a14e5e8d2038563b29c2e818d167b06e6a33412e6beb8347fcc44520691347aea9ee21fcf804e39')
def test_axon_eatbytes(self):
self.thisHostMustNot(platform='windows')
with self.getTestDir() as dirname:
with s_axon.Axon(dirname) as axon:
blob0 = axon.eatbytes(b'visi')
with io.BytesIO(b'vertex') as fd:
blob1 = axon.eatfd(fd)
port = axon.getAxonInfo()[1].get('link')[1].get('port')
with s_telepath.openurl('tcp://127.0.0.1/axon', port=port) as prox:
blob2 = prox.eatbytes(b'hurr')
with io.BytesIO(b'durr') as fd:
blob3 = prox.eatfd(fd)
self.eq(blob0[1].get('axon:blob'), '442f602ecf8230b2a59a44b4f845be27')
self.eq(blob1[1].get('axon:blob'), 'd4552906c1f6966b96d27e6fc79441b5')
self.eq(blob2[1].get('axon:blob'), '0d60960570ef6da0a15f68c24b420334')
self.eq(blob3[1].get('axon:blob'), '97c11d1057f75c9c0b79090131709f62')
def test_axonpath(self):
with self.getTestDir() as axondir:
axon = s_axon.Axon(axondir)
axon.core.formTufoByProp('axon:path', '/foo/bar/baz/faz/')
self.nn(axon.core.getTufoByProp('axon:path', '/foo/bar/baz/faz'))
self.nn(axon.core.getTufoByProp('file:base', 'faz'))
self.nn(axon.core.getTufoByProp('axon:path', '/foo/bar/baz'))
self.nn(axon.core.getTufoByProp('file:base', 'baz'))
self.nn(axon.core.getTufoByProp('axon:path', '/foo/bar'))
self.nn(axon.core.getTufoByProp('file:base', 'bar'))
self.nn(axon.core.getTufoByProp('axon:path', '/foo'))
self.nn(axon.core.getTufoByProp('file:base', 'foo'))
self.nn(axon.core.getTufoByProp('axon:path', '/'))
self.raises(BadTypeValu, axon.core.getTufoByProp, 'axon:path', '')
self.raises(BadTypeValu, axon.core.getTufoByProp, 'axon:path', 3)
self.none(axon.core.getTufoByProp('file:base', ''))
self.raises(BadTypeValu, axon.core.getTufoByProp, 'file:base', '/')
def test_axon_restrictions(self):
with self.getTestDir() as axondir:
with s_axon.Axon(axondir) as axon:
iden0 = axon.alloc(8)
self.nn(axon.chunk(iden0, b'asdfasdf'))
# set the axon as read-only and ensure we cannot write new blobs to it
axon.setConfOpt('axon:ro', 1)
axfo = axon.getAxonInfo()
self.eq(axfo[1].get('opts').get('axon:ro'), 1)
self.raises(NotSupported, axon.alloc, 8)
# But we can still read from axon:ro=1 axons
byts = b''.join(axon.bytes('guid', asdfhash_iden))
self.eq(byts, b'asdfasdf')
# non-clones cannot sync events
self.raises(NotSupported, axon.sync, ())
# clones cannot alloc new blobs directly
axon.setConfOpt('axon:clone', 1)
self.raises(NotSupported, axon.alloc, 8)
def test_axon_fs_utimens(self, *args, **kwargs):
with self.getTestDir() as dirname:
with s_axon.Axon(dirname) as axon:
axon.fs_create('/foofile', 33204)
actual = axon.fs_getattr('/foofile')
ctime = actual['st_ctime']
self.gt(actual['st_atime'], 1000000000)
self.gt(actual['st_ctime'], 1000000000)
self.gt(actual['st_mtime'], 1000000000)
axon.fs_utimens('/foofile', (0, 0))
actual = axon.fs_getattr('/foofile')
self.eq(actual['st_mtime'], 0)
self.eq(actual['st_ctime'], ctime)
self.eq(actual['st_atime'], 0)
axon.fs_utimens('/foofile')
self.eq(actual['st_mtime'], 0)
self.eq(actual['st_ctime'], ctime)
self.eq(actual['st_atime'], 0)
def test_axon_fs_rename(self, *args, **kwargs):
with self.getTestDir() as dirname:
with s_axon.Axon(dirname) as axon:
axon.fs_mkdir('/dir', 0x1FD)
axon.fs_create('/dir/a', 33204)
axon.fs_rename('/dir/a', '/dir/b')
actual = axon.fs_readdir('/dir')
self.eq(sorted(actual), ['.', '..', 'b'])
# source doesnt exist
self.raises(NoSuchEntity, axon.fs_rename, 'fake', 'other')
# source doesnt have parent
tufo = axon.core.getTufoByProp('axon:path', '/dir')
axon.core.delTufo(tufo)
self.raises(NoSuchDir, axon.fs_rename, '/dir/b', '/dir/c')
# dst doesnt have parent
axon.fs_mkdir('/dira', 0x1FD)
axon.fs_mkdir('/dirb', 0x1FD)
axon.fs_create('/dira/a', 33204)
tufo = axon.core.getTufoByProp('axon:path', '/dirb')
axon.core.delTufo(tufo)
self.raises(NoSuchDir, axon.fs_rename, '/dira/a', '/dirb/b')
# dst not empty
axon.fs_mkdir('/flda', 0x1FD)
axon.fs_mkdir('/fldb', 0x1FD)
axon.fs_create('/flda/a', 33204)
axon.fs_create('/fldb/a', 33204)
self.raises(NotEmpty, axon.fs_rename, '/flda', '/fldb')
# overwrite a regular file with a directory
axon.fs_mkdir('/adir', 0x1FD)
axon.fs_create('/reg', 33204)
axon.fs_rename('/adir', '/reg')
self.none(axon.fs_getattr('/reg').get('st_size'))
# update all the kids
axon.fs_mkdir('/cool', 0x1FD)
axon.fs_create('/cool/a', 33204)
axon.fs_create('/cool/b', 33204)
axon.fs_create('/cool/c', 33204)
axon.fs_rename('/cool', '/cooler')
self.raises(NoSuchEntity, axon.fs_getattr, '/cool')
self.raises(NoSuchEntity, axon.fs_getattr, '/cool/a')
self.raises(NoSuchEntity, axon.fs_getattr, '/cool/b')
self.raises(NoSuchEntity, axon.fs_getattr, '/cool/c')
self.nn(axon.fs_getattr('/cooler'))
self.nn(axon.fs_getattr('/cooler/a'))
self.nn(axon.fs_getattr('/cooler/b'))
self.nn(axon.fs_getattr('/cooler/c'))
# nested dirs
axon.fs_mkdir('/nest1', 0x1FD)
axon.fs_mkdir('/nest1/nest2', 0x1FD)
axon.fs_mkdir('/nest1/nest2/nest3', 0x1FD)
axon.fs_create('/nest1/nest2/nest3/reg', 33204)
axon.fs_rename('/nest1', '/nest')
self.nn(axon.fs_getattr('/nest'))
self.nn(axon.fs_getattr('/nest/nest2'))
self.nn(axon.fs_getattr('/nest/nest2/nest3'))
self.nn(axon.fs_getattr('/nest/nest2/nest3/reg'))
