def test_certdir_importfile(self):
with self.getCertDir() as cdir: # type: s_certdir.CertDir
with self.getTestDir() as testpath:
# File doesn't exist
fpath = s_common.genpath(testpath, 'not_real.crt')
self.raises(s_exc.NoSuchFile, cdir.importFile, fpath, 'cas')
# File has unsupported extension
fpath = s_common.genpath(testpath, 'coolpic.bmp')
with s_common.genfile(fpath) as fd:
self.raises(s_exc.BadFileExt, cdir.importFile, fpath,
'cas')
tests = (
('cas', 'coolca.crt'),
('cas', 'coolca.key'),
('hosts', 'coolhost.crt'),
('hosts', 'coolhost.key'),
('users', 'cooluser.crt'),
('users', 'cooluser.key'),
('users', 'cooluser.p12'),
)
for ftype, fname in tests:
srcpath = s_common.genpath(testpath, fname)
dstpath = s_common.genpath(cdir.certdirs[0], ftype, fname)
with s_common.genfile(srcpath) as fd:
fd.write(b'arbitrary data')
fd.seek(0)
# Make sure the file is not there
self.raises(s_exc.NoSuchFile, s_common.reqfile,
dstpath)
# Import it and make sure it exists
self.none(cdir.importFile(srcpath, ftype))
with s_common.reqfile(dstpath) as dstfd:
self.eq(dstfd.read(), b'arbitrary data')
# Make sure it can't be overwritten
self.raises(s_exc.FileExists, cdir.importFile, srcpath,
ftype)
def loadfile(*paths):
'''
Load a json ingest def from file and construct an Ingest class.
This routine is useful because it implements the convention
for adding runtime info to the ingest json to facilitate path
relative file opening etc...
'''
path = s_common.genpath(*paths)
# FIXME universal open
with s_common.reqfile(path) as fd:
jsfo = json.loads(fd.read().decode('utf8'))
gest = Ingest(jsfo)
gest.set('basedir', os.path.dirname(path))
return gest
def test_certdir_importfile(self):
with self.getCertDir() as cdir: # type: s_certdir.CertDir
with self.getTestDir() as testpath:
# File doesn't exist
fpath = s_common.genpath(testpath, 'not_real.crt')
self.raises(s_exc.NoSuchFile, cdir.importFile, fpath, 'cas')
# File has unsupported extension
fpath = s_common.genpath(testpath, 'coolpic.bmp')
with s_common.genfile(fpath) as fd:
self.raises(s_exc.BadFileExt, cdir.importFile, fpath, 'cas')
tests = (
('cas', 'coolca.crt'),
('cas', 'coolca.key'),
('hosts', 'coolhost.crt'),
('hosts', 'coolhost.key'),
('users', 'cooluser.crt'),
('users', 'cooluser.key'),
('users', 'cooluser.p12'),
)
for ftype, fname in tests:
srcpath = s_common.genpath(testpath, fname)
dstpath = s_common.genpath(cdir.path, ftype, fname)
with s_common.genfile(srcpath) as fd:
fd.write(b'arbitrary data')
fd.seek(0)
# Make sure the file is not there
self.raises(s_exc.NoSuchFile, s_common.reqfile, dstpath)
# Import it and make sure it exists
self.none(cdir.importFile(srcpath, ftype))
with s_common.reqfile(dstpath) as dstfd:
self.eq(dstfd.read(), b'arbitrary data')
# Make sure it can't be overwritten
self.raises(s_exc.FileExists, cdir.importFile, srcpath, ftype)
async def main(argv, outp=None):
if outp is None: # pragma: no cover
outp = s_output.OutPut()
path = s_common.getSynPath('telepath.yaml')
telefini = await s_telepath.loadTeleEnv(path)
pars = makeargparser()
opts = pars.parse_args(argv)
axon = await s_telepath.openurl(opts.axon)
core = None
if opts.cortex:
core = await s_telepath.openurl(opts.cortex)
tags = set()
if opts.tags:
for tag in opts.tags.split(','):
tags.add(tag)
tags = tuple(tags)
if tags:
outp.printf(f'adding tags: {tags}')
filepaths = set()
for item in opts.filenames:
paths = glob.glob(item, recursive=opts.recursive)
if not paths:
outp.printf(f'filepath does not contain any files: {item}')
continue
filepaths.update([path for path in paths if os.path.isfile(path)])
for path in filepaths:
bname = os.path.basename(path)
hset = s_hashset.HashSet()
with s_common.reqfile(path) as fd:
hset.eatfd(fd)
fhashes = {htyp: hasher.hexdigest() for htyp, hasher in hset.hashes}
sha256 = fhashes.get('sha256')
bsha256 = s_common.uhex(sha256)
if not await axon.has(bsha256):
async with await axon.upload() as upfd:
with s_common.genfile(path) as fd:
for byts in s_common.iterfd(fd):
await upfd.write(byts)
size, hashval = await upfd.save()
if hashval != bsha256: # pragma: no cover
raise s_exc.SynErr(mesg='hashes do not match',
ehash=s_common.ehex(hashval),
ahash=hashval)
outp.printf(f'Uploaded [{bname}] to axon')
else:
outp.printf(f'Axon already had [{bname}]')
if core:
opts = {
'vars': {
'md5': fhashes.get('md5'),
'sha1': fhashes.get('sha1'),
'sha256': fhashes.get('sha256'),
'size': hset.size,
'name': bname,
'tags': tags,
}
}
q = '[file:bytes=$sha256 :md5=$md5 :sha1=$sha1 :size=$size :name=$name] ' \
'{ for $tag in $tags { [+#$tag] } }'
msgs = await core.storm(q, opts=opts).list()
node = [m[1] for m in msgs if m[0] == 'node'][0]
iden = node[0][1]
size = node[1]['props']['size']
name = node[1]['props']['name']
mesg = f'file: {bname} ({size}) added to core ({iden}) as {name}'
outp.printf(mesg)
await axon.fini()
if core:
await core.fini()
if telefini: # pragma: no cover
await telefini()
return 0
def main(argv, outp=None):
if outp is None: # pragma: no cover
outp = s_output.OutPut()
pars = makeargparser()
opts = pars.parse_args(argv)
axon = s_telepath.openurl(opts.axon)
core = None
if opts.cortex:
core = s_telepath.openurl(opts.cortex)
tags = {}
if opts.tags:
for tag in opts.tags.split(','):
tags[tag] = (None, None)
if tags:
outp.printf('adding tags: %r' % (list(tags.keys())))
filepaths = set()
for item in opts.filenames:
paths = glob.glob(item, recursive=opts.recursive)
if not paths:
outp.printf(f'filepath does not contain any files: {item}')
continue
filepaths.update([path for path in paths if os.path.isfile(path)])
for path in filepaths:
bname = os.path.basename(path)
hset = s_hashset.HashSet()
with s_common.reqfile(path) as fd:
hset.eatfd(fd)
fhashes = {htyp: hasher.hexdigest() for htyp, hasher in hset.hashes}
sha256 = fhashes.get('sha256')
bsha256 = s_common.uhex(sha256)
if not axon.has(bsha256):
with axon.upload() as upfd:
with s_common.genfile(path) as fd:
for byts in s_common.iterfd(fd):
upfd.write(byts)
size, hashval = upfd.save()
if hashval != bsha256: # pragma: no cover
raise s_exc.SynErr(mesg='hashes do not match',
ehash=s_common.ehex(hashval),
ahash=hashval)
outp.printf(f'Uploaded [{bname}] to axon')
else:
outp.printf(f'Axon already had [{bname}]')
if core:
pnode = (('file:bytes', f'sha256:{sha256}'), {
'props': {
'md5': fhashes.get('md5'),
'sha1': fhashes.get('sha1'),
'sha256': fhashes.get('sha256'),
'size': hset.size,
'name': bname,
},
'tags': tags,
})
node = list(core.addNodes([pnode]))[0]
iden = node[0][1]
size = node[1]['props']['size']
name = node[1]['props']['name']
mesg = f'file: {bname} ({size}) added to core ({iden}) as {name}'
outp.printf(mesg)
s_glob.sync(axon.fini())
if core:
s_glob.sync(core.fini())
return 0
def test_common_file_helpers(self):
# genfile
with self.getTestDir() as testdir:
fd = s_common.genfile(testdir, 'woot', 'foo.bin')
fd.write(b'genfile_test')
fd.close()
with open(os.path.join(testdir, 'woot', 'foo.bin'), 'rb') as fd:
buf = fd.read()
self.eq(buf, b'genfile_test')
# reqpath
with self.getTestDir() as testdir:
with s_common.genfile(testdir, 'test.txt') as fd:
fd.write(b'')
self.eq(os.path.join(testdir, 'test.txt'),
s_common.reqpath(testdir, 'test.txt'))
self.raises(s_exc.NoSuchFile, s_common.reqpath, testdir, 'newp')
# reqfile
with self.getTestDir() as testdir:
with s_common.genfile(testdir, 'test.txt') as fd:
fd.write(b'reqfile_test')
fd = s_common.reqfile(testdir, 'test.txt')
buf = fd.read()
self.eq(buf, b'reqfile_test')
fd.close()
self.raises(s_exc.NoSuchFile, s_common.reqfile, testdir, 'newp')
# getfile
with self.getTestDir() as testdir:
with s_common.genfile(testdir, 'test.txt') as fd:
fd.write(b'getfile_test')
fd = s_common.getfile(testdir, 'test.txt')
buf = fd.read()
self.eq(buf, b'getfile_test')
fd.close()
self.none(s_common.getfile(testdir, 'newp'))
# getbytes
with self.getTestDir() as testdir:
with s_common.genfile(testdir, 'test.txt') as fd:
fd.write(b'getbytes_test')
buf = s_common.getbytes(testdir, 'test.txt')
self.eq(buf, b'getbytes_test')
self.none(s_common.getbytes(testdir, 'newp'))
# reqbytes
with self.getTestDir() as testdir:
with s_common.genfile(testdir, 'test.txt') as fd:
fd.write(b'reqbytes_test')
buf = s_common.reqbytes(testdir, 'test.txt')
self.eq(buf, b'reqbytes_test')
self.raises(s_exc.NoSuchFile, s_common.reqbytes, testdir, 'newp')
# listdir
with self.getTestDir() as dirn:
path = os.path.join(dirn, 'woot.txt')
with open(path, 'wb') as fd:
fd.write(b'woot')
os.makedirs(os.path.join(dirn, 'nest'))
with open(os.path.join(dirn, 'nest', 'nope.txt'), 'wb') as fd:
fd.write(b'nope')
retn = tuple(s_common.listdir(dirn))
self.len(2, retn)
retn = tuple(s_common.listdir(dirn, glob='*.txt'))
self.eq(retn, ((path, )))
def loadPkgProto(path, opticdir=None, no_docs=False):
full = s_common.genpath(path)
pkgdef = s_common.yamlload(full)
if isinstance(pkgdef['version'], str):
pkgdef['version'] = chopSemVer(pkgdef['version'])
protodir = os.path.dirname(full)
pkgname = pkgdef.get('name')
logodef = pkgdef.get('logo')
if logodef is not None:
path = logodef.pop('path', None)
if path is not None:
with s_common.reqfile(protodir, path) as fd:
logodef['file'] = base64.b64encode(fd.read()).decode()
if logodef.get('mime') is None:
mesg = 'Mime type must be specified for logo file.'
raise s_exc.BadPkgDef(mesg=mesg)
if logodef.get('file') is None:
mesg = 'Logo def must contain path or file.'
raise s_exc.BadPkgDef(mesg=mesg)
for docdef in pkgdef.get('docs', ()):
if docdef.get('title') is None:
mesg = 'Each entry in docs must have a title.'
raise s_exc.BadPkgDef(mesg=mesg)
if no_docs:
docdef['content'] = ''
continue
path = docdef.pop('path', None)
if path is not None:
with s_common.reqfile(protodir, path) as fd:
docdef['content'] = fd.read().decode()
if docdef.get('content') is None:
mesg = 'Docs entry has no path or content.'
raise s_exc.BadPkgDef(mesg=mesg)
for mod in pkgdef.get('modules', ()):
name = mod.get('name')
with s_common.genfile(protodir, 'storm', 'modules', name) as fd:
mod['storm'] = fd.read().decode()
for extmod in pkgdef.get('external_modules', ()):
fpth = extmod.get('file_path')
if fpth is not None:
extmod['storm'] = getStormStr(fpth)
else:
path = extmod.get('package_path')
extpkg = s_dyndeps.tryDynMod(extmod.get('package'))
extmod['storm'] = extpkg.getAssetStr(path)
extname = extmod.get('name')
extmod['name'] = f'{pkgname}.{extname}'
pkgdef.setdefault('modules', [])
pkgdef['modules'].append(extmod)
pkgdef.pop('external_modules', None)
for cmd in pkgdef.get('commands', ()):
name = cmd.get('name')
with s_common.genfile(protodir, 'storm', 'commands', name) as fd:
cmd['storm'] = fd.read().decode()
if opticdir is None:
opticdir = s_common.genpath(protodir, 'optic')
if os.path.isdir(opticdir):
pkgdef.setdefault('optic', {})
pkgdef['optic'].setdefault('files', {})
loadOpticFiles(pkgdef, opticdir)
return pkgdef
def test_common_file_helpers(self):
# genfile
with self.getTestDir() as testdir:
fd = s_common.genfile(testdir, 'woot', 'foo.bin')
fd.write(b'genfile_test')
fd.close()
with open(os.path.join(testdir, 'woot', 'foo.bin'), 'rb') as fd:
buf = fd.read()
self.eq(buf, b'genfile_test')
# reqpath
with self.getTestDir() as testdir:
with s_common.genfile(testdir, 'test.txt') as fd:
fd.write(b'')
self.eq(os.path.join(testdir, 'test.txt'),
s_common.reqpath(testdir, 'test.txt'))
self.raises(s_exc.NoSuchFile, s_common.reqpath, testdir, 'newp')
# reqfile
with self.getTestDir() as testdir:
with s_common.genfile(testdir, 'test.txt') as fd:
fd.write(b'reqfile_test')
fd = s_common.reqfile(testdir, 'test.txt')
buf = fd.read()
self.eq(buf, b'reqfile_test')
fd.close()
self.raises(s_exc.NoSuchFile, s_common.reqfile, testdir, 'newp')
# getfile
with self.getTestDir() as testdir:
with s_common.genfile(testdir, 'test.txt') as fd:
fd.write(b'getfile_test')
fd = s_common.getfile(testdir, 'test.txt')
buf = fd.read()
self.eq(buf, b'getfile_test')
fd.close()
self.none(s_common.getfile(testdir, 'newp'))
# getbytes
with self.getTestDir() as testdir:
with s_common.genfile(testdir, 'test.txt') as fd:
fd.write(b'getbytes_test')
buf = s_common.getbytes(testdir, 'test.txt')
self.eq(buf, b'getbytes_test')
self.none(s_common.getbytes(testdir, 'newp'))
# reqbytes
with self.getTestDir() as testdir:
with s_common.genfile(testdir, 'test.txt') as fd:
fd.write(b'reqbytes_test')
buf = s_common.reqbytes(testdir, 'test.txt')
self.eq(buf, b'reqbytes_test')
self.raises(s_exc.NoSuchFile, s_common.reqbytes, testdir, 'newp')
# listdir
with self.getTestDir() as dirn:
path = os.path.join(dirn, 'woot.txt')
with open(path, 'wb') as fd:
fd.write(b'woot')
os.makedirs(os.path.join(dirn, 'nest'))
with open(os.path.join(dirn, 'nest', 'nope.txt'), 'wb') as fd:
fd.write(b'nope')
retn = tuple(s_common.listdir(dirn))
self.len(2, retn)
retn = tuple(s_common.listdir(dirn, glob='*.txt'))
self.eq(retn, ((path, )))
# getDirSize: check against du
real, appr = s_common.getDirSize(dirn)
durealstr = subprocess.check_output(['du', '-B', '1', '-s', dirn])
dureal = int(durealstr.split()[0])
duapprstr = subprocess.check_output(['du', '-bs', dirn])
duappr = int(duapprstr.split()[0])
self.eq(dureal, real)
self.eq(duappr, appr)
def test_common_file_helpers(self):
# genfile
with self.getTestDir() as testdir:
fd = s_common.genfile(testdir, 'woot', 'foo.bin')
fd.write(b'genfile_test')
fd.close()
with open(os.path.join(testdir, 'woot', 'foo.bin'), 'rb') as fd:
buf = fd.read()
self.eq(buf, b'genfile_test')
# reqpath
with self.getTestDir() as testdir:
with s_common.genfile(testdir, 'test.txt') as fd:
fd.write(b'')
self.eq(os.path.join(testdir, 'test.txt'), s_common.reqpath(testdir, 'test.txt'))
self.raises(s_exc.NoSuchFile, s_common.reqpath, testdir, 'newp')
# reqfile
with self.getTestDir() as testdir:
with s_common.genfile(testdir, 'test.txt') as fd:
fd.write(b'reqfile_test')
fd = s_common.reqfile(testdir, 'test.txt')
buf = fd.read()
self.eq(buf, b'reqfile_test')
fd.close()
self.raises(s_exc.NoSuchFile, s_common.reqfile, testdir, 'newp')
# getfile
with self.getTestDir() as testdir:
with s_common.genfile(testdir, 'test.txt') as fd:
fd.write(b'getfile_test')
fd = s_common.getfile(testdir, 'test.txt')
buf = fd.read()
self.eq(buf, b'getfile_test')
fd.close()
self.none(s_common.getfile(testdir, 'newp'))
# getbytes
with self.getTestDir() as testdir:
with s_common.genfile(testdir, 'test.txt') as fd:
fd.write(b'getbytes_test')
buf = s_common.getbytes(testdir, 'test.txt')
self.eq(buf, b'getbytes_test')
self.none(s_common.getbytes(testdir, 'newp'))
# reqbytes
with self.getTestDir() as testdir:
with s_common.genfile(testdir, 'test.txt') as fd:
fd.write(b'reqbytes_test')
buf = s_common.reqbytes(testdir, 'test.txt')
self.eq(buf, b'reqbytes_test')
self.raises(s_exc.NoSuchFile, s_common.reqbytes, testdir, 'newp')
# listdir
with self.getTestDir() as dirn:
path = os.path.join(dirn, 'woot.txt')
with open(path, 'wb') as fd:
fd.write(b'woot')
os.makedirs(os.path.join(dirn, 'nest'))
with open(os.path.join(dirn, 'nest', 'nope.txt'), 'wb') as fd:
fd.write(b'nope')
retn = tuple(s_common.listdir(dirn))
self.len(2, retn)
retn = tuple(s_common.listdir(dirn, glob='*.txt'))
self.eq(retn, ((path,)))
def loadPkgProto(path, opticdir=None, no_docs=False, readonly=False):
'''
Get a Storm Package definition from disk.
Args:
fp (str): Path to the package .yaml file on disk.
opticdir (str): Path to optional Optic module code to add to the Storm Package.
no_docs (bool): If true, omit inline documentation content if it is not present on disk.
readonly (bool): If set, open files in read-only mode. If files are missing, that will raise a NoSuchFile
exception.
Returns:
dict: A Storm package definition.
'''
full = s_common.genpath(path)
pkgdef = s_common.yamlload(full)
if isinstance(pkgdef['version'], str):
pkgdef['version'] = chopSemVer(pkgdef['version'])
protodir = os.path.dirname(full)
pkgname = pkgdef.get('name')
logodef = pkgdef.get('logo')
if logodef is not None:
path = logodef.pop('path', None)
if path is not None:
with s_common.reqfile(protodir, path) as fd:
logodef['file'] = base64.b64encode(fd.read()).decode()
if logodef.get('mime') is None:
mesg = 'Mime type must be specified for logo file.'
raise s_exc.BadPkgDef(mesg=mesg)
if logodef.get('file') is None:
mesg = 'Logo def must contain path or file.'
raise s_exc.BadPkgDef(mesg=mesg)
for docdef in pkgdef.get('docs', ()):
if docdef.get('title') is None:
mesg = 'Each entry in docs must have a title.'
raise s_exc.BadPkgDef(mesg=mesg)
if no_docs:
docdef['content'] = ''
continue
path = docdef.pop('path', None)
if path is not None:
with s_common.reqfile(protodir, path) as fd:
docdef['content'] = fd.read().decode()
if docdef.get('content') is None:
mesg = 'Docs entry has no path or content.'
raise s_exc.BadPkgDef(mesg=mesg)
for mod in pkgdef.get('modules', ()):
name = mod.get('name')
mod_path = s_common.genpath(protodir, 'storm', 'modules', name)
if readonly:
mod['storm'] = getStormStr(mod_path)
else:
with s_common.genfile(mod_path) as fd:
mod['storm'] = fd.read().decode()
for extmod in pkgdef.get('external_modules', ()):
fpth = extmod.get('file_path')
if fpth is not None:
extmod['storm'] = getStormStr(fpth)
else:
path = extmod.get('package_path')
extpkg = s_dyndeps.tryDynMod(extmod.get('package'))
extmod['storm'] = extpkg.getAssetStr(path)
extname = extmod.get('name')
extmod['name'] = f'{pkgname}.{extname}'
pkgdef.setdefault('modules', [])
pkgdef['modules'].append(extmod)
pkgdef.pop('external_modules', None)
for cmd in pkgdef.get('commands', ()):
name = cmd.get('name')
cmd_path = s_common.genpath(protodir, 'storm', 'commands', name)
if readonly:
cmd['storm'] = getStormStr(cmd_path)
else:
with s_common.genfile(cmd_path) as fd:
cmd['storm'] = fd.read().decode()
wflowdir = s_common.genpath(protodir, 'workflows')
if os.path.isdir(wflowdir):
pkgdef.setdefault('optic', {})
pkgdef['optic'].setdefault('workflows', {})
loadOpticWorkflows(pkgdef, wflowdir)
if opticdir is None:
opticdir = s_common.genpath(protodir, 'optic')
if os.path.isdir(opticdir):
pkgdef.setdefault('optic', {})
pkgdef['optic'].setdefault('files', {})
loadOpticFiles(pkgdef, opticdir)
return pkgdef
