def test_certdir_importfile(self): with self.getCertDir() as cdir: # type: s_certdir.CertDir with self.getTestDir() as testpath: # File doesn't exist fpath = s_common.genpath(testpath, 'not_real.crt') self.raises(s_exc.NoSuchFile, cdir.importFile, fpath, 'cas') # File has unsupported extension fpath = s_common.genpath(testpath, 'coolpic.bmp') with s_common.genfile(fpath) as fd: self.raises(s_exc.BadFileExt, cdir.importFile, fpath, 'cas') tests = ( ('cas', 'coolca.crt'), ('cas', 'coolca.key'), ('hosts', 'coolhost.crt'), ('hosts', 'coolhost.key'), ('users', 'cooluser.crt'), ('users', 'cooluser.key'), ('users', 'cooluser.p12'), ) for ftype, fname in tests: srcpath = s_common.genpath(testpath, fname) dstpath = s_common.genpath(cdir.certdirs[0], ftype, fname) with s_common.genfile(srcpath) as fd: fd.write(b'arbitrary data') fd.seek(0) # Make sure the file is not there self.raises(s_exc.NoSuchFile, s_common.reqfile, dstpath) # Import it and make sure it exists self.none(cdir.importFile(srcpath, ftype)) with s_common.reqfile(dstpath) as dstfd: self.eq(dstfd.read(), b'arbitrary data') # Make sure it can't be overwritten self.raises(s_exc.FileExists, cdir.importFile, srcpath, ftype)
def loadfile(*paths): ''' Load a json ingest def from file and construct an Ingest class. This routine is useful because it implements the convention for adding runtime info to the ingest json to facilitate path relative file opening etc... ''' path = s_common.genpath(*paths) # FIXME universal open with s_common.reqfile(path) as fd: jsfo = json.loads(fd.read().decode('utf8')) gest = Ingest(jsfo) gest.set('basedir', os.path.dirname(path)) return gest
def test_certdir_importfile(self): with self.getCertDir() as cdir: # type: s_certdir.CertDir with self.getTestDir() as testpath: # File doesn't exist fpath = s_common.genpath(testpath, 'not_real.crt') self.raises(s_exc.NoSuchFile, cdir.importFile, fpath, 'cas') # File has unsupported extension fpath = s_common.genpath(testpath, 'coolpic.bmp') with s_common.genfile(fpath) as fd: self.raises(s_exc.BadFileExt, cdir.importFile, fpath, 'cas') tests = ( ('cas', 'coolca.crt'), ('cas', 'coolca.key'), ('hosts', 'coolhost.crt'), ('hosts', 'coolhost.key'), ('users', 'cooluser.crt'), ('users', 'cooluser.key'), ('users', 'cooluser.p12'), ) for ftype, fname in tests: srcpath = s_common.genpath(testpath, fname) dstpath = s_common.genpath(cdir.path, ftype, fname) with s_common.genfile(srcpath) as fd: fd.write(b'arbitrary data') fd.seek(0) # Make sure the file is not there self.raises(s_exc.NoSuchFile, s_common.reqfile, dstpath) # Import it and make sure it exists self.none(cdir.importFile(srcpath, ftype)) with s_common.reqfile(dstpath) as dstfd: self.eq(dstfd.read(), b'arbitrary data') # Make sure it can't be overwritten self.raises(s_exc.FileExists, cdir.importFile, srcpath, ftype)
async def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() path = s_common.getSynPath('telepath.yaml') telefini = await s_telepath.loadTeleEnv(path) pars = makeargparser() opts = pars.parse_args(argv) axon = await s_telepath.openurl(opts.axon) core = None if opts.cortex: core = await s_telepath.openurl(opts.cortex) tags = set() if opts.tags: for tag in opts.tags.split(','): tags.add(tag) tags = tuple(tags) if tags: outp.printf(f'adding tags: {tags}') filepaths = set() for item in opts.filenames: paths = glob.glob(item, recursive=opts.recursive) if not paths: outp.printf(f'filepath does not contain any files: {item}') continue filepaths.update([path for path in paths if os.path.isfile(path)]) for path in filepaths: bname = os.path.basename(path) hset = s_hashset.HashSet() with s_common.reqfile(path) as fd: hset.eatfd(fd) fhashes = {htyp: hasher.hexdigest() for htyp, hasher in hset.hashes} sha256 = fhashes.get('sha256') bsha256 = s_common.uhex(sha256) if not await axon.has(bsha256): async with await axon.upload() as upfd: with s_common.genfile(path) as fd: for byts in s_common.iterfd(fd): await upfd.write(byts) size, hashval = await upfd.save() if hashval != bsha256: # pragma: no cover raise s_exc.SynErr(mesg='hashes do not match', ehash=s_common.ehex(hashval), ahash=hashval) outp.printf(f'Uploaded [{bname}] to axon') else: outp.printf(f'Axon already had [{bname}]') if core: opts = { 'vars': { 'md5': fhashes.get('md5'), 'sha1': fhashes.get('sha1'), 'sha256': fhashes.get('sha256'), 'size': hset.size, 'name': bname, 'tags': tags, } } q = '[file:bytes=$sha256 :md5=$md5 :sha1=$sha1 :size=$size :name=$name] ' \ '{ for $tag in $tags { [+#$tag] } }' msgs = await core.storm(q, opts=opts).list() node = [m[1] for m in msgs if m[0] == 'node'][0] iden = node[0][1] size = node[1]['props']['size'] name = node[1]['props']['name'] mesg = f'file: {bname} ({size}) added to core ({iden}) as {name}' outp.printf(mesg) await axon.fini() if core: await core.fini() if telefini: # pragma: no cover await telefini() return 0
def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() pars = makeargparser() opts = pars.parse_args(argv) axon = s_telepath.openurl(opts.axon) core = None if opts.cortex: core = s_telepath.openurl(opts.cortex) tags = {} if opts.tags: for tag in opts.tags.split(','): tags[tag] = (None, None) if tags: outp.printf('adding tags: %r' % (list(tags.keys()))) filepaths = set() for item in opts.filenames: paths = glob.glob(item, recursive=opts.recursive) if not paths: outp.printf(f'filepath does not contain any files: {item}') continue filepaths.update([path for path in paths if os.path.isfile(path)]) for path in filepaths: bname = os.path.basename(path) hset = s_hashset.HashSet() with s_common.reqfile(path) as fd: hset.eatfd(fd) fhashes = {htyp: hasher.hexdigest() for htyp, hasher in hset.hashes} sha256 = fhashes.get('sha256') bsha256 = s_common.uhex(sha256) if not axon.has(bsha256): with axon.upload() as upfd: with s_common.genfile(path) as fd: for byts in s_common.iterfd(fd): upfd.write(byts) size, hashval = upfd.save() if hashval != bsha256: # pragma: no cover raise s_exc.SynErr(mesg='hashes do not match', ehash=s_common.ehex(hashval), ahash=hashval) outp.printf(f'Uploaded [{bname}] to axon') else: outp.printf(f'Axon already had [{bname}]') if core: pnode = (('file:bytes', f'sha256:{sha256}'), { 'props': { 'md5': fhashes.get('md5'), 'sha1': fhashes.get('sha1'), 'sha256': fhashes.get('sha256'), 'size': hset.size, 'name': bname, }, 'tags': tags, }) node = list(core.addNodes([pnode]))[0] iden = node[0][1] size = node[1]['props']['size'] name = node[1]['props']['name'] mesg = f'file: {bname} ({size}) added to core ({iden}) as {name}' outp.printf(mesg) s_glob.sync(axon.fini()) if core: s_glob.sync(core.fini()) return 0
def test_common_file_helpers(self): # genfile with self.getTestDir() as testdir: fd = s_common.genfile(testdir, 'woot', 'foo.bin') fd.write(b'genfile_test') fd.close() with open(os.path.join(testdir, 'woot', 'foo.bin'), 'rb') as fd: buf = fd.read() self.eq(buf, b'genfile_test') # reqpath with self.getTestDir() as testdir: with s_common.genfile(testdir, 'test.txt') as fd: fd.write(b'') self.eq(os.path.join(testdir, 'test.txt'), s_common.reqpath(testdir, 'test.txt')) self.raises(s_exc.NoSuchFile, s_common.reqpath, testdir, 'newp') # reqfile with self.getTestDir() as testdir: with s_common.genfile(testdir, 'test.txt') as fd: fd.write(b'reqfile_test') fd = s_common.reqfile(testdir, 'test.txt') buf = fd.read() self.eq(buf, b'reqfile_test') fd.close() self.raises(s_exc.NoSuchFile, s_common.reqfile, testdir, 'newp') # getfile with self.getTestDir() as testdir: with s_common.genfile(testdir, 'test.txt') as fd: fd.write(b'getfile_test') fd = s_common.getfile(testdir, 'test.txt') buf = fd.read() self.eq(buf, b'getfile_test') fd.close() self.none(s_common.getfile(testdir, 'newp')) # getbytes with self.getTestDir() as testdir: with s_common.genfile(testdir, 'test.txt') as fd: fd.write(b'getbytes_test') buf = s_common.getbytes(testdir, 'test.txt') self.eq(buf, b'getbytes_test') self.none(s_common.getbytes(testdir, 'newp')) # reqbytes with self.getTestDir() as testdir: with s_common.genfile(testdir, 'test.txt') as fd: fd.write(b'reqbytes_test') buf = s_common.reqbytes(testdir, 'test.txt') self.eq(buf, b'reqbytes_test') self.raises(s_exc.NoSuchFile, s_common.reqbytes, testdir, 'newp') # listdir with self.getTestDir() as dirn: path = os.path.join(dirn, 'woot.txt') with open(path, 'wb') as fd: fd.write(b'woot') os.makedirs(os.path.join(dirn, 'nest')) with open(os.path.join(dirn, 'nest', 'nope.txt'), 'wb') as fd: fd.write(b'nope') retn = tuple(s_common.listdir(dirn)) self.len(2, retn) retn = tuple(s_common.listdir(dirn, glob='*.txt')) self.eq(retn, ((path, )))
def loadPkgProto(path, opticdir=None, no_docs=False): full = s_common.genpath(path) pkgdef = s_common.yamlload(full) if isinstance(pkgdef['version'], str): pkgdef['version'] = chopSemVer(pkgdef['version']) protodir = os.path.dirname(full) pkgname = pkgdef.get('name') logodef = pkgdef.get('logo') if logodef is not None: path = logodef.pop('path', None) if path is not None: with s_common.reqfile(protodir, path) as fd: logodef['file'] = base64.b64encode(fd.read()).decode() if logodef.get('mime') is None: mesg = 'Mime type must be specified for logo file.' raise s_exc.BadPkgDef(mesg=mesg) if logodef.get('file') is None: mesg = 'Logo def must contain path or file.' raise s_exc.BadPkgDef(mesg=mesg) for docdef in pkgdef.get('docs', ()): if docdef.get('title') is None: mesg = 'Each entry in docs must have a title.' raise s_exc.BadPkgDef(mesg=mesg) if no_docs: docdef['content'] = '' continue path = docdef.pop('path', None) if path is not None: with s_common.reqfile(protodir, path) as fd: docdef['content'] = fd.read().decode() if docdef.get('content') is None: mesg = 'Docs entry has no path or content.' raise s_exc.BadPkgDef(mesg=mesg) for mod in pkgdef.get('modules', ()): name = mod.get('name') with s_common.genfile(protodir, 'storm', 'modules', name) as fd: mod['storm'] = fd.read().decode() for extmod in pkgdef.get('external_modules', ()): fpth = extmod.get('file_path') if fpth is not None: extmod['storm'] = getStormStr(fpth) else: path = extmod.get('package_path') extpkg = s_dyndeps.tryDynMod(extmod.get('package')) extmod['storm'] = extpkg.getAssetStr(path) extname = extmod.get('name') extmod['name'] = f'{pkgname}.{extname}' pkgdef.setdefault('modules', []) pkgdef['modules'].append(extmod) pkgdef.pop('external_modules', None) for cmd in pkgdef.get('commands', ()): name = cmd.get('name') with s_common.genfile(protodir, 'storm', 'commands', name) as fd: cmd['storm'] = fd.read().decode() if opticdir is None: opticdir = s_common.genpath(protodir, 'optic') if os.path.isdir(opticdir): pkgdef.setdefault('optic', {}) pkgdef['optic'].setdefault('files', {}) loadOpticFiles(pkgdef, opticdir) return pkgdef
def test_common_file_helpers(self): # genfile with self.getTestDir() as testdir: fd = s_common.genfile(testdir, 'woot', 'foo.bin') fd.write(b'genfile_test') fd.close() with open(os.path.join(testdir, 'woot', 'foo.bin'), 'rb') as fd: buf = fd.read() self.eq(buf, b'genfile_test') # reqpath with self.getTestDir() as testdir: with s_common.genfile(testdir, 'test.txt') as fd: fd.write(b'') self.eq(os.path.join(testdir, 'test.txt'), s_common.reqpath(testdir, 'test.txt')) self.raises(s_exc.NoSuchFile, s_common.reqpath, testdir, 'newp') # reqfile with self.getTestDir() as testdir: with s_common.genfile(testdir, 'test.txt') as fd: fd.write(b'reqfile_test') fd = s_common.reqfile(testdir, 'test.txt') buf = fd.read() self.eq(buf, b'reqfile_test') fd.close() self.raises(s_exc.NoSuchFile, s_common.reqfile, testdir, 'newp') # getfile with self.getTestDir() as testdir: with s_common.genfile(testdir, 'test.txt') as fd: fd.write(b'getfile_test') fd = s_common.getfile(testdir, 'test.txt') buf = fd.read() self.eq(buf, b'getfile_test') fd.close() self.none(s_common.getfile(testdir, 'newp')) # getbytes with self.getTestDir() as testdir: with s_common.genfile(testdir, 'test.txt') as fd: fd.write(b'getbytes_test') buf = s_common.getbytes(testdir, 'test.txt') self.eq(buf, b'getbytes_test') self.none(s_common.getbytes(testdir, 'newp')) # reqbytes with self.getTestDir() as testdir: with s_common.genfile(testdir, 'test.txt') as fd: fd.write(b'reqbytes_test') buf = s_common.reqbytes(testdir, 'test.txt') self.eq(buf, b'reqbytes_test') self.raises(s_exc.NoSuchFile, s_common.reqbytes, testdir, 'newp') # listdir with self.getTestDir() as dirn: path = os.path.join(dirn, 'woot.txt') with open(path, 'wb') as fd: fd.write(b'woot') os.makedirs(os.path.join(dirn, 'nest')) with open(os.path.join(dirn, 'nest', 'nope.txt'), 'wb') as fd: fd.write(b'nope') retn = tuple(s_common.listdir(dirn)) self.len(2, retn) retn = tuple(s_common.listdir(dirn, glob='*.txt')) self.eq(retn, ((path, ))) # getDirSize: check against du real, appr = s_common.getDirSize(dirn) durealstr = subprocess.check_output(['du', '-B', '1', '-s', dirn]) dureal = int(durealstr.split()[0]) duapprstr = subprocess.check_output(['du', '-bs', dirn]) duappr = int(duapprstr.split()[0]) self.eq(dureal, real) self.eq(duappr, appr)
def test_common_file_helpers(self): # genfile with self.getTestDir() as testdir: fd = s_common.genfile(testdir, 'woot', 'foo.bin') fd.write(b'genfile_test') fd.close() with open(os.path.join(testdir, 'woot', 'foo.bin'), 'rb') as fd: buf = fd.read() self.eq(buf, b'genfile_test') # reqpath with self.getTestDir() as testdir: with s_common.genfile(testdir, 'test.txt') as fd: fd.write(b'') self.eq(os.path.join(testdir, 'test.txt'), s_common.reqpath(testdir, 'test.txt')) self.raises(s_exc.NoSuchFile, s_common.reqpath, testdir, 'newp') # reqfile with self.getTestDir() as testdir: with s_common.genfile(testdir, 'test.txt') as fd: fd.write(b'reqfile_test') fd = s_common.reqfile(testdir, 'test.txt') buf = fd.read() self.eq(buf, b'reqfile_test') fd.close() self.raises(s_exc.NoSuchFile, s_common.reqfile, testdir, 'newp') # getfile with self.getTestDir() as testdir: with s_common.genfile(testdir, 'test.txt') as fd: fd.write(b'getfile_test') fd = s_common.getfile(testdir, 'test.txt') buf = fd.read() self.eq(buf, b'getfile_test') fd.close() self.none(s_common.getfile(testdir, 'newp')) # getbytes with self.getTestDir() as testdir: with s_common.genfile(testdir, 'test.txt') as fd: fd.write(b'getbytes_test') buf = s_common.getbytes(testdir, 'test.txt') self.eq(buf, b'getbytes_test') self.none(s_common.getbytes(testdir, 'newp')) # reqbytes with self.getTestDir() as testdir: with s_common.genfile(testdir, 'test.txt') as fd: fd.write(b'reqbytes_test') buf = s_common.reqbytes(testdir, 'test.txt') self.eq(buf, b'reqbytes_test') self.raises(s_exc.NoSuchFile, s_common.reqbytes, testdir, 'newp') # listdir with self.getTestDir() as dirn: path = os.path.join(dirn, 'woot.txt') with open(path, 'wb') as fd: fd.write(b'woot') os.makedirs(os.path.join(dirn, 'nest')) with open(os.path.join(dirn, 'nest', 'nope.txt'), 'wb') as fd: fd.write(b'nope') retn = tuple(s_common.listdir(dirn)) self.len(2, retn) retn = tuple(s_common.listdir(dirn, glob='*.txt')) self.eq(retn, ((path,)))
def loadPkgProto(path, opticdir=None, no_docs=False, readonly=False): ''' Get a Storm Package definition from disk. Args: fp (str): Path to the package .yaml file on disk. opticdir (str): Path to optional Optic module code to add to the Storm Package. no_docs (bool): If true, omit inline documentation content if it is not present on disk. readonly (bool): If set, open files in read-only mode. If files are missing, that will raise a NoSuchFile exception. Returns: dict: A Storm package definition. ''' full = s_common.genpath(path) pkgdef = s_common.yamlload(full) if isinstance(pkgdef['version'], str): pkgdef['version'] = chopSemVer(pkgdef['version']) protodir = os.path.dirname(full) pkgname = pkgdef.get('name') logodef = pkgdef.get('logo') if logodef is not None: path = logodef.pop('path', None) if path is not None: with s_common.reqfile(protodir, path) as fd: logodef['file'] = base64.b64encode(fd.read()).decode() if logodef.get('mime') is None: mesg = 'Mime type must be specified for logo file.' raise s_exc.BadPkgDef(mesg=mesg) if logodef.get('file') is None: mesg = 'Logo def must contain path or file.' raise s_exc.BadPkgDef(mesg=mesg) for docdef in pkgdef.get('docs', ()): if docdef.get('title') is None: mesg = 'Each entry in docs must have a title.' raise s_exc.BadPkgDef(mesg=mesg) if no_docs: docdef['content'] = '' continue path = docdef.pop('path', None) if path is not None: with s_common.reqfile(protodir, path) as fd: docdef['content'] = fd.read().decode() if docdef.get('content') is None: mesg = 'Docs entry has no path or content.' raise s_exc.BadPkgDef(mesg=mesg) for mod in pkgdef.get('modules', ()): name = mod.get('name') mod_path = s_common.genpath(protodir, 'storm', 'modules', name) if readonly: mod['storm'] = getStormStr(mod_path) else: with s_common.genfile(mod_path) as fd: mod['storm'] = fd.read().decode() for extmod in pkgdef.get('external_modules', ()): fpth = extmod.get('file_path') if fpth is not None: extmod['storm'] = getStormStr(fpth) else: path = extmod.get('package_path') extpkg = s_dyndeps.tryDynMod(extmod.get('package')) extmod['storm'] = extpkg.getAssetStr(path) extname = extmod.get('name') extmod['name'] = f'{pkgname}.{extname}' pkgdef.setdefault('modules', []) pkgdef['modules'].append(extmod) pkgdef.pop('external_modules', None) for cmd in pkgdef.get('commands', ()): name = cmd.get('name') cmd_path = s_common.genpath(protodir, 'storm', 'commands', name) if readonly: cmd['storm'] = getStormStr(cmd_path) else: with s_common.genfile(cmd_path) as fd: cmd['storm'] = fd.read().decode() wflowdir = s_common.genpath(protodir, 'workflows') if os.path.isdir(wflowdir): pkgdef.setdefault('optic', {}) pkgdef['optic'].setdefault('workflows', {}) loadOpticWorkflows(pkgdef, wflowdir) if opticdir is None: opticdir = s_common.genpath(protodir, 'optic') if os.path.isdir(opticdir): pkgdef.setdefault('optic', {}) pkgdef['optic'].setdefault('files', {}) loadOpticFiles(pkgdef, opticdir) return pkgdef