def initSslCtx(self, certpath, keypath):
sslctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
if not os.path.isfile(keypath):
raise s_exc.NoSuchFile(name=keypath)
if not os.path.isfile(certpath):
raise s_exc.NoSuchFile(name=certpath)
sslctx.load_cert_chain(certpath, keypath)
return sslctx
async def hashset(self, sha256):
'''
Calculate additional hashes for a file in the Axon.
Args:
sha256 (bytes): The sha256 hash of the file in bytes.
Returns:
dict: A dictionary containing hashes of the file.
'''
if not await self.has(sha256):
raise s_exc.NoSuchFile(
mesg='Axon does not contain the requested file.',
sha256=s_common.ehex(sha256))
fhash = s_common.ehex(sha256)
logger.debug(f'Getting blob [{fhash}].',
extra=await self.getLogExtra(sha256=fhash))
hashset = s_hashset.HashSet()
async for byts in self._get(sha256):
hashset.update(byts)
await asyncio.sleep(0)
return dict([(n, s_common.ehex(h)) for (n, h) in hashset.digests()])
async def get(self, sha256):
'''
Get bytes of a file.
Args:
sha256 (bytes): The sha256 hash of the file in bytes.
Examples:
Get the bytes from an Axon and process them::
buf = b''
async for bytz in axon.get(sha256):
buf =+ bytz
await dostuff(buf)
Yields:
bytes: Chunks of the file bytes.
Raises:
synapse.exc.NoSuchFile: If the file does not exist.
'''
if not await self.has(sha256):
raise s_exc.NoSuchFile(
mesg='Axon does not contain the requested file.',
sha256=s_common.ehex(sha256))
fhash = s_common.ehex(sha256)
logger.debug(f'Getting blob [{fhash}].',
extra=await self.getLogExtra(sha256=fhash))
async for byts in self._get(sha256):
yield byts
async def get(self, sha256):
if not await self.has(sha256):
raise s_exc.NoSuchFile(sha256=s_common.ehex(sha256))
for lkey, byts in self.blobslab.scanByPref(sha256, db=self.blobs):
yield byts
def getStormStr(fn):
if not os.path.isfile(fn):
raise s_exc.NoSuchFile(mesg='Storm file {} not found'.format(fn),
path=fn)
with open(fn, 'rb') as f:
return f.read().decode()
def getFileMappedRegion(filename):
'''
Return a tuple of address and length of a particular file memory mapped into this process
'''
# /proc/<pid>/maps has a bunch of entries that look like this:
# 7fb5195fc000-7fb519ffc000 r--s 00000000 fd:01 5245137 /tmp/foo.lmdb/data.mdb
filename = str(filename)
largest = None
with open(f'/proc/{os.getpid()}/maps') as maps:
for line in maps:
if len(line) < 50:
continue
if line.rstrip().endswith(filename):
addrs = line.split(' ', 1)[0]
start, end = addrs.split('-')
start_addr = int(start, 16)
end_addr = int(end, 16)
memlen = end_addr - start_addr
if largest is None or memlen > largest[1]:
largest = (start_addr, memlen)
if largest is None:
raise s_exc.NoSuchFile(
f'{filename} is not mapped into current process')
return largest
def _shimHttpCalls(self, vcr_kwargs):
path = self.ctx.get('mock-http-path')
if not vcr_kwargs:
vcr_kwargs = {}
if path:
path = os.path.abspath(path)
# try it as json first (since yaml can load json...). if it parses, we're old school
# if it doesn't, either it doesn't exist/we can't read it/we can't parse it.
# in any of those cases, default to using vcr
try:
with open(path, 'r') as fd:
byts = json.load(fd)
except (FileNotFoundError, json.decoder.JSONDecodeError):
byts = None
if not byts:
with vcr.use_cassette(os.path.abspath(path),
**vcr_kwargs) as cass:
yield cass
self.ctx.pop('mock-http-path', None)
else: # backwards compat
if not os.path.isfile(path):
raise s_exc.NoSuchFile(
mesg='Storm HTTP mock filepath does not exist',
path=path)
self.ctx['mock-http'] = byts
with mock.patch('synapse.lib.stormhttp.LibHttp._httpRequest',
new=self._mockHttp):
yield
else:
yield
async def get(self, sha256):
if not await self.has(sha256):
raise s_exc.NoSuchFile(
mesg='Axon does not contain the requested file.',
sha256=s_common.ehex(sha256))
async for byts in self._get(sha256):
yield byts
def reqpath(*paths):
'''
Return the absolute path of the joining of the arguments, raising an exception if a file doesn't exist at resulting
path
Args:
*paths ([str,...]): A list of path elements
'''
path = genpath(*paths)
if not os.path.isfile(path):
raise s_exc.NoSuchFile(mesg=f'No such path {path}', path=path)
return path
def genClientCert(self, name, outp=None):
'''
Generates a user PKCS #12 archive.
Please note that the resulting file will contain private key material.
Args:
name (str): The name of the user keypair.
outp (synapse.lib.output.Output): The output buffer.
Examples:
Make the PKC12 object for user "myuser":
myuserpkcs12 = cdir.genClientCert('myuser')
Returns:
OpenSSL.crypto.PKCS12: The PKCS #12 archive.
'''
ucert = self.getUserCert(name)
if not ucert:
raise s_exc.NoSuchFile(mesg='missing User cert', name=name)
capath = self._getCaPath(ucert)
cacert = self._loadCertPath(capath)
if not cacert:
raise s_exc.NoSuchFile(mesg='missing CA cert', path=capath)
ukey = self.getUserKey(name)
if not ukey:
raise s_exc.NoSuchFile(mesg='missing User private key', name=name)
ccert = crypto.PKCS12()
ccert.set_friendlyname(name.encode('utf-8'))
ccert.set_ca_certificates([cacert])
ccert.set_certificate(ucert)
ccert.set_privatekey(ukey)
crtpath = self._saveP12To(ccert, 'users', '%s.p12' % name)
if outp is not None:
outp.printf('client cert saved: %s' % (crtpath, ))
async def _handleStormPkg(self, text):
'''
Load a Storm package into the Cortex by path.
Args:
text (str): The path to a Storm package YAML file.
'''
if not os.path.isfile(text):
raise s_exc.NoSuchFile(
mesg='Storm Package filepath does not exist', path=text)
core = self._reqCore()
pkg = s_genpkg.loadPkgProto(text)
await core.addStormPkg(pkg)
async def loadDmonCell(self, name):
dirn = s_common.gendir(self.dirn, 'cells', name)
logger.info(f'loading cell from: {dirn}')
path = os.path.join(dirn, 'boot.yaml')
if not os.path.exists(path):
raise s_exc.NoSuchFile(name=path)
conf = self._loadYamlPath(path)
kind = conf.get('type')
cell = await s_cells.init(kind, dirn)
self.share(name, cell)
self.cells[name] = cell
def reqfile(*paths, **opts):
'''
Return a file at the path resulting from joining of the arguments, raising an exception if the file does not
exist.
Args:
*paths ([str,...]): A list of path elements
**opts: arguments as kwargs to io.open
Returns:
io.BufferedRandom: A file-object which can be read/written too.
'''
path = genpath(*paths)
if not os.path.isfile(path):
raise s_exc.NoSuchFile(mesg=f'No such file {path}', path=path)
opts.setdefault('mode', 'rb')
return io.open(path, **opts)
async def _handleStormMockHttp(self, text):
'''
Setup an HTTP mock file to be used with a later Storm command.
Response file format:
{
"code": int,
"body": {
"data": json or a json str
}
}
Args:
text (str): Path to a json file with the response.
'''
if not os.path.isfile(text):
raise s_exc.NoSuchFile(
mesg='Storm HTTP mock filepath does not exist', path=text)
self.context['mock-http'] = s_common.jsload(text)
async def get(self, hashval, proxykeeper=None):
'''
Yield bytes for the given SHA256.
Args:
hashval (str): The SHA256 hash bytes.
Yields:
bytes: Bytes of the file requested.
Raises:
RetnErr: If the file requested does not exist.
'''
if proxykeeper is None:
proxykeeper = self._proxykeeper
locs = await self.locs(hashval)
if not locs:
raise s_exc.NoSuchFile(f'{hashval} not present')
_, blobstor = await proxykeeper.randoproxy(locs)
# that await is due to the current async generator telepath asymmetry
async for bloc in await blobstor.get(hashval):
yield bloc
def importFile(self, path, mode, outp=None):
'''
Imports certs and keys into the Synapse cert directory
Args:
path (str): The path of the file to be imported.
mode (str): The certdir subdirectory to import the file into.
Examples:
Import CA certifciate 'mycoolca.crt' to the 'cas' directory.
certdir.importFile('mycoolca.crt', 'cas')
Notes:
importFile does not perform any validation on the files it imports.
Returns:
None
'''
if not os.path.isfile(path):
raise s_exc.NoSuchFile('File does not exist')
fname = os.path.split(path)[1]
parts = fname.rsplit('.', 1)
ext = parts[1] if len(parts) is 2 else None
if not ext or ext not in ('crt', 'key', 'p12'):
mesg = 'importFile only supports .crt, .key, .p12 extensions'
raise s_exc.BadFileExt(mesg=mesg, ext=ext)
newpath = s_common.genpath(self.certdir, mode, fname)
if os.path.isfile(newpath):
raise s_exc.FileExists('File already exists')
shutil.copy(path, newpath)
if outp is not None:
outp.printf('copied %s to %s' % (path, newpath))
def getStemCell(dirn):
if not os.path.isdir(dirn):
mesg = f'Directory {dirn} does not exist!'
raise s_exc.NoSuchDir(mesg=mesg)
ctorname = os.getenv('SYN_STEM_CELL_CTOR')
cellyaml = os.path.join(dirn, 'cell.yaml')
if os.path.isfile(cellyaml):
conf = s_common.yamlload(cellyaml)
ctorname = conf.get('cell:ctor', ctorname)
if ctorname is not None:
ctorname = ctorname.strip()
ctor = s_dyndeps.getDynLocal(ctorname)
if ctor is None:
raise s_exc.NoSuchCtor(mesg=f'Unable to resolve ctor [{ctorname}]',
ctor=ctorname)
return ctor
mesg = f'No such file: {cellyaml} and SYN_STEM_CELL_CTOR environmt variable is not set.'
raise s_exc.NoSuchFile(mesg=mesg, path=cellyaml)
def reqfile(*paths, **opts):
path = genpath(*paths)
if not os.path.isfile(path):
raise s_exc.NoSuchFile(path=path)
opts.setdefault('mode', 'rb')
return io.open(path, **opts)
def reqpath(*paths):
path = genpath(*paths)
if not os.path.isfile(path):
raise s_exc.NoSuchFile(name=path)
return path