def format(self, record: logging.LogRecord):
record.message = record.getMessage()
mesg = self.formatMessage(record)
ret = {
'message': mesg,
'logger': {
'name': record.name,
'process': record.processName,
'filename': record.filename,
'func': record.funcName,
},
'level': record.levelname,
'time': self.formatTime(record, self.datefmt),
}
if record.exc_info:
name, info = s_common.err(record.exc_info[1], fulltb=True)
# This is the actual exception name. The ename key is the function name.
info['errname'] = name
ret['err'] = info
# stuffing our extra into a single dictionary avoids a loop
# over record.__dict__ extracting fields which are not known
# attributes for each log record.
extras = record.__dict__.get('synapse')
if extras:
ret.update({k: v for k, v in extras.items() if k not in ret})
return json.dumps(ret, default=str)
async def runStorm():
cancelled = False
tick = s_common.now()
count = 0
try:
# Always start with an init message.
await chan.put(('init', {'tick': tick, 'text': text, 'task': synt.iden}))
# Try text parsing. If this fails, we won't be able to get a storm
# runtime in the snap, so catch and pass the `err` message
await self.core.getStormQuery(text, mode=mode)
shownode = (not show or 'node' in show)
async with await self.snap(user=user) as snap:
if not show:
snap.link(chan.put)
else:
[snap.on(n, chan.put) for n in show]
if shownode:
async for pode in snap.iterStormPodes(text, opts=opts, user=user):
await chan.put(('node', pode))
count += 1
else:
async for item in snap.storm(text, opts=opts, user=user):
count += 1
except s_stormctrl.StormExit:
pass
except asyncio.CancelledError:
logger.warning('Storm runtime cancelled.',
extra={'synapse': {'text': text, 'username': user.name, 'user': user.iden}})
cancelled = True
raise
except Exception as e:
logger.exception(f'Error during storm execution for {{ {text} }}',
extra={'synapse': {'text': text, 'username': user.name, 'user': user.iden}})
enfo = s_common.err(e)
enfo[1].pop('esrc', None)
enfo[1].pop('ename', None)
await chan.put(('err', enfo))
finally:
if not cancelled:
tock = s_common.now()
took = tock - tick
await chan.put(('fini', {'tock': tock, 'took': took, 'count': count}))
async def _httpRequest(self, meth, url, headers=None, json=None, body=None, ssl_verify=True,
params=None, timeout=300):
meth = await s_stormtypes.tostr(meth)
url = await s_stormtypes.tostr(url)
json = await s_stormtypes.toprim(json)
body = await s_stormtypes.toprim(body)
headers = await s_stormtypes.toprim(headers)
params = await s_stormtypes.toprim(params)
timeout = await s_stormtypes.toint(timeout, noneok=True)
kwargs = {}
if not ssl_verify:
kwargs['ssl'] = False
if params:
kwargs['params'] = params
todo = s_common.todo('getConfOpt', 'http:proxy')
proxyurl = await self.runt.dyncall('cortex', todo)
connector = None
if proxyurl is not None:
connector = aiohttp_socks.ProxyConnector.from_url(proxyurl)
timeout = aiohttp.ClientTimeout(total=timeout)
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as sess:
try:
async with sess.request(meth, url, headers=headers, json=json, data=body, **kwargs) as resp:
info = {
'code': resp.status,
'headers': dict(resp.headers),
'url': str(resp.url),
'body': await resp.read(),
}
return HttpResp(info)
# return HttpResp(code=resp.status, body=await resp.content.read())
except asyncio.CancelledError: # pragma: no cover
raise
except Exception as e:
logger.exception(f'Error during http {meth} @ {url}')
err = s_common.err(e)
info = {
'code': -1,
'headers': dict(),
'url': url,
'body': b'',
'err': err,
}
return HttpResp(info)
async def runStorm():
cancelled = False
tick = s_common.now()
count = 0
try:
# First, try text parsing. If this fails, we won't be able to get
# a storm runtime in the snap, so catch and pass the `err` message
# before handing a `fini` message along.
self.getStormQuery(text)
await chan.put(('init', {'tick': tick, 'text': text, 'task': synt.iden}))
async with await self.snap(user=user) as snap:
snap.link(chan.put)
async for pode in snap.iterStormPodes(text, opts=opts, user=user):
await chan.put(('node', pode))
count += 1
except asyncio.CancelledError:
logger.warning('Storm runtime cancelled.')
cancelled = True
raise
except Exception as e:
logger.exception('Error during storm execution')
enfo = s_common.err(e)
enfo[1].pop('esrc', None)
enfo[1].pop('ename', None)
await chan.put(('err', enfo))
finally:
if cancelled:
return
tock = s_common.now()
took = tock - tick
await chan.put(('fini', {'tock': tock, 'took': took, 'count': count}))
async def _httpRequest(
self,
meth,
url,
headers=None,
json=None,
body=None,
ssl_verify=True,
params=None,
timeout=300,
allow_redirects=True,
fields=None,
):
meth = await s_stormtypes.tostr(meth)
url = await s_stormtypes.tostr(url)
json = await s_stormtypes.toprim(json)
body = await s_stormtypes.toprim(body)
fields = await s_stormtypes.toprim(fields)
headers = await s_stormtypes.toprim(headers)
params = await s_stormtypes.toprim(params)
timeout = await s_stormtypes.toint(timeout, noneok=True)
ssl_verify = await s_stormtypes.tobool(ssl_verify, noneok=True)
allow_redirects = await s_stormtypes.tobool(allow_redirects)
kwargs = {'allow_redirects': allow_redirects}
if params:
kwargs['params'] = self.strify(params)
headers = self.strify(headers)
if fields:
if any(['sha256' in field for field in fields]):
self.runt.confirm(('storm', 'lib', 'axon', 'wput'))
axon = self.runt.snap.core.axon
info = await axon.postfiles(fields,
url,
headers=headers,
params=params,
method=meth,
ssl=ssl_verify,
timeout=timeout)
return HttpResp(info)
else:
data = self._buildFormData(fields)
else:
data = body
proxyurl = self.runt.snap.core.conf.get('http:proxy')
cadir = self.runt.snap.core.conf.get('tls:ca:dir')
connector = None
if proxyurl is not None:
connector = aiohttp_socks.ProxyConnector.from_url(proxyurl)
if ssl_verify is False:
kwargs['ssl'] = False
elif cadir:
kwargs['ssl'] = s_common.getSslCtx(cadir)
else:
# default aiohttp behavior
kwargs['ssl'] = None
timeout = aiohttp.ClientTimeout(total=timeout)
async with aiohttp.ClientSession(connector=connector,
timeout=timeout) as sess:
try:
async with sess.request(meth,
url,
headers=headers,
json=json,
data=data,
**kwargs) as resp:
info = {
'code': resp.status,
'headers': dict(resp.headers),
'url': str(resp.url),
'body': await resp.read(),
}
return HttpResp(info)
except asyncio.CancelledError: # pragma: no cover
raise
except Exception as e:
logger.exception(f'Error during http {meth} @ {url}')
err = s_common.err(e)
info = {
'code': -1,
'headers': dict(),
'url': url,
'body': b'',
'err': err,
}
return HttpResp(info)