async def format(self, valu, format):
'''
Format a Synapse timestamp into a string value using strftime.
'''
timetype = self.runt.snap.model.type('time')
# Give a times string a shot at being normed prior to formating.
try:
norm, _ = timetype.norm(valu)
except s_exc.BadTypeValu as e:
mesg = f'Failed to norm a time value prior to formatting - {str(e)}'
raise s_exc.StormRuntimeError(mesg=mesg, valu=valu,
format=format) from None
if norm == timetype.futsize:
mesg = 'Cannot format a timestamp for ongoing/future time.'
raise s_exc.StormRuntimeError(mesg=mesg, valu=valu, format=format)
try:
dt = datetime.datetime(1970, 1,
1) + datetime.timedelta(milliseconds=norm)
ret = dt.strftime(format)
except Exception as e:
mesg = f'Error during time format - {str(e)}'
raise s_exc.StormRuntimeError(mesg=mesg, valu=valu,
format=format) from None
return ret
async def _whoisGuid(self, props, form):
form = await s_stormtypes.tostr(form)
props = await s_stormtypes.toprim(props)
if form == 'iprec':
guid_props = ('net4', 'net6', 'asof', 'id')
elif form == 'ipcontact':
guid_props = ('contact', 'asof', 'id', 'updated')
elif form == 'ipquery':
guid_props = ('time', 'fqdn', 'url', 'ipv4', 'ipv6')
else:
mesg = f'No guid helpers available for this inet:whois form'
raise s_exc.StormRuntimeError(mesg=mesg, form=form)
guid_vals = []
try:
for prop in guid_props:
val = props.get(prop)
if val is not None:
guid_vals.append(str(val))
except AttributeError as e:
mesg = f'Failed to iterate over props {str(e)}'
raise s_exc.StormRuntimeError(mesg=mesg)
if len(guid_vals) <= 1:
await self.runt.snap.warn(
f'Insufficient guid vals identified, using random guid: {guid_vals}'
)
return s_common.guid()
return s_common.guid(sorted(guid_vals))
async def deref(self, name):
# method used by storm runtime library on deref
try:
await self.client.waitready()
return getattr(self.client, name)
except asyncio.TimeoutError:
mesg = 'Timeout waiting for storm service'
raise s_exc.StormRuntimeError(mesg=mesg, name=name) from None
except AttributeError as e: # pragma: no cover
# possible client race condition seen in the real world
mesg = f'Error dereferencing storm service - {str(e)}'
raise s_exc.StormRuntimeError(mesg=mesg, name=name) from None
async def _httpPost(self,
url,
headers=None,
json=None,
body=None,
ssl_verify=True):
url = await s_stormtypes.toprim(url)
json = await s_stormtypes.toprim(json)
body = await s_stormtypes.toprim(body)
headers = await s_stormtypes.toprim(headers)
kwargs = {}
if not ssl_verify:
kwargs['ssl'] = False
async with aiohttp.ClientSession() as sess:
try:
async with sess.post(url,
headers=headers,
json=json,
data=body,
**kwargs) as resp:
info = {
'code': resp.status,
'body': await resp.content.read()
}
return HttpResp(info)
except ValueError as e:
mesg = f'Error during http post - {str(e)}'
raise s_exc.StormRuntimeError(mesg=mesg,
headers=headers,
json=json,
body=body) from None
async def _methNodeRepr(self, name=None, defv=None):
'''
Get the repr for the primary property or secondary propert of a Node.
Args:
name (str): Optional name of the secondary property to get the repr for.
defv (str): Optional default value to return if the secondary property does not exist.
Returns:
String repr for the property.
Raises:
s_exc.StormRuntimeError: If the secondary property does not exist for the Node form.
'''
try:
return self.valu.repr(name=name)
except s_exc.NoPropValu:
return defv
except s_exc.NoSuchProp as e:
form = e.get('form')
prop = e.get('prop')
mesg = f'Requested property [{prop}] does not exist for the form [{form}].'
raise s_exc.StormRuntimeError(mesg=mesg, form=form,
prop=prop) from None
async def execStormCmd(self, runt, genr):
if not self.runtsafe:
mesg = 'macro.exec does not support per-node invocation'
raise s_exc.StormRuntimeError(mesg=mesg)
name = await s_stormtypes.tostr(self.opts.name)
hivepath = ('cortex', 'storm', 'macros', name)
mdef = await runt.snap.core.getHiveKey(hivepath)
if mdef is None:
mesg = f'Macro name not found: {name}'
raise s_exc.NoSuchName(mesg=mesg)
query = await runt.getStormQuery(mdef['storm'])
subr = await runt.getScopeRuntime(query)
async def wrapgenr():
async for node, path in genr:
path.initframe(initrunt=subr)
yield node, path
async for nnode, npath in subr.iterStormQuery(query, genr=wrapgenr()):
yield nnode, npath
async def _methSign(self,
baseurl,
method='GET',
headers=None,
params=None,
body=None):
url = yarl.URL(baseurl).with_query(await s_stormtypes.toprim(params))
headers = await s_stormtypes.toprim(headers)
body = await s_stormtypes.toprim(body)
if self.sigtype == oauth1.SIGNATURE_TYPE_BODY:
if not headers:
headers = {
'Content-Type': oauth1.rfc5849.CONTENT_TYPE_FORM_URLENCODED
}
else:
headers[
'Content-Type'] = oauth1.rfc5849.CONTENT_TYPE_FORM_URLENCODED
try:
return self.client.sign(str(url),
http_method=method,
headers=headers,
body=body)
except ValueError as e:
mesg = f'Request signing failed ({str(e)})'
raise s_exc.StormRuntimeError(mesg=mesg) from None
async def connect(self, host, port=993, timeout=30, ssl=True):
self.runt.confirm(('storm', 'inet', 'imap', 'connect'))
ssl = await s_stormtypes.tobool(ssl)
host = await s_stormtypes.tostr(host)
port = await s_stormtypes.toint(port)
timeout = await s_stormtypes.toint(timeout, noneok=True)
if ssl:
imap_cli = aioimaplib.IMAP4_SSL(host=host, port=port, timeout=timeout)
else:
imap_cli = aioimaplib.IMAP4(host=host, port=port, timeout=timeout)
async def fini():
# call protocol.logout() so fini() doesn't hang
await asyncio.wait_for(imap_cli.protocol.logout(), 5)
self.runt.snap.onfini(fini)
try:
await imap_cli.wait_hello_from_server()
except asyncio.TimeoutError:
raise s_exc.StormRuntimeError(mesg='Timed out waiting for IMAP server hello.') from None
return ImapServer(self.runt, imap_cli)
async def send(self,
host,
port=25,
user=None,
passwd=None,
usetls=False,
starttls=False,
timeout=60):
self.runt.confirm(('storm', 'inet', 'smtp', 'send'))
try:
if self.bodytext is None and self.bodyhtml is None:
mesg = 'The storm:smtp:message has no HTML or text body.'
raise s_exc.StormRuntimeError(mesg=mesg)
host = await s_stormtypes.tostr(host)
port = await s_stormtypes.toint(port)
usetls = await s_stormtypes.tobool(usetls)
starttls = await s_stormtypes.tobool(starttls)
timeout = await s_stormtypes.toint(timeout)
user = await s_stormtypes.tostr(user, noneok=True)
passwd = await s_stormtypes.tostr(passwd, noneok=True)
message = MIMEMultipart('alternative')
if self.bodytext is not None:
message.attach(MIMEText(self.bodytext, 'plain', 'utf-8'))
if self.bodyhtml is not None:
message.attach(MIMEText(self.bodyhtml, 'html', 'utf-8'))
for name, valu in self.headers.items():
message[await s_stormtypes.tostr(
name)] = await s_stormtypes.tostr(valu)
recipients = [await s_stormtypes.tostr(e) for e in self.recipients]
futu = aiosmtplib.send(message,
port=port,
hostname=host,
sender=self.sender,
recipients=recipients,
use_tls=usetls,
start_tls=starttls,
username=user,
password=passwd)
await asyncio.wait_for(futu, timeout=timeout)
except asyncio.CancelledError: # pragma: no cover
raise
except Exception as e:
return (False, s_common.excinfo(e))
return (True, {})
async def deref(self, name):
valu = self.path.getVar(name)
if valu is not s_common.novalu:
return valu
mesg = f'No var with name: {name}.'
raise s_exc.StormRuntimeError(mesg=mesg)
async def run_imap_coro(coro):
'''
Raises or returns data
'''
try:
status, data = await coro
except asyncio.TimeoutError:
raise s_exc.StormRuntimeError(mesg='Timed out waiting for IMAP server response.') from None
if status == 'OK':
return data
try:
mesg = data[0].decode()
except (TypeError, AttributeError, IndexError, UnicodeDecodeError):
mesg = 'IMAP server returned an error'
raise s_exc.StormRuntimeError(mesg=mesg, status=status)
async def add(self, node, stixtype=None):
if len(self.objs) >= self.maxsize:
mesg = f'STIX Bundle is at maxsize ({self.maxsize}).'
raise s_exc.StormRuntimeError(mesg=mesg)
if not isinstance(node, s_node.Node):
await self.runt.warnonce(
'STIX bundle add() method requires a node.')
return None
formconf = self.config['forms'].get(node.form.name)
if formconf is None:
await self.runt.warnonce(
f'STIX bundle has no config for mapping {node.form.name}.')
return None
if stixtype is None:
stixtype = formconf.get('default')
# cyber observables have UUIDv5 the rest have UUIDv4
if stixtype in stix_observables:
stixid = f'{stixtype}--{uuid5(node.ndef)}'
else:
stixid = f'{stixtype}--{uuid4(node.ndef)}'
if self.objs.get(stixid) is not None:
return stixid
stixconf = formconf['stix'].get(stixtype)
if stixconf is None:
await self.runt.warnonce(
f'STIX bundle config has no config to map {node.form.name} to {stixtype}.'
)
return None
stixitem = self.objs.get(stixid)
if stixitem is None:
stixitem = self.objs[stixid] = self._initStixItem(
stixid, stixtype, node)
props = stixconf.get('props')
if props is not None:
for name, storm in props.items():
valu = await self._callStorm(storm, node)
if valu is s_common.novalu:
continue
stixitem[name] = valu
for (relname, reltype, relstorm) in stixconf.get('rels', ()):
async for relnode, relpath in node.storm(self.runt, relstorm):
n2id = await self.add(relnode, stixtype=reltype)
await self._addRel(stixid, relname, n2id)
return stixid
async def _expand(self, valu):
valu = await s_stormtypes.tostr(valu)
valu = valu.strip()
try:
ipv6 = ipaddress.IPv6Address(valu)
return ipv6.exploded
except ipaddress.AddressValueError as e:
mesg = f'Error expanding ipv6: {e} for valu={valu}'
raise s_exc.StormRuntimeError(mesg=mesg, valu=valu)
async def _decode(self, valu, urlsafe=True):
try:
if urlsafe:
return base64.urlsafe_b64decode(valu)
return base64.b64decode(valu)
except binascii.Error as e:
mesg = f'Error during base64 decoding - {str(e)}'
raise s_exc.StormRuntimeError(mesg=mesg,
valu=valu,
urlsafe=urlsafe) from None
async def _libVarsGet(self, name):
'''
Resolve a variable in a storm query
'''
ret = self.runt.getVar(name, defv=s_common.novalu)
if ret is s_common.novalu:
mesg = f'No var with name: {name}'
raise s_exc.StormRuntimeError(mesg=mesg, name=name)
return ret
async def _whoisGuid(self, props, form):
'''
Provides standard patterns for creating guids for certain inet:whois forms.
Args:
props (dict): Dictionary of properties used to create the form
form (str): The inet:whois form to create the guid for
Returns:
(str): A guid from synapse.common
Raises:
StormRuntimeError: If form is not supported in this method
'''
if form == 'iprec':
guid_props = ('net4', 'net6', 'asof', 'id')
elif form == 'ipcontact':
guid_props = ('contact', 'asof', 'id', 'updated')
elif form == 'ipquery':
guid_props = ('time', 'fqdn', 'url', 'ipv4', 'ipv6')
else:
mesg = f'No guid helpers available for this inet:whois form'
raise s_exc.StormRuntimeError(mesg=mesg, form=form)
guid_vals = []
try:
for prop in guid_props:
val = props.get(prop)
if val is not None:
guid_vals.append(str(val))
except AttributeError as e:
mesg = f'Failed to iterate over props {str(e)}'
raise s_exc.StormRuntimeError(mesg=mesg)
if len(guid_vals) <= 1:
await self.runt.snap.warn(
f'Insufficient guid vals identified, using random guid: {guid_vals}'
)
return s_common.guid()
return s_common.guid(sorted(guid_vals))
async def parse(self, valu, format):
'''
Parse a timestamp string using datetimte.strptime formatting.
'''
try:
dt = datetime.datetime.strptime(valu, format)
except ValueError as e:
mesg = f'Error during time parsing - {str(e)}'
raise s_exc.StormRuntimeError(mesg=mesg, valu=valu,
format=format) from None
return int((dt - s_time.EPOCH).total_seconds() * 1000)
async def _httpRequest(self,
meth,
url,
headers=None,
json=None,
body=None,
ssl_verify=True,
params=None):
meth = await s_stormtypes.tostr(meth)
url = await s_stormtypes.tostr(url)
json = await s_stormtypes.toprim(json)
body = await s_stormtypes.toprim(body)
headers = await s_stormtypes.toprim(headers)
params = await s_stormtypes.toprim(params)
kwargs = {}
if not ssl_verify:
kwargs['ssl'] = False
if params:
kwargs['params'] = params
todo = s_common.todo('getConfOpt', 'http:proxy')
proxyurl = await self.runt.dyncall('cortex', todo)
connector = None
if proxyurl is not None:
connector = aiohttp_socks.ProxyConnector.from_url(proxyurl)
async with aiohttp.ClientSession(connector=connector) as sess:
try:
async with sess.request(meth,
url,
headers=headers,
json=json,
data=body,
**kwargs) as resp:
info = {
'code': resp.status,
'headers': dict(resp.headers),
'url': str(resp.url),
'body': await resp.read(),
}
return HttpResp(info)
# return HttpResp(code=resp.status, body=await resp.content.read())
except asyncio.CancelledError: # pragma: no cover
raise
except Exception as e:
mesg = f'Error during http {meth} - {str(e)}'
raise s_exc.StormRuntimeError(mesg=mesg,
headers=headers,
json=json,
body=body,
params=params) from None
async def _methListIndex(self, valu):
'''
Return a single field from the list by index.
'''
indx = intify(valu)
try:
return self.valu[indx]
except IndexError as e:
raise s_exc.StormRuntimeError(mesg=str(e),
valurepr=repr(self.valu),
len=len(self.valu),
indx=indx) from None
async def _jsonSchema(self, schema):
schema = await s_stormtypes.toprim(schema)
# We have to ensure that we have a valid schema for making the object.
try:
await s_coro.spawn((compileJsSchema, (schema, ), {}))
except asyncio.CancelledError: # pragma: no cover
raise
except Exception as e:
raise s_exc.StormRuntimeError(
mesg=f'Unable to compile Json Schema: {str(e)}',
schema=schema) from e
return JsonSchema(self.runt, schema)
async def _httpPost(self, url, headers=None, json=None, body=None):
async with aiohttp.ClientSession() as sess:
try:
async with sess.post(url,
headers=headers,
json=json,
data=body) as resp:
info = {
'code': resp.status,
'body': await resp.content.read()
}
return HttpResp(info)
except ValueError as e:
mesg = f'Error during http post - {str(e)}'
raise s_exc.StormRuntimeError(mesg=mesg,
headers=headers,
json=json,
body=body) from None
async def _handleStormCli(self, text):
core = self._reqCore()
outp = OutPutRst()
text = self._getStormMultiline(text)
self._printf('::\n')
self._printf('\n')
cli = await StormCliOutput.anit(item=core, outp=outp)
self._printf(await cli.runRstCmdLine(
text, self.context, stormopts=self.context.get('storm-opts')))
if self.context.pop('storm-fail', None):
raise s_exc.StormRuntimeError(
mesg='Expected a failure, but none occurred.')
self._printf('\n')
async def execStormCmd(self, runt, genr):
if not self.opts.query:
raise s_exc.StormRuntimeError(
mesg='Tee command must take at least one query as input.',
name=self.name)
async for node, path in genr: # type: s_node.Node, s_node.Path
for query in self.opts.query:
query = query[1:-1]
# This does update path with any vars set in the last npath (node.storm behavior)
async for nnode, npath in node.storm(query,
user=runt.user,
path=path):
yield nnode, npath
if self.opts.join:
yield node, path
async def execStormCmd(self, runt, genr):
if not self.runtsafe:
mesg = 'macro.exec does not support per-node invocation'
raise s_exc.StormRuntimeError(mesg=mesg)
name = await s_stormtypes.tostr(self.opts.name)
hivepath = ('cortex', 'storm', 'macros', name)
mdef = await runt.snap.core.getHiveKey(hivepath)
if mdef is None:
mesg = f'Macro name not found: {name}'
raise s_exc.NoSuchName(mesg=mesg)
query = await runt.getStormQuery(mdef['storm'])
async with runt.getSubRuntime(query) as subr:
async for nnode, npath in subr.execute(genr=genr):
yield nnode, npath
async def _handleStorm(self, text):
'''
Run a Storm command and generate text from the output.
Args:
text (str): A valid Storm query.
'''
core = self._reqCore()
text = self._getStormMultiline(text)
self._printf('::\n')
self._printf('\n')
soutp = StormOutput(core,
self.context,
stormopts=self.context.get('storm-opts'))
self._printf(await soutp.runCmdLine(text))
if self.context.pop('storm-fail', None):
raise s_exc.StormRuntimeError(
mesg='Expected a failure, but none occurred.')
self._printf('\n\n')
async def genStormRst(path, debug=False):
outp = []
context = {}
with open(path, 'r') as fd:
lines = fd.readlines()
for line in lines:
if line.startswith('.. storm-cortex::'):
ctor = line.split('::', 1)[1].strip()
core = await (s_dyndeps.getDynLocal(ctor))()
if context.get('cortex') is not None:
await (context.pop('cortex')).fini()
context['cortex'] = core
continue
if line.startswith('.. storm-opts::'):
item = json.loads(line.split('::', 1)[1].strip())
context['opts'] = item
continue
if line.startswith('.. storm-expect::'):
# TODO handle some light weight output confirmation.
continue
if line.startswith('.. storm-pre::'):
# runt a storm query to prepare the cortex (do not output)
text = line.split('::', 1)[1].strip()
core = context.get('cortex')
if core is None:
mesg = 'No cortex set. Use .. storm-cortex::'
raise s_exc.NoSuchVar(mesg=mesg)
opts = context.get('opts')
await core.callStorm(text, opts=opts)
continue
if line.startswith('.. storm::'):
text = line.split('::', 1)[1].strip()
core = context.get('cortex')
if core is None:
mesg = 'No cortex set. Use .. storm-cortex::'
raise s_exc.NoSuchVar(mesg=mesg)
outp.append('::\n')
outp.append('\n')
outp.append(f' > {text}\n')
opts = context.get('opts')
msgs = await core.stormlist(text, opts=opts)
# TODO use StormOutput
for mesg in await core.stormlist(text, opts=opts):
if mesg[0] == 'print':
ptxt = mesg[1]['mesg']
outp.append(f' {ptxt}\n')
continue
if mesg[0] == 'warn':
ptxt = mesg[1]['mesg']
outp.append(f' WARNING: {ptxt}\n')
continue
if mesg[0] == 'err':
raise s_exc.StormRuntimeError(mesg=mesg)
outp.append('\n')
continue
outp.append(line)
core = context.get('cortex')
if core is not None:
await core.fini()
return outp
async def _httpRequest(self,
meth,
url,
headers=None,
json=None,
body=None,
ssl_verify=True,
params=None):
'''
Make an HTTP request using the given HTTP method to the url.
Args:
meth (str): The HTTP method. (ex. PUT)
url (str): The URL to post to.
headers (dict): HTTP headers to send with the request.
json: The data to post, as JSON object.
body: The data to post, as binary object.
ssl_verify (bool): Perform SSL/TLS verification. Defaults to true.
params (dict): Optional parameters which may be passed to the request.
Returns:
HttpResp: A Storm HttpResp object.
'''
meth = await s_stormtypes.tostr(meth)
url = await s_stormtypes.tostr(url)
json = await s_stormtypes.toprim(json)
body = await s_stormtypes.toprim(body)
headers = await s_stormtypes.toprim(headers)
params = await s_stormtypes.toprim(params)
kwargs = {}
if not ssl_verify:
kwargs['ssl'] = False
if params:
kwargs['params'] = params
async with aiohttp.ClientSession() as sess:
try:
async with sess.request(meth,
url,
headers=headers,
json=json,
data=body,
**kwargs) as resp:
info = {
'code': resp.status,
'body': await resp.content.read()
}
return HttpResp(info)
except (TypeError, ValueError) as e:
mesg = f'Error during http {meth} - {str(e)}'
raise s_exc.StormRuntimeError(mesg=mesg,
headers=headers,
json=json,
body=body,
params=params) from None
def _onErr(self, mesg, opts):
# raise on err for rst
raise s_exc.StormRuntimeError(mesg=mesg)
async def setitem(self, name, valu):
mesg = f'{self.__class__.__name__} does not support assignment.'
raise s_exc.StormRuntimeError(mesg=mesg)
def _reqStr(self, name):
if not isinstance(name, str):
mesg = 'The name of a persistent variable must be a string.'
raise s_exc.StormRuntimeError(mesg=mesg, name=name)
