async def format(self, valu, format): ''' Format a Synapse timestamp into a string value using strftime. ''' timetype = self.runt.snap.model.type('time') # Give a times string a shot at being normed prior to formating. try: norm, _ = timetype.norm(valu) except s_exc.BadTypeValu as e: mesg = f'Failed to norm a time value prior to formatting - {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg, valu=valu, format=format) from None if norm == timetype.futsize: mesg = 'Cannot format a timestamp for ongoing/future time.' raise s_exc.StormRuntimeError(mesg=mesg, valu=valu, format=format) try: dt = datetime.datetime(1970, 1, 1) + datetime.timedelta(milliseconds=norm) ret = dt.strftime(format) except Exception as e: mesg = f'Error during time format - {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg, valu=valu, format=format) from None return ret
async def _whoisGuid(self, props, form): form = await s_stormtypes.tostr(form) props = await s_stormtypes.toprim(props) if form == 'iprec': guid_props = ('net4', 'net6', 'asof', 'id') elif form == 'ipcontact': guid_props = ('contact', 'asof', 'id', 'updated') elif form == 'ipquery': guid_props = ('time', 'fqdn', 'url', 'ipv4', 'ipv6') else: mesg = f'No guid helpers available for this inet:whois form' raise s_exc.StormRuntimeError(mesg=mesg, form=form) guid_vals = [] try: for prop in guid_props: val = props.get(prop) if val is not None: guid_vals.append(str(val)) except AttributeError as e: mesg = f'Failed to iterate over props {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg) if len(guid_vals) <= 1: await self.runt.snap.warn( f'Insufficient guid vals identified, using random guid: {guid_vals}' ) return s_common.guid() return s_common.guid(sorted(guid_vals))
async def deref(self, name): # method used by storm runtime library on deref try: await self.client.waitready() return getattr(self.client, name) except asyncio.TimeoutError: mesg = 'Timeout waiting for storm service' raise s_exc.StormRuntimeError(mesg=mesg, name=name) from None except AttributeError as e: # pragma: no cover # possible client race condition seen in the real world mesg = f'Error dereferencing storm service - {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg, name=name) from None
async def _httpPost(self, url, headers=None, json=None, body=None, ssl_verify=True): url = await s_stormtypes.toprim(url) json = await s_stormtypes.toprim(json) body = await s_stormtypes.toprim(body) headers = await s_stormtypes.toprim(headers) kwargs = {} if not ssl_verify: kwargs['ssl'] = False async with aiohttp.ClientSession() as sess: try: async with sess.post(url, headers=headers, json=json, data=body, **kwargs) as resp: info = { 'code': resp.status, 'body': await resp.content.read() } return HttpResp(info) except ValueError as e: mesg = f'Error during http post - {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg, headers=headers, json=json, body=body) from None
async def _methNodeRepr(self, name=None, defv=None): ''' Get the repr for the primary property or secondary propert of a Node. Args: name (str): Optional name of the secondary property to get the repr for. defv (str): Optional default value to return if the secondary property does not exist. Returns: String repr for the property. Raises: s_exc.StormRuntimeError: If the secondary property does not exist for the Node form. ''' try: return self.valu.repr(name=name) except s_exc.NoPropValu: return defv except s_exc.NoSuchProp as e: form = e.get('form') prop = e.get('prop') mesg = f'Requested property [{prop}] does not exist for the form [{form}].' raise s_exc.StormRuntimeError(mesg=mesg, form=form, prop=prop) from None
async def execStormCmd(self, runt, genr): if not self.runtsafe: mesg = 'macro.exec does not support per-node invocation' raise s_exc.StormRuntimeError(mesg=mesg) name = await s_stormtypes.tostr(self.opts.name) hivepath = ('cortex', 'storm', 'macros', name) mdef = await runt.snap.core.getHiveKey(hivepath) if mdef is None: mesg = f'Macro name not found: {name}' raise s_exc.NoSuchName(mesg=mesg) query = await runt.getStormQuery(mdef['storm']) subr = await runt.getScopeRuntime(query) async def wrapgenr(): async for node, path in genr: path.initframe(initrunt=subr) yield node, path async for nnode, npath in subr.iterStormQuery(query, genr=wrapgenr()): yield nnode, npath
async def _methSign(self, baseurl, method='GET', headers=None, params=None, body=None): url = yarl.URL(baseurl).with_query(await s_stormtypes.toprim(params)) headers = await s_stormtypes.toprim(headers) body = await s_stormtypes.toprim(body) if self.sigtype == oauth1.SIGNATURE_TYPE_BODY: if not headers: headers = { 'Content-Type': oauth1.rfc5849.CONTENT_TYPE_FORM_URLENCODED } else: headers[ 'Content-Type'] = oauth1.rfc5849.CONTENT_TYPE_FORM_URLENCODED try: return self.client.sign(str(url), http_method=method, headers=headers, body=body) except ValueError as e: mesg = f'Request signing failed ({str(e)})' raise s_exc.StormRuntimeError(mesg=mesg) from None
async def connect(self, host, port=993, timeout=30, ssl=True): self.runt.confirm(('storm', 'inet', 'imap', 'connect')) ssl = await s_stormtypes.tobool(ssl) host = await s_stormtypes.tostr(host) port = await s_stormtypes.toint(port) timeout = await s_stormtypes.toint(timeout, noneok=True) if ssl: imap_cli = aioimaplib.IMAP4_SSL(host=host, port=port, timeout=timeout) else: imap_cli = aioimaplib.IMAP4(host=host, port=port, timeout=timeout) async def fini(): # call protocol.logout() so fini() doesn't hang await asyncio.wait_for(imap_cli.protocol.logout(), 5) self.runt.snap.onfini(fini) try: await imap_cli.wait_hello_from_server() except asyncio.TimeoutError: raise s_exc.StormRuntimeError(mesg='Timed out waiting for IMAP server hello.') from None return ImapServer(self.runt, imap_cli)
async def send(self, host, port=25, user=None, passwd=None, usetls=False, starttls=False, timeout=60): self.runt.confirm(('storm', 'inet', 'smtp', 'send')) try: if self.bodytext is None and self.bodyhtml is None: mesg = 'The storm:smtp:message has no HTML or text body.' raise s_exc.StormRuntimeError(mesg=mesg) host = await s_stormtypes.tostr(host) port = await s_stormtypes.toint(port) usetls = await s_stormtypes.tobool(usetls) starttls = await s_stormtypes.tobool(starttls) timeout = await s_stormtypes.toint(timeout) user = await s_stormtypes.tostr(user, noneok=True) passwd = await s_stormtypes.tostr(passwd, noneok=True) message = MIMEMultipart('alternative') if self.bodytext is not None: message.attach(MIMEText(self.bodytext, 'plain', 'utf-8')) if self.bodyhtml is not None: message.attach(MIMEText(self.bodyhtml, 'html', 'utf-8')) for name, valu in self.headers.items(): message[await s_stormtypes.tostr( name)] = await s_stormtypes.tostr(valu) recipients = [await s_stormtypes.tostr(e) for e in self.recipients] futu = aiosmtplib.send(message, port=port, hostname=host, sender=self.sender, recipients=recipients, use_tls=usetls, start_tls=starttls, username=user, password=passwd) await asyncio.wait_for(futu, timeout=timeout) except asyncio.CancelledError: # pragma: no cover raise except Exception as e: return (False, s_common.excinfo(e)) return (True, {})
async def deref(self, name): valu = self.path.getVar(name) if valu is not s_common.novalu: return valu mesg = f'No var with name: {name}.' raise s_exc.StormRuntimeError(mesg=mesg)
async def run_imap_coro(coro): ''' Raises or returns data ''' try: status, data = await coro except asyncio.TimeoutError: raise s_exc.StormRuntimeError(mesg='Timed out waiting for IMAP server response.') from None if status == 'OK': return data try: mesg = data[0].decode() except (TypeError, AttributeError, IndexError, UnicodeDecodeError): mesg = 'IMAP server returned an error' raise s_exc.StormRuntimeError(mesg=mesg, status=status)
async def add(self, node, stixtype=None): if len(self.objs) >= self.maxsize: mesg = f'STIX Bundle is at maxsize ({self.maxsize}).' raise s_exc.StormRuntimeError(mesg=mesg) if not isinstance(node, s_node.Node): await self.runt.warnonce( 'STIX bundle add() method requires a node.') return None formconf = self.config['forms'].get(node.form.name) if formconf is None: await self.runt.warnonce( f'STIX bundle has no config for mapping {node.form.name}.') return None if stixtype is None: stixtype = formconf.get('default') # cyber observables have UUIDv5 the rest have UUIDv4 if stixtype in stix_observables: stixid = f'{stixtype}--{uuid5(node.ndef)}' else: stixid = f'{stixtype}--{uuid4(node.ndef)}' if self.objs.get(stixid) is not None: return stixid stixconf = formconf['stix'].get(stixtype) if stixconf is None: await self.runt.warnonce( f'STIX bundle config has no config to map {node.form.name} to {stixtype}.' ) return None stixitem = self.objs.get(stixid) if stixitem is None: stixitem = self.objs[stixid] = self._initStixItem( stixid, stixtype, node) props = stixconf.get('props') if props is not None: for name, storm in props.items(): valu = await self._callStorm(storm, node) if valu is s_common.novalu: continue stixitem[name] = valu for (relname, reltype, relstorm) in stixconf.get('rels', ()): async for relnode, relpath in node.storm(self.runt, relstorm): n2id = await self.add(relnode, stixtype=reltype) await self._addRel(stixid, relname, n2id) return stixid
async def _expand(self, valu): valu = await s_stormtypes.tostr(valu) valu = valu.strip() try: ipv6 = ipaddress.IPv6Address(valu) return ipv6.exploded except ipaddress.AddressValueError as e: mesg = f'Error expanding ipv6: {e} for valu={valu}' raise s_exc.StormRuntimeError(mesg=mesg, valu=valu)
async def _decode(self, valu, urlsafe=True): try: if urlsafe: return base64.urlsafe_b64decode(valu) return base64.b64decode(valu) except binascii.Error as e: mesg = f'Error during base64 decoding - {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg, valu=valu, urlsafe=urlsafe) from None
async def _libVarsGet(self, name): ''' Resolve a variable in a storm query ''' ret = self.runt.getVar(name, defv=s_common.novalu) if ret is s_common.novalu: mesg = f'No var with name: {name}' raise s_exc.StormRuntimeError(mesg=mesg, name=name) return ret
async def _whoisGuid(self, props, form): ''' Provides standard patterns for creating guids for certain inet:whois forms. Args: props (dict): Dictionary of properties used to create the form form (str): The inet:whois form to create the guid for Returns: (str): A guid from synapse.common Raises: StormRuntimeError: If form is not supported in this method ''' if form == 'iprec': guid_props = ('net4', 'net6', 'asof', 'id') elif form == 'ipcontact': guid_props = ('contact', 'asof', 'id', 'updated') elif form == 'ipquery': guid_props = ('time', 'fqdn', 'url', 'ipv4', 'ipv6') else: mesg = f'No guid helpers available for this inet:whois form' raise s_exc.StormRuntimeError(mesg=mesg, form=form) guid_vals = [] try: for prop in guid_props: val = props.get(prop) if val is not None: guid_vals.append(str(val)) except AttributeError as e: mesg = f'Failed to iterate over props {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg) if len(guid_vals) <= 1: await self.runt.snap.warn( f'Insufficient guid vals identified, using random guid: {guid_vals}' ) return s_common.guid() return s_common.guid(sorted(guid_vals))
async def parse(self, valu, format): ''' Parse a timestamp string using datetimte.strptime formatting. ''' try: dt = datetime.datetime.strptime(valu, format) except ValueError as e: mesg = f'Error during time parsing - {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg, valu=valu, format=format) from None return int((dt - s_time.EPOCH).total_seconds() * 1000)
async def _httpRequest(self, meth, url, headers=None, json=None, body=None, ssl_verify=True, params=None): meth = await s_stormtypes.tostr(meth) url = await s_stormtypes.tostr(url) json = await s_stormtypes.toprim(json) body = await s_stormtypes.toprim(body) headers = await s_stormtypes.toprim(headers) params = await s_stormtypes.toprim(params) kwargs = {} if not ssl_verify: kwargs['ssl'] = False if params: kwargs['params'] = params todo = s_common.todo('getConfOpt', 'http:proxy') proxyurl = await self.runt.dyncall('cortex', todo) connector = None if proxyurl is not None: connector = aiohttp_socks.ProxyConnector.from_url(proxyurl) async with aiohttp.ClientSession(connector=connector) as sess: try: async with sess.request(meth, url, headers=headers, json=json, data=body, **kwargs) as resp: info = { 'code': resp.status, 'headers': dict(resp.headers), 'url': str(resp.url), 'body': await resp.read(), } return HttpResp(info) # return HttpResp(code=resp.status, body=await resp.content.read()) except asyncio.CancelledError: # pragma: no cover raise except Exception as e: mesg = f'Error during http {meth} - {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg, headers=headers, json=json, body=body, params=params) from None
async def _methListIndex(self, valu): ''' Return a single field from the list by index. ''' indx = intify(valu) try: return self.valu[indx] except IndexError as e: raise s_exc.StormRuntimeError(mesg=str(e), valurepr=repr(self.valu), len=len(self.valu), indx=indx) from None
async def _jsonSchema(self, schema): schema = await s_stormtypes.toprim(schema) # We have to ensure that we have a valid schema for making the object. try: await s_coro.spawn((compileJsSchema, (schema, ), {})) except asyncio.CancelledError: # pragma: no cover raise except Exception as e: raise s_exc.StormRuntimeError( mesg=f'Unable to compile Json Schema: {str(e)}', schema=schema) from e return JsonSchema(self.runt, schema)
async def _httpPost(self, url, headers=None, json=None, body=None): async with aiohttp.ClientSession() as sess: try: async with sess.post(url, headers=headers, json=json, data=body) as resp: info = { 'code': resp.status, 'body': await resp.content.read() } return HttpResp(info) except ValueError as e: mesg = f'Error during http post - {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg, headers=headers, json=json, body=body) from None
async def _handleStormCli(self, text): core = self._reqCore() outp = OutPutRst() text = self._getStormMultiline(text) self._printf('::\n') self._printf('\n') cli = await StormCliOutput.anit(item=core, outp=outp) self._printf(await cli.runRstCmdLine( text, self.context, stormopts=self.context.get('storm-opts'))) if self.context.pop('storm-fail', None): raise s_exc.StormRuntimeError( mesg='Expected a failure, but none occurred.') self._printf('\n')
async def execStormCmd(self, runt, genr): if not self.opts.query: raise s_exc.StormRuntimeError( mesg='Tee command must take at least one query as input.', name=self.name) async for node, path in genr: # type: s_node.Node, s_node.Path for query in self.opts.query: query = query[1:-1] # This does update path with any vars set in the last npath (node.storm behavior) async for nnode, npath in node.storm(query, user=runt.user, path=path): yield nnode, npath if self.opts.join: yield node, path
async def execStormCmd(self, runt, genr): if not self.runtsafe: mesg = 'macro.exec does not support per-node invocation' raise s_exc.StormRuntimeError(mesg=mesg) name = await s_stormtypes.tostr(self.opts.name) hivepath = ('cortex', 'storm', 'macros', name) mdef = await runt.snap.core.getHiveKey(hivepath) if mdef is None: mesg = f'Macro name not found: {name}' raise s_exc.NoSuchName(mesg=mesg) query = await runt.getStormQuery(mdef['storm']) async with runt.getSubRuntime(query) as subr: async for nnode, npath in subr.execute(genr=genr): yield nnode, npath
async def _handleStorm(self, text): ''' Run a Storm command and generate text from the output. Args: text (str): A valid Storm query. ''' core = self._reqCore() text = self._getStormMultiline(text) self._printf('::\n') self._printf('\n') soutp = StormOutput(core, self.context, stormopts=self.context.get('storm-opts')) self._printf(await soutp.runCmdLine(text)) if self.context.pop('storm-fail', None): raise s_exc.StormRuntimeError( mesg='Expected a failure, but none occurred.') self._printf('\n\n')
async def genStormRst(path, debug=False): outp = [] context = {} with open(path, 'r') as fd: lines = fd.readlines() for line in lines: if line.startswith('.. storm-cortex::'): ctor = line.split('::', 1)[1].strip() core = await (s_dyndeps.getDynLocal(ctor))() if context.get('cortex') is not None: await (context.pop('cortex')).fini() context['cortex'] = core continue if line.startswith('.. storm-opts::'): item = json.loads(line.split('::', 1)[1].strip()) context['opts'] = item continue if line.startswith('.. storm-expect::'): # TODO handle some light weight output confirmation. continue if line.startswith('.. storm-pre::'): # runt a storm query to prepare the cortex (do not output) text = line.split('::', 1)[1].strip() core = context.get('cortex') if core is None: mesg = 'No cortex set. Use .. storm-cortex::' raise s_exc.NoSuchVar(mesg=mesg) opts = context.get('opts') await core.callStorm(text, opts=opts) continue if line.startswith('.. storm::'): text = line.split('::', 1)[1].strip() core = context.get('cortex') if core is None: mesg = 'No cortex set. Use .. storm-cortex::' raise s_exc.NoSuchVar(mesg=mesg) outp.append('::\n') outp.append('\n') outp.append(f' > {text}\n') opts = context.get('opts') msgs = await core.stormlist(text, opts=opts) # TODO use StormOutput for mesg in await core.stormlist(text, opts=opts): if mesg[0] == 'print': ptxt = mesg[1]['mesg'] outp.append(f' {ptxt}\n') continue if mesg[0] == 'warn': ptxt = mesg[1]['mesg'] outp.append(f' WARNING: {ptxt}\n') continue if mesg[0] == 'err': raise s_exc.StormRuntimeError(mesg=mesg) outp.append('\n') continue outp.append(line) core = context.get('cortex') if core is not None: await core.fini() return outp
async def _httpRequest(self, meth, url, headers=None, json=None, body=None, ssl_verify=True, params=None): ''' Make an HTTP request using the given HTTP method to the url. Args: meth (str): The HTTP method. (ex. PUT) url (str): The URL to post to. headers (dict): HTTP headers to send with the request. json: The data to post, as JSON object. body: The data to post, as binary object. ssl_verify (bool): Perform SSL/TLS verification. Defaults to true. params (dict): Optional parameters which may be passed to the request. Returns: HttpResp: A Storm HttpResp object. ''' meth = await s_stormtypes.tostr(meth) url = await s_stormtypes.tostr(url) json = await s_stormtypes.toprim(json) body = await s_stormtypes.toprim(body) headers = await s_stormtypes.toprim(headers) params = await s_stormtypes.toprim(params) kwargs = {} if not ssl_verify: kwargs['ssl'] = False if params: kwargs['params'] = params async with aiohttp.ClientSession() as sess: try: async with sess.request(meth, url, headers=headers, json=json, data=body, **kwargs) as resp: info = { 'code': resp.status, 'body': await resp.content.read() } return HttpResp(info) except (TypeError, ValueError) as e: mesg = f'Error during http {meth} - {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg, headers=headers, json=json, body=body, params=params) from None
def _onErr(self, mesg, opts): # raise on err for rst raise s_exc.StormRuntimeError(mesg=mesg)
async def setitem(self, name, valu): mesg = f'{self.__class__.__name__} does not support assignment.' raise s_exc.StormRuntimeError(mesg=mesg)
def _reqStr(self, name): if not isinstance(name, str): mesg = 'The name of a persistent variable must be a string.' raise s_exc.StormRuntimeError(mesg=mesg, name=name)