def openlink(link):
'''
Construct a telepath proxy from a link tufo.
Example:
foo = openlink(link)
foo.bar(20)
'''
# special case for dmon://fooname/ which refers to a
# named object within whatever dmon is currently in scope
if link[0] == 'dmon':
dmon = s_scope.get('dmon')
if dmon is None:
raise s_common.NoSuchName(name='dmon', link=link, mesg='no dmon instance in current scope')
# the "host" part is really a dmon local
host = link[1].get('host')
item = dmon.locs.get(host)
if item is None:
raise s_common.NoSuchName(name=host, link=link, mesg='dmon instance has no local with that name')
return item
relay = s_link.getLinkRelay(link)
name = link[1].get('path')[1:]
sock = relay.connect()
synack = teleSynAck(sock, name=name)
bases = ()
return Proxy(relay, sock=sock)
def openlink(link):
'''
Construct a Telepath Proxy from a link tufo.
Args:
link ((str, dict)): A link dictionary.
Examples:
Get a proxy object, call a function then close the object::
foo = openlink(link)
foo.bar(20)
foo.fini()
Get a proxy object as a context manager, which will result in the object being automatically closed::
with openlink as foo:
foo.dostuff(30)
Returns:
Proxy: A Proxy object for calling remote tasks.
'''
# special case for dmon://fooname/ which refers to a
# named object within whatever dmon is currently in scope
if link[0] == 'dmon':
dmon = s_scope.get('dmon')
if dmon is None:
raise s_common.NoSuchName(name='dmon',
link=link,
mesg='no dmon instance in current scope')
# the "host" part is really a dmon local
host = link[1].get('host')
item = dmon.locs.get(host)
if item is None:
raise s_common.NoSuchName(
name=host,
link=link,
mesg='dmon instance has no local with that name')
return item
relay = s_link.getLinkRelay(link)
name = link[1].get('path')[1:]
sock = relay.connect()
synack = teleSynAck(sock, name=name)
bases = ()
return Proxy(relay, sock=sock)
def _parseGestConfig(self, gest_data=None): # type: (dict) -> None
if gest_data is None:
return
self.gest_name = gest_data.get('name')
gestdef = gest_data.get('definition')
self.gest_open = gestdef.get('open')
self.gest = s_ingest.Ingest(gestdef)
# Blow up on missing data early
if not self.gest_name:
raise s_common.NoSuchName(
name='name', mesg='API Ingest definition is missing its name.')
if not self.gest_open:
raise s_common.NoSuchName(
name='open',
mesg='Ingest definition is missing a open directive.')
def delWebConf(self, namespace):
'''
Safely remove a namespace.
Removes a given namespace, APIs and any corresponding event handlers
which have been snapped into the Hypnos object and its cortex via
the addWebConfig API.
Args:
namespace (str): Namespace to remove.
Returns:
None
Raises:
NoSuchName: If the namespace requested does not exist.
'''
if namespace not in self._web_namespaces:
raise s_common.NoSuchName('Namespace is not registered.')
self._web_namespaces.remove(namespace)
self._web_docs.pop(namespace, None)
self._web_default_http_args.pop(namespace, None)
apis_to_remove = []
for api_name in list(self._web_apis.keys()):
ns, name = api_name.split(':', 1)
if ns == namespace:
apis_to_remove.append(api_name)
for api_name in apis_to_remove:
self._delWebApi(api_name)
self.fire('hypnos:register:namespace:del', namespace=namespace)
def _delBlobValu(self, key):
ret = self._getBlobValu(key)
if ret is None: # pragma: no cover
# We should never get here, but if we do, throw an exception.
raise s_common.NoSuchName(name=key, mesg='Cannot delete key which is not present in the blobstore.')
self.delete(self._q_blob_del, key=key)
return ret
def _delBlobValu(self, key):
key_byts = s_msgpack.en(key.encode('utf-8'))
with self._getTxn(write=True) as txn: # type: lmdb.Transaction
ret = txn.pop(key_byts, db=self.blob_store)
if ret is None: # pragma: no cover
# We should never get here, but if we do, throw an exception.
raise s_common.NoSuchName(name=key, mesg='Cannot delete key which is not present in the blobstore.')
return ret
def _delWebApi(self, name):
if name not in self._web_apis:
raise s_common.NoSuchName(name=name,
mesg='API name not registered.')
self._web_apis.pop(name, None)
self._web_api_gest_opens.pop(name, None)
funclist = self._web_api_ingests.pop(name, [])
for action_name, gest_glue in funclist:
self.off(name, gest_glue)
self.fire('hypnos:register:api:del', api=name)
def delBlobValu(self, key):
'''
Remove and return a value from the blob store.
Args:
key (str): Key to remove.
Returns:
Content in the blob store for a given key.
Raises:
NoSuchName: If the key is not present in the store.
'''
if not self.hasBlobValu(key):
raise s_common.NoSuchName(
name=key,
mesg='Cannot delete key which is not present in the blobstore.'
)
buf = self._delBlobValu(key)
self.savebus.fire('syn:core:blob:del', key=key)
return s_msgpack.un(buf)
def _parseConfig(self):
for key in self.required_keys:
if key not in self._raw_config:
logger.error('Remcycle config is missing a required value %s.',
key)
raise s_common.NoSuchName(name=key,
mesg='Missing required key.')
self.url_template = self._raw_config.get('url')
self.doc = self._raw_config.get('doc')
self.url_vars.update(self._raw_config.get('vars', {}))
self.request_defaults = self._raw_config.get('http', {})
self._parseGestConfig(self._raw_config.get('ingest'))
self.api_args.extend(self._raw_config.get('api_args', []))
for key in self.reserved_api_args:
if key in self.api_args:
raise s_common.BadConfValu(name=key,
valu=None,
mesg='Reserved api_arg used.')
self.api_kwargs.update(self._raw_config.get('api_optargs', {}))
# Set effective url
self.effective_url = self.url_template.format(**self.url_vars)
def _parseWebConf(self, config, reload_config):
for key in self._web_required_keys:
if key not in config:
logger.error('Remcycle config is missing a required value %s.',
key)
raise s_common.NoSuchName(name=key,
mesg='Missing required key.')
_apis = config.get('apis')
_namespace = config.get('namespace')
_doc = config.get('doc')
if _namespace in self._web_namespaces:
if reload_config:
self.delWebConf(_namespace)
else:
raise NameError('Namespace is already registered.')
self._web_docs[_namespace] = _doc
self._web_default_http_args[_namespace] = {
k: v
for k, v in config.get('http', {}).items()
}
# Register APIs
for varn, val in _apis:
name = ':'.join([_namespace, varn])
# Stamp api http config ontop of global config, then stamp it into the API config
_http = self._web_default_http_args[_namespace].copy()
_http.update(val.get('http', {}))
val['http'] = _http
nyx_obj = Nyx(val)
self._registerWebApi(name, nyx_obj)
self._web_namespaces.add(_namespace)
self.fire('hypnos:register:namespace:add', namespace=_namespace)
def buildHttpRequest(self,
api_args=None): # type: (dict) -> t_http.HttpRequest
'''
Build the HTTPRequest object for a given configuration and arguments.
Args:
api_args (dict): Arguments support either required or optional URL
values.
Notes:
A HTTP body can be provided to the request by passing its contents
in by adding the "req_body" value to api_args argument.
Returns:
tornado.httpclient.HTTPRequest: HTTPRequest object with the
configured url and attributes.
Raises:
NoSuchName: If the api_args is missing a required API value.
'''
body = None
t_args = {}
if api_args:
body = api_args.pop('req_body', None)
for argn in self.api_args:
argv = api_args.get(argn, s_common.novalu)
if argv is s_common.novalu:
logger.error('Missing argument: %s', argn)
raise s_common.NoSuchName(name=argn,
mesg='Missing an expected argument')
t_args[argn] = urllib.parse.quote_plus(str(argv))
for argn, defval in self.api_kwargs.items():
t_args[argn] = urllib.parse.quote_plus(
str(api_args.get(argn, defval)))
url = self.effective_url.format(**t_args)
req = t_http.HTTPRequest(url, body=body, **self.request_defaults)
return req
def _eval(self, syms):
name = self.tokn[1].get('name')
valu = syms.get(name, undefined)
if valu is undefined:
raise s_common.NoSuchName(name=name)
return valu
def fireWebApi(self, name, *args, **kwargs):
'''
Fire a request to a registered API.
The API response is serviced by a thread in the Hypnos thread pool,
which will fire either an event on the Hypnos service bus or a caller
provided callback function. The default action is to fire an event
on the service bus with the same name as the API itself.
A flattened version of the response, error information and the Boss
job id will be stamped into the kwargs passed along to the the
callbacks. If the API name has a ingest associated with it, the
response data will be pushed into a generator created according to
the ingest open directive.
The flattened response is a dictionary, accessed from kwargs using
the 'resp' key. It contains the following information:
* request: A dictionary containing the requested URL and headers.
This is guaranteed to exist. It has the following values:
- url: URL requested by the remote server.
- headers: Headers passed to the remote server.
* code: HTTP Response code. This will only be present on a
successfull request or if a HTTPError is encountered.
* data: This may be one of three values:
- A SpooledTemporaryFile containing the raw bytes of the
response. This will be present if there is a ingest associated
with the named response. A corresponding generator will be
created and placed in the "ingdata" field and consumed by the
ingest. Post-consumption, seek(0) will be called on the
file-like object. If there are multiple post-ingest consumers
of the job, each one may want to call seek(0) on the file
object before consuming it.
- The decoded data as a string or a decoded json blob. We will
attempt to parse the data based on the Content-Type header.
This is a best effort decoding.
- In the event that the best effort decoding fails, the response
will be available as raw bytes.
* effective_url: The effective url returned by the server.
By default, Tornado will follow redirects, so this URL may
differ from the request URL. It will only be present on a
successful request or if a HTTPError is encountered.
* headers: The response headers. It will only be present on a
successful request or if a HTTPError is encountered.
The flattened error is a dictionary, accessed from kwargs using the
'errinfo' key. It mimics the synapse excinfo output, but without
investigating a stack trace for performance reasons. It contains
the following information:
* err: The Exception class raised during the request.
* errmsg: The str() representation of the exception.
* errfile: Empty string.
* errline: Empty string.
The Hypnos boss job id is a str which can be accessed from kwargs
using the 'jid' key.
Notes:
The following items may be used via kwargs to set request parameters:
* api_args: This should be a dictionary containing any required
or optional arguments the API rquires.
The following items may be passed via kwargs to change the job
execution parameters:
* callback: A function which will be called by the servicing
thread. By default, this will be wrapped to fire boss.err()
if excinfo is present in the callback's kwargs.
* ondone: A function to be executed by the job:fini handler
when the job has been completed. If the api we're firing has an
ingest associated with it, the response data may not be
available to be consumed by the ondone handler.
* job_timeout: A timeout on how long the job can run from the
perspective of the boss. This isn't related to the request
or connect timeouts.
* wrap_callback: By default, the callback function is wrapped to
perform error checking (and fast job failure) in the event of an
error encountered during the request, and additional processing
of the HTTP response data to perform decoding and content-type
processing. If this value is set to false, the decorator will
not be applied to a provided callback function, and the error
handling and additional data procesing will be the
responsibility of any event handlers or the provided callback
function. The fast failure behavior is handled by boss.err()
on the job associated with the API call.
A HTTP body can be provided to the request by passing its contents
in by adding the “req_body” value to api_args argument. See the
Nyx object documentation for more details.
If caching is enabled, the caching will be performed as the first
thing done by the worker thread handling the response data. This
is done separately from the wrap_callback step mentioned above.
Args:
name (str): Name of the API to send a request for.
*args: Additional args passed to the callback functions.
**kwargs: Additional args passed to the callback functions or for
changing the job execution.
Returns:
str: String containing a Job ID which can be used to look up a
job against the Hypnos.web_boss object.
Raises:
NoSuchName: If the requested API name does not exist.
'''
# First, make sure the name is good
nyx = self.getNyxApi(name)
# Fail fast on a bad name before creating a reference in the self.boss
# for the job.
if nyx is None:
raise s_common.NoSuchName(name=name, mesg='Invalid API name')
# Grab things out of kwargs
callback = kwargs.pop('callback', None)
ondone = kwargs.pop('ondone', None)
job_timeout = kwargs.pop('job_timeout', None)
wrap_callback = kwargs.pop('wrap_callback', True)
api_args = kwargs.get('api_args', {})
if not callback:
# Setup the default callback
def default_callback(*cbargs, **cbkwargs):
self.fire(name, **{'args': cbargs, 'kwargs': cbkwargs})
callback = default_callback
# Wrap the callback so that it will fail fast in the case of a request error.
if wrap_callback:
callback = self._webFailRespWrapper(callback)
# If the cache is enabled, wrap the callback so we cache the result before
# The job is executed.
if self.web_cache_enabled:
callback = self._webCacheRespWrapper(callback)
# Construct the job tufo
jid = s_async.jobid()
t = s_async.newtask(callback, *args, **kwargs)
job = self.web_boss.initJob(jid,
task=t,
ondone=ondone,
timeout=job_timeout)
# Create our Async callback function - it enjoys the locals().
def response_nommer(resp):
job_kwargs = job[1]['task'][2]
# Stamp the job id and the web_api_name into the kwargs dictionary.
job_kwargs['web_api_name'] = name
job_kwargs['jid'] = job[0]
if resp.error:
_e = resp.error
_execinfo = {
'err': _e.__class__.__name__,
'errmsg': str(_e),
'errfile': '',
'errline': '',
}
job_kwargs['excinfo'] = _execinfo
resp_dict = self._webFlattenHttpResponse(resp)
job_kwargs['resp'] = resp_dict
self.web_pool.call(self.web_boss._runJob, job)
# Construct the request object
req = nyx.buildHttpRequest(api_args)
self.web_loop.add_callback(self.web_client.fetch, req, response_nommer)
return jid
