def addRequest(self, proxy):
'''push another request proxy thread onto the running stack.'''
if not issubclass(proxy, QualysStatusMonitor):
raise exceptions.QualysFrameworkException('\'%s\' is not a \
subclass of QualysStatusMonitor.' % type(proxy).__name__)
else:
self.monitors.append(proxy)
proxy.setPool(self.pool_sema)
proxy.start()
def __init__(self, *args, **kwargs):
''' Simple interface to threading out multiple QualysStatusMonitor
classes. We build out a thread pool of them and then keep an eye on
them and clean them up when they finish. This class is not designed to
be subclassed or to do anything with results. If you want that,
your're going to have to write your own threadpool manager.
Params:
request_proxies -- A list of QualysStatusMonitor or subclasses.
max_sockets -- (Optional) The maximum number of concurrent requests to
allow to qualys API servers. This is default at 10.
callback -- (Optional) A callback that recieves a result set of
objects when the monitored qualys process finishes and returns actual
results. This is discouraged in favor of subclassing a consumer
process, but for small result sets or tasks requiring more than one
result set it can be useful or even necessary.
'''
max_sockets = 20
request_proxies = None
if len(args):
request_proxies = args[0]
else:
request_proxies = kwargs.pop('request_proxies', [])
if not request_proxies: # should catch none or []
raise exceptions.QualysFrameworkException('You have to pass in \
QualysStatusMonitor objects or subclasses...')
# maximum request sockets...
self.max_sockets = kwargs.pop('max_sockets', self.max_sockets)
self.pool_sema = threading.BoundedSemaphore(value=max_sockets)
for proxy in request_proxies:
if not issubclass(proxy, QualysStatusMonitor):
raise exceptions.QualysFrameworkException('\'%s\' is not a \
subclass of QualysStatusMonitor.' % type(proxy).__name__)
else:
self.monitors.append(proxy)
proxy.setPool(self.pool_sema)
proxy.start()
def __init__(self, qconfig, **kwargs):
'''
initialize this report runner.
@Parms
Parent Params:
qconfig -- the current api configuration from the parent threadpool.
Keyword Params (passed to the action object):
'''
if 'mapr' not in kwargs:
raise exceptions.QualysFrameworkException('A map result is \
required for the report runner to monitor and update.')
self.__mapr = kwargs.pop('mapr')
super(QualysStatusMonitor, self).__init__(qconfig,
**kwargs) #pass to parent
def launchMapReports(self, **kwargs):
'''
This is a type of automation function that should really be used in a
process server or manager context. Consider yourself warned.
This function gatheres a list of finished maps, turns them into json
strings, stores them in redis and then performs a series of automation
tasks on them.
Steps followed:
* get a list of finished maps from Qualys. Serialize
each map object as a json string to redis using a key generated from
the map name. This means that only the most recent run of a map with a
given name is cached or operated on by default.
* after the maps have been cached, a filter is applied to the map
objects including or excluding name patterns or date ranges.
* the resulting filtered list of maps is handed off to a report running
multiprocessing queue which will ensure that reports are started for
each of the maps. Each process will respond to qualys indications of
concurrent report quests by sleeping and periodically checking to see
if qualys is ready to begin another report. After a report is started,
the process will periodically check to see if the report is finished
and then download the report. IDs will be stored with and associated
with the map and report.
* each process will update the cache with the current state of the map
and report each time it wakes up. This allows processes which want to
consume available or specific map reports to simply check if the report is
available for processing against the cache and continue sleeping.
* Consumption of specific map reports is outside the purview of this
process manager.
@params
include_pattern -- an optional pattern by which to include map names.
exclude_pattern -- an optional pattern by which to exclude map names.
map_refs -- an optional list of map references to operate on.
map_names -- an optional list of map names to operate on.
sleep_period -- override the default sleep period for map report
checking. Each processes will sleep for 30 minutes by default.
@returns -- a list of @see qualysapi.api_objects.Maps that this processes
is or was operating on when called. If this call is non-blocking then
anything other than the name, ref, and map date will almost certainly
be out of date with the cache (map report ids will need to be refreshed from
the cache, so think of the results returned as useful for reference to
state rather than completed states.)
'''
# verify we have a cache connection since a cache is required for this
from qualysapi import qcache
if not isinstance(self.conn, qcache.APICacheInstance):
raise exceptions.QualysFrameworkException('This method requires \
that you use the redis cache.')
maps = self.listMaps()
# filter the list to only include those we want
if include_pattern in kwargs:
#TODO: compile regex and filter matches
pass
if exclude_pattern in kwargs:
#TODO: compile regex and filter matches
pass
if 'map_refs' in kwargs:
maps[:] = [mapr for mapr in maps if mapr.map_ref in \
kwargs.get('map_refs', []) ]
if 'map_names' in kwargs:
maps[:] = [mapr for mapr in maps if mapr.name in \
kwargs.get('map_names', []) ]
def parseResponse(self, **kwargs):
'''
An internal utility method that implements an lxml parser capable of
handling streams and mapping objects to elements.
Please note that this utiliy is only capable of parsing known Qualys
API DTDs properly.
@Params
source -- An api endpoint (mapped using the api_methods sets) or an IO
source. If this is an instance of str it is treated as an api
endpoint, otherwise it is treated as a file-like object yielding binary
xml data. This should be sufficient to allow any kind of
processing while still being convenient for standard uses.
block -- an optional parameter which binds the caller to the parse
buffer for consumption. It is generally assumed by the design that
parse consumers will handle themselves internally and that this method
can return once it kicks off an action. When block is set to True,
however, this method will join the parse buffer and wait for the
consumers to clear the queue before returning. By default this is true
for ease of implementation.
completion_callback -- an optional method that gets called when consumption
of a parse completes. This method will receive all of the objects
handled by the buffer consumers as a callback rather than a threaded
parse consumer.
'''
source = kwargs.pop('source', None)
if not source:
raise QualysException('No source file or URL or raw stream found.')
block = kwargs.pop('block', True)
callback = kwargs.pop('completion_callback', None)
#TODO: consider passing in an import_buffer for thread management reuse
#of this object
#TODO: consider replacing this requirement
if not block and not callback:
raise exceptions.QualysFrameworkException("A callback outlet is \
required for nonblocking calls to the parser/consumer framework.")
#select the response file-like object
response = None
if isinstance(source, str):
response = self.stream_request(source, **kwargs)
else:
response = source
if self.import_buffer is None:
self.import_buffer = ImportBuffer(callback=callback)
else:
self.import_buffer.setCallback(callback)
block = kwargs.pop('block', True)
callback = kwargs.pop('completion_callback', None)
if not block and not callback:
raise exceptions.QualysFrameworkException("A callback outlet is \
required for nonblocking calls to the parser/consumer framework.")
clear_ok = False
context = lxml.etree.iterparse(response, events=('end', ))
for event, elem in context:
# Use QName to avoid specifying or stripping the namespace, which we don't need
if lxml.etree.QName(elem.tag).localname.upper() in obj_elem_map:
self.import_buffer.add(obj_elem_map[lxml.etree.QName(
elem.tag).localname.upper()](elem=elem))
clear_ok = True
if clear_ok:
elem.clear() #don't fill up a dom we don't need.
clear_ok = False
return self.import_buffer.finish() if block else self.import_buffer
def singleRequestResponse(self):
'''This is the method ot override in your implementation.'''
raise exceptions.QualysFrameworkException('Abstract thread subclass. \
You need to implement your own subclass.')