def cache_request(self,*args,**kwargs):
'''
Just build a redis key from the endpoint + arguments and
check the cache. If not found cache.
if an endpoint is included as the first argument that is fine
(backwards compatible with old qualysapi style) but the preferred
method is to use the keyword endpoint = 'api-endpoint' as in the below
example.
'''
conn = kwargs.pop('connection', None)
if not conn:
conn = self.getConnection(**kwargs)
if len(args):
endpoint = util.preformat_call(args[0])
if self.__config.__defaults__.get(endpoint, None) is None:
raise exceptions.QCacheExceptio('first argument for args \'' + endpoint \
+ '\' not a valid qualys api endpoint.')
else:
endpoint = kwargs.pop('endpoint', None)
if not endpoint:
raise exceptions.QCacheException('can\'t find your endpoint in args or keyword args')
#check the cache
data = kwargs.get('data', None)
if data is None: data = {}
key = self.build_redis_key(endpoint, **data)
result = None
if kwargs.pop('force_cache_refresh', False):
conn.delete(key)
else:
result = conn.get(key)
#not in cache or force refresh then go to qualys
if not result:
logging.debug('Connecting with username \'' + \
self.__config.qusername + '\'')
qgs = connect(
username = self.__config.qusername,
password = self.__config.qpassword)
result = qgs.request(endpoint, data=data)
conn.set(key,result)
# set the default expiration on the cache
if key in self.__config.__defaults__:
conn.expire(key, self.__config.__defaults__[key])
else:
conn.expire(key, self.__config.__defaults__[endpoint])
return result
def cache_flush(self, *args, **kwargs):
'''
Deletes a key or the entire database specified by the configuration
'''
conn = kwargs.get('connection', None)
if not conn: conn = self.getConnection(**kwargs)
if kwargs.get('all', False):
return conn.flushdb()
else:
endpoint = None
if len(args):
endpoint = util.preformat_call(args[0])
if self.__config.__defaults__.get(endpoint, None) is None:
raise exceptions.QCacheException('first argument for args \'' + endpoint \
+ '\' not a valid qualys api endpoint.')
else:
endpoint = kwargs.pop('endpoint', None)
if not endpoint:
raise exceptions.QCacheException('can\'t find your endpoint in args or keyword args')
data = kwargs.get('data', None)
key = self.build_redis_key(endpoint, **data)
return conn.delete(key)
def request(self, api_call, data=None, api_version=None, http_method=None, concurrent_scans_retries=0,
concurrent_scans_retry_delay=0):
""" Return QualysGuard API response.
"""
logger.debug('api_call =\n%s' % api_call)
logger.debug('api_version =\n%s' % api_version)
logger.debug('data %s =\n %s' % (type(data), str(data)))
logger.debug('http_method =\n%s' % http_method)
logger.debug('concurrent_scans_retries =\n%s' % str(concurrent_scans_retries))
logger.debug('concurrent_scans_retry_delay =\n%s' % str(concurrent_scans_retry_delay))
concurrent_scans_retries = int(concurrent_scans_retries)
concurrent_scans_retry_delay = int(concurrent_scans_retry_delay)
#
# Determine API version.
# Preformat call.
api_call = util.preformat_call(api_call)
if api_version:
# API version specified, format API version inputted.
api_version = self.format_api_version(api_version)
else:
# API version not specified, determine automatically.
api_version = self.which_api_version(api_call)
#
# Set up base url.
url = self.url_api_version(api_version)
#
# Set up headers.
headers = {"X-Requested-With": "Parag Baxi QualysAPI (python) v%s" % (qualysapi.version.__version__,)}
logger.debug('headers =\n%s' % (str(headers)))
# Portal API takes in XML text, requiring custom header.
if api_version in ('am', 'was'):
headers['Content-type'] = 'text/xml'
#
# Set up http request method, if not specified.
if not http_method:
http_method = self.format_http_method(api_version, api_call, data)
logger.debug('http_method =\n%s' % http_method)
#
# Format API call.
api_call = self.format_call(api_version, api_call)
logger.debug('api_call =\n%s' % (api_call))
# Append api_call to url.
url += api_call
#
# Format data, if applicable.
if data is not None:
data = self.format_payload(api_version, data)
# Make request at least once (more if concurrent_retry is enabled).
retries = 0
while retries <= concurrent_scans_retries:
# Make request.
logger.debug('url =\n%s' % (str(url)))
logger.debug('data =\n%s' % (str(data)))
logger.debug('headers =\n%s' % (str(headers)))
if http_method == 'get':
# GET
logger.debug('GET request.')
request = self.session.get(url, params=data, auth=self.auth, headers=headers, proxies=self.proxies)
else:
# POST
logger.debug('POST request.')
# Make POST request.
request = self.session.post(url, data=data, auth=self.auth, headers=headers, proxies=self.proxies)
logger.debug('response headers =\n%s' % (str(request.headers)))
#
# Remember how many times left user can make against api_call.
try:
self.__rate_limit_remaining[api_call] = int(request.headers['x-ratelimit-remaining'])
logger.debug('rate limit for api_call, %s = %s' % (api_call, self.__rate_limit_remaining[api_call]))
except KeyError as e:
# Likely a bad api_call.
logger.debug(e)
pass
except TypeError as e:
# Likely an asset search api_call.
logger.debug(e)
pass
# handle authentication exception special and early
if request.status_code == 401:
request.close()
raise QualysAuthenticationException('Bad Qualys username or \
password.')
# Response received
for buffblock in request.iter_content(chunk_size=8192,
decode_unicode=True):
logger.debug(pprint.pformat(buffblock))
response = b"".join([buffblock for buffblock in
request.iter_content(chunk_size=8192, decode_unicode=False)])
#logging.debug(pprint.pformat(response))
# for buffblock in request.iter_content(chunk_size=8192, decode_unicode=True):
# response_str += unicode(buffblock)
# response = str(request.content)
#logger.debug('response text =\n%s' % (response))
# Keep track of how many retries.
retries += 1
# Check for concurrent scans limit.
if not (b'<responseCode>INVALID_REQUEST</responseCode>' in response and \
b'<errorMessage>You have reached the maximum number of concurrent running scans' in response and \
b'<errorResolution>Please wait until your previous scans have completed</errorResolution>' in response):
# Did not hit concurrent scan limit.
break
else:
# Hit concurrent scan limit.
logger.critical(response)
# If trying again, delay next try by concurrent_scans_retry_delay.
if retries <= concurrent_scans_retries:
logger.warning('Waiting %d seconds until next try.' % \
concurrent_scans_retry_delay)
time.sleep(concurrent_scans_retry_delay)
# Inform user of how many retries.
logger.critical('Retry #%d' % retries)
else:
# Ran out of retries. Let user know.
print('Alert! Ran out of concurrent_scans_retries!')
logger.critical('Alert! Ran out of concurrent_scans_retries!')
return False
# Check to see if there was an error.
try:
request.raise_for_status()
except requests.exceptions.HTTPError as e:
# Error
# only do this logging if we don't know the reason for the
# exception (have already handled it IOTW)
logger.exception('Error! Received a 4XX client error or 5XX server error response.')
logger.error(' Content = \n%s' % response)
logger.error(' Headers = \n%s' % str(request.headers))
request.raise_for_status()
if b'<RETURN status="FAILED" number="2007">' in response:
logger.error('Error! Your IP address is not in the list of secure IPs. Manager must include this IP (QualysGuard VM > Users > Security).')
logger.error(' Content = \n%s' % response)
logger.error(' Headers = \n%s' % str(request.headers))
request.close()
return response
def request(self,
api_call,
data=None,
api_version=None,
http_method=None,
concurrent_scans_retries=0,
concurrent_scans_retry_delay=0):
""" Return QualysGuard API response.
"""
logger.debug('api_call =\n%s' % api_call)
logger.debug('api_version =\n%s' % api_version)
logger.debug('data %s =\n %s' % (type(data), str(data)))
logger.debug('http_method =\n%s' % http_method)
logger.debug('concurrent_scans_retries =\n%s' %
str(concurrent_scans_retries))
logger.debug('concurrent_scans_retry_delay =\n%s' %
str(concurrent_scans_retry_delay))
concurrent_scans_retries = int(concurrent_scans_retries)
concurrent_scans_retry_delay = int(concurrent_scans_retry_delay)
#
# Determine API version.
# Preformat call.
api_call = util.preformat_call(api_call)
if api_version:
# API version specified, format API version inputted.
api_version = self.format_api_version(api_version)
else:
# API version not specified, determine automatically.
api_version = self.which_api_version(api_call)
#
# Set up base url.
url = self.url_api_version(api_version)
#
# Set up headers.
headers = {
"X-Requested-With":
"Parag Baxi QualysAPI (python) v%s" %
(qualysapi.version.__version__, )
}
logger.debug('headers =\n%s' % (str(headers)))
# Portal API takes in XML text, requiring custom header.
if api_version in ('am', 'was', 'am2'):
headers['Content-type'] = 'text/xml'
#
# Set up http request method, if not specified.
if not http_method:
http_method = self.format_http_method(api_version, api_call, data)
logger.debug('http_method =\n%s' % http_method)
#
# Format API call.
api_call = self.format_call(api_version, api_call)
logger.debug('api_call =\n%s' % (api_call))
# Append api_call to url.
url += api_call
#
# Format data, if applicable.
if data is not None:
data = self.format_payload(api_version, data)
# Make request at least once (more if concurrent_retry is enabled).
retries = 0
while retries <= concurrent_scans_retries:
# Make request.
logger.debug('url =\n%s' % (str(url)))
logger.debug('data =\n%s' % (str(data)))
logger.debug('headers =\n%s' % (str(headers)))
if http_method == 'get':
# GET
logger.debug('GET request.')
request = self.session.get(url,
params=data,
auth=self.auth,
headers=headers,
proxies=self.proxies)
else:
# POST
logger.debug('POST request.')
# Make POST request.
request = self.session.post(url,
data=data,
auth=self.auth,
headers=headers,
proxies=self.proxies)
logger.debug('response headers =\n%s' % (str(request.headers)))
#
# Remember how many times left user can make against api_call.
try:
self.rate_limit_remaining[api_call] = int(
request.headers['x-ratelimit-remaining'])
logger.debug('rate limit for api_call, %s = %s' %
(api_call, self.rate_limit_remaining[api_call]))
if (self.rate_limit_remaining[api_call] > rate_warn_threshold):
logger.debug(
'rate limit for api_call, %s = %s' %
(api_call, self.rate_limit_remaining[api_call]))
elif (self.rate_limit_remaining[api_call] <=
rate_warn_threshold) and (
self.rate_limit_remaining[api_call] > 0):
logger.warning(
'Rate limit is about to being reached (remaining api calls = %s)'
% self.rate_limit_remaining[api_call])
elif self.rate_limit_remaining[api_call] <= 0:
logger.critical(
'ATTENTION! RATE LIMIT HAS BEEN REACHED (remaining api calls = %s)!'
% self.rate_limit_remaining[api_call])
except KeyError as e:
# Likely a bad api_call.
logger.debug(e)
pass
except TypeError as e:
# Likely an asset search api_call.
logger.debug(e)
pass
# handle authentication exception special and early
if request.status_code == 401:
request.close()
raise QualysAuthenticationException('Bad Qualys username or \
password.')
# Response received
for buffblock in request.iter_content(chunk_size=8192,
decode_unicode=True):
logger.debug(pprint.pformat(buffblock))
response = b"".join([
buffblock
for buffblock in request.iter_content(chunk_size=8192,
decode_unicode=False)
])
#logging.debug(pprint.pformat(response))
# for buffblock in request.iter_content(chunk_size=8192, decode_unicode=True):
# response_str += unicode(buffblock)
# response = str(request.content)
#logger.debug('response text =\n%s' % (response))
# Keep track of how many retries.
retries += 1
# Check for concurrent scans limit.
if not ('<responseCode>INVALID_REQUEST</responseCode>' in response
and
'<errorMessage>You have reached the maximum number of concurrent running scans'
in response and
'<errorResolution>Please wait until your previous scans have completed</errorResolution>'
in response):
# Did not hit concurrent scan limit.
break
else:
# Hit concurrent scan limit.
logger.critical(response)
# If trying again, delay next try by concurrent_scans_retry_delay.
if retries <= concurrent_scans_retries:
logger.warning('Waiting %d seconds until next try.' % \
concurrent_scans_retry_delay)
time.sleep(concurrent_scans_retry_delay)
# Inform user of how many retries.
logger.critical('Retry #%d' % retries)
else:
# Ran out of retries. Let user know.
print('Alert! Ran out of concurrent_scans_retries!')
logger.critical(
'Alert! Ran out of concurrent_scans_retries!')
return False
# Check to see if there was an error.
try:
request.raise_for_status()
except requests.exceptions.HTTPError as e:
# Error
print(
'Error! Received a 4XX client error or 5XX server error response.'
)
print('Content = \n', response)
logger.error('Content = \n%s' % response)
print('Headers = \n', request.headers)
logger.error('Headers = \n%s' % str(request.headers))
request.raise_for_status()
if '<RETURN status="FAILED" number="2007">' in response:
print(
'Error! Your IP address is not in the list of secure IPs. Manager must include this IP (QualysGuard VM > Users > Security).'
)
print('Content = \n', response)
logger.error('Content = \n%s' % response)
print('Headers = \n', request.headers)
logger.error('Headers = \n%s' % str(request.headers))
return False
return response
