def __do_invoke_resource(self,
context,
method,
resource_type,
resource_id=None,
**kwargs):
"""Implements invoke_resource()."""
variables = self.resource_method_to_variables(method,
resource_type,
resource_id=resource_id,
**kwargs)
variables = context.eval(variables)
resource_obj = self.resource_type_to_resource_obj(resource_type)
logging.debug('Calling %s.%s', resource_type, method)
JournalLogger.journal_or_log(
'Requesting {type} {method} {vars}'.format(type=resource_type,
method=method,
vars=variables),
_logging=self.logger.name,
_context='request')
request = getattr(resource_obj(), method)(**variables)
response = request.execute()
JournalLogger.journal_or_log(
json.JSONEncoder(separators=(',', ': ')).encode(response),
_logger=self.logger,
_context='response',
format='json')
return response
def _log_call_method_response(self, method, response):
# pylint: disable=unused-argument
JournalLogger.journal_or_log(
json.JSONEncoder(separators=(',', ': ')).encode(response),
_logger=self.logger,
_context='response',
format='json')
def __do_list_resource(self,
context,
resource_type,
method_variant='list',
item_list_transform=None,
**kwargs):
"""Helper function implementing list_resource()."""
resource_obj = self.resource_type_to_resource_obj(resource_type)
method_container = resource_obj()
variables = self.resource_method_to_variables(method_variant,
resource_type, **kwargs)
variables = context.eval(variables)
request = getattr(method_container, method_variant)(**variables)
all_objects = []
more = ''
while request:
logging.debug('Calling %s.%s',
resource_type,
method_variant,
extra={'citest_journal': {
'nojournal': True
}})
JournalLogger.journal_or_log('Listing {0}{1}'.format(
more, resource_type),
_logger=self.logger,
_context='request')
response = request.execute()
JournalLogger.journal_or_log(
json.JSONEncoder(separators=(',', ': ')).encode(response),
_logger=self.logger,
_context='response',
format='json')
response_items = response.get('items', None)
if response_items is None:
# Assume item reponse is named by the type being listed.
response_items = response.get(resource_type.split('.')[-1], [])
all_items = (item_list_transform(response_items)
if item_list_transform else response_items)
if not isinstance(all_items, list):
all_items = [all_items]
all_objects.extend(all_items)
try:
request = method_container.list_next(request, response)
if request:
logging.debug(
'Iterate over another page of %s',
resource_type,
extra={'citest_journal': {
'nojournal': True
}})
except AttributeError:
request = None
more = ' more '
self.logger.debug('Found total=%d %s', len(all_objects), resource_type)
return all_objects
def __do_call_method(self, method_name, method, context, *pos_args, **kwargs):
"""Helper function implementing call_method()."""
eval_pos_args = context.eval(pos_args)
eval_kwargs = context.eval(kwargs)
arg_text_list = [repr(arg) for arg in eval_pos_args]
arg_text_list.extend(['{0}={1!r}'.format(key, value)
for key, value in eval_kwargs.items()])
arg_text = ','.join(arg_text_list)
JournalLogger.journal_or_log(
'{0}({1})'.format(method_name, arg_text),
_logger=self.logger, _context='request')
response = method(*eval_pos_args, **eval_kwargs)
self._log_call_method_response(method, response)
return response
def __do_list_resource(self, context, resource_type, method_variant='list',
item_list_transform=None, **kwargs):
"""Helper function implementing list_resource()."""
resource_obj = self.resource_type_to_resource_obj(resource_type)
method_container = resource_obj()
variables = self.resource_method_to_variables(
method_variant, resource_type, **kwargs)
variables = context.eval(variables)
request = getattr(method_container, method_variant)(**variables)
all_objects = []
more = ''
while request:
logging.info('Calling %s.%s', resource_type, method_variant,
extra={'citest_journal':{'nojournal':True}})
JournalLogger.journal_or_log(
'Listing {0}{1}'.format(more, resource_type),
_logger=self.logger, _context='request')
response = request.execute()
JournalLogger.journal_or_log(
json.JSONEncoder(
encoding='utf-8', separators=(',', ': ')).encode(response),
_logger=self.logger, _context='response', format='json')
response_items = response.get('items', None)
if response_items is None:
# Assume item reponse is named by the type being listed.
response_items = response.get(resource_type.split('.')[-1], [])
all_items = (item_list_transform(response_items)
if item_list_transform
else response_items)
if not isinstance(all_items, list):
all_items = [all_items]
all_objects.extend(all_items)
try:
request = method_container.list_next(request, response)
if request:
logging.debug('Iterate over another page of %s', resource_type,
extra={'citest_journal':{'nojournal':True}})
except AttributeError:
request = None
more = ' more '
self.logger.info('Found total=%d %s', len(all_objects), resource_type)
return all_objects
def __do_call_method(self, method_name, method, context, *pos_args,
**kwargs):
"""Helper function implementing call_method()."""
eval_pos_args = context.eval(pos_args)
eval_kwargs = context.eval(kwargs)
arg_text_list = [repr(arg) for arg in eval_pos_args]
arg_text_list.extend([
'{0}={1!r}'.format(key, value)
for key, value in eval_kwargs.items()
])
arg_text = ','.join(arg_text_list)
JournalLogger.journal_or_log('{0}({1})'.format(method_name, arg_text),
_logger=self.logger,
_context='request')
response = method(*eval_pos_args, **eval_kwargs)
self._log_call_method_response(method, response)
return response
def __do_invoke_resource(self, context, method, resource_type,
resource_id=None, **kwargs):
"""Implements invoke_resource()."""
variables = self.resource_method_to_variables(
method, resource_type, resource_id=resource_id, **kwargs)
variables = context.eval(variables)
resource_obj = self.resource_type_to_resource_obj(resource_type)
logging.debug('Calling %s.%s', resource_type, method)
JournalLogger.journal_or_log(
'Requesting {type} {method} {vars}'.format(
type=resource_type, method=method, vars=variables),
_logging=self.logger.name,
_context='request')
request = getattr(resource_obj(), method)(**variables)
response = request.execute()
JournalLogger.journal_or_log(
json.JSONEncoder(separators=(',', ': ')).encode(response),
_logger=self.logger, _context='response', format='json')
return response
def run(self, args, trace=True, output_scrubber=None):
"""Run the specified command.
Args:
args: The list of command-line arguments for self.__program.
trace: If True then we should trace the call/response.
Returns:
CliResponseType tuple containing program execution results.
"""
command = self._args_to_full_commandline(args)
log_msg = 'spawn {0} "{1}"'.format(command[0], '" "'.join(command[1:]))
JournalLogger.journal_or_log(log_msg,
_module=self.logger.name,
_alwayslog=trace,
_context='request')
process = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
stdout, stderr = process.communicate()
scrubber = output_scrubber or self.__output_scrubber
if scrubber:
log_msg = 'Scrubbing output with {0}'.format(
scrubber.__class__.__name__)
JournalLogger.journal_or_log(log_msg,
_module=self.logger.name,
_alwayslog=trace)
stdout = scrubber(stdout)
# Strip leading/trailing eolns that program may add to errors and output.
stderr = stderr.strip()
stdout = stdout.strip()
code = process.returncode
# Always log to journal
if stdout and stderr:
which = 'both stdout and stderr'
output_json = {'stdout': stdout, 'stderr': stderr}
else:
which = 'stderr' if stderr else 'stdout'
output_json = stderr if stderr else stdout
if output_json:
JournalLogger.journal_or_log_detail('Result Code {0} / {1}'.format(
code, which),
output_json,
_module=self.logger.name,
_alwayslog=trace,
_context='response')
else:
JournalLogger.journal_or_log(
'Result Code {0} / no ouptut'.format(code),
_module=self.logger.name,
_alwayslog=trace,
_context='response')
return CliResponseType(code, stdout, stderr)
def run(self, args, output_scrubber=None):
"""Run the specified command.
Args:
args: The list of command-line arguments for self.__program.
Returns:
CliResponseType tuple containing program execution results.
"""
command = self._args_to_full_commandline(args)
log_msg = 'spawn {0} "{1}"'.format(command[0], '" "'.join(command[1:]))
JournalLogger.journal_or_log(log_msg,
_logger=self.logger,
_context='request')
process = subprocess.Popen(
command,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
stdout, stderr = process.communicate()
if stdout is not None:
stdout = bytes.decode(stdout)
if stderr is not None:
stderr = bytes.decode(stderr)
scrubber = output_scrubber or self.__output_scrubber
if scrubber:
log_msg = 'Scrubbing output with {0}'.format(scrubber.__class__.__name__)
JournalLogger.journal_or_log(log_msg, _logger=self.logger)
stdout = scrubber(stdout)
# Strip leading/trailing eolns that program may add to errors and output.
stderr = stderr.strip()
stdout = stdout.strip()
code = process.returncode
# Always log to journal
if stdout and stderr:
which = 'both stdout and stderr'
output_json = {'stdout':stdout, 'stderr':stderr}
else:
which = 'stderr' if stderr else 'stdout'
output_json = stderr if stderr else stdout
if output_json:
JournalLogger.journal_or_log_detail(
'Result Code {0} / {1}'.format(code, which), output_json,
_logger=self.logger, _context='response')
else:
JournalLogger.journal_or_log(
'Result Code {0} / no ouptut'.format(code),
_logger=self.logger,
_context='response')
return CliResponseType(code, stdout, stderr)
def _log_call_method_response(self, method, response):
JournalLogger.journal_or_log(
AwsJsonEncoder(separators=(',', ':')).encode(response),
_logger=self.logger,
_context='response',
format='json')
def __send_http_request(self, path, http_type, data=None, headers=None):
"""Send an HTTP message.
Args:
path: [string] The URL path to send to (without network location)
http_type: [string] The HTTP message type (e.g. POST)
data: [string] Data payload to send, if any.
headers: [dict] Headers to write, if any.
Returns:
HttpResponseType
"""
if headers is None:
all_headers = self.__headers
else:
all_headers = self.__headers.copy()
all_headers.update(headers)
if path[0] == '/':
path = path[1:]
url = '{0}/{1}'.format(self.__base_url, path)
encoded_data = str.encode(data) if data is not None else None
req = Request(url=url, data=encoded_data, headers=all_headers)
req.get_method = lambda: http_type
scrubbed_url = self.__http_scrubber.scrub_url(url)
scrubbed_data = self.__http_scrubber.scrub_request(data)
if data is not None:
JournalLogger.journal_or_log_detail(
'{type} {url}'.format(type=http_type, url=scrubbed_url),
scrubbed_data,
_logger=self.logger,
_context='request')
else:
JournalLogger.journal_or_log(
'{type} {url}'.format(type=http_type, url=scrubbed_url),
_logger=self.logger,
_context='request')
if self.__ignore_ssl_cert_verification:
context = ssl._create_unverified_context()
opener = build_opener(HTTPSHandler(context=context), HTTPCookieProcessor())
else:
opener = build_opener(HTTPCookieProcessor())
code = None
output = None
exception = None
headers = None
try:
response = opener.open(req)
code = response.getcode()
output = bytes.decode(response.read())
if sys.version_info[0] > 2:
headers = dict(response.headers.items())
else:
headers = response.info().headers
scrubbed_output = self.__http_scrubber.scrub_response(output)
JournalLogger.journal_or_log_detail(
'HTTP {code}'.format(code=code),
scrubbed_output,
_logger=self.logger,
_context='response')
except HTTPError as ex:
code = ex.getcode()
output = bytes.decode(ex.read())
scrubbed_error = self.__http_scrubber.scrub_response(output)
JournalLogger.journal_or_log_detail(
'HTTP {code}'.format(code=code), scrubbed_error,
_logger=self.logger,
_context='response')
except URLError as ex:
JournalLogger.journal_or_log(
'Caught exception: {ex}\n{stack}'.format(
ex=ex, stack=traceback.format_exc()),
_logger=self.logger)
exception = ex
return HttpResponseType(http_code=code, output=output,
exception=exception, headers=headers)
def _log_call_method_response(self, method, response):
# pylint: disable=unused-argument
JournalLogger.journal_or_log(
json.JSONEncoder(separators=(',', ': ')).encode(response),
_logger=self.logger, _context='response', format='json')
def _log_call_method_response(self, method, response):
JournalLogger.journal_or_log(
AwsJsonEncoder(separators=(',', ':')).encode(response),
_logger=self.logger, _context='response', format='json')
def __send_http_request(self, path, http_type, data=None, headers=None):
"""Send an HTTP message.
Args:
path: [string] The URL path to send to (without network location)
http_type: [string] The HTTP message type (e.g. POST)
data: [string] Data payload to send, if any.
headers: [dict] Headers to write, if any.
Returns:
HttpResponseType
"""
if headers is None:
all_headers = self.__headers
else:
all_headers = self.__headers.copy()
all_headers.update(headers)
if path[0] == '/':
path = path[1:]
url = '{0}/{1}'.format(self.__base_url, path)
req = urllib2.Request(url=url, data=data, headers=all_headers)
req.get_method = lambda: http_type
scrubbed_url = self.__http_scrubber.scrub_url(url)
scrubbed_data = self.__http_scrubber.scrub_request(data)
if data is not None:
JournalLogger.journal_or_log_detail(
'{type} {url}'.format(type=http_type, url=scrubbed_url),
scrubbed_data,
_logger=self.logger,
_context='request')
else:
JournalLogger.journal_or_log(
'{type} {url}'.format(type=http_type, url=scrubbed_url),
_logger=self.logger,
_context='request')
code = None
output = None
exception = None
try:
response = urllib2.urlopen(req)
code = response.getcode()
output = response.read()
scrubbed_output = self.__http_scrubber.scrub_response(output)
JournalLogger.journal_or_log_detail(
'HTTP {code}'.format(code=code),
scrubbed_output,
_logger=self.logger,
_context='response')
except urllib2.HTTPError as ex:
code = ex.getcode()
output = ex.read()
scrubbed_error = self.__http_scrubber.scrub_response(output)
JournalLogger.journal_or_log_detail(
'HTTP {code}'.format(code=code), scrubbed_error,
_logger=self.logger,
_context='response')
except urllib2.URLError as ex:
JournalLogger.journal_or_log(
'Caught exception: {ex}\n{stack}'.format(
ex=ex, stack=traceback.format_exc()),
_logger=self.logger)
exception = ex
return HttpResponseType(code, output, exception)
def _log_call_method_response(self, method, response):
JournalLogger.journal_or_log(AwsJsonEncoder(
encoding='utf-8', separators=(',', ':')).encode(response),
_module=self.logger.name,
_context='response',
format='json')
def __send_http_request(self, path, http_type, data=None, headers=None):
"""Send an HTTP message.
Args:
path: [string] The URL path to send to (without network location)
http_type: [string] The HTTP message type (e.g. POST)
data: [string] Data payload to send, if any.
headers: [dict] Headers to write, if any.
Returns:
HttpResponseType
"""
if headers is None:
all_headers = self.__headers
else:
all_headers = self.__headers.copy()
all_headers.update(headers)
if path[0] == '/':
path = path[1:]
url = '{0}/{1}'.format(self.__base_url, path)
req = urllib2.Request(url=url, data=data, headers=all_headers)
req.get_method = lambda: http_type
scrubbed_url = self.__http_scrubber.scrub_url(url)
scrubbed_data = self.__http_scrubber.scrub_request(data)
if data is not None:
JournalLogger.journal_or_log_detail('{type} {url}'.format(
type=http_type, url=scrubbed_url),
scrubbed_data,
_logger=self.logger,
_context='request')
else:
JournalLogger.journal_or_log('{type} {url}'.format(
type=http_type, url=scrubbed_url),
_logger=self.logger,
_context='request')
code = None
output = None
exception = None
try:
response = urllib2.urlopen(req)
code = response.getcode()
output = response.read()
scrubbed_output = self.__http_scrubber.scrub_response(output)
JournalLogger.journal_or_log_detail(
'HTTP {code}'.format(code=code),
scrubbed_output,
_logger=self.logger,
_context='response')
except urllib2.HTTPError as ex:
code = ex.getcode()
output = ex.read()
scrubbed_error = self.__http_scrubber.scrub_response(output)
JournalLogger.journal_or_log_detail(
'HTTP {code}'.format(code=code),
scrubbed_error,
_logger=self.logger,
_context='response')
except urllib2.URLError as ex:
JournalLogger.journal_or_log(
'Caught exception: {ex}\n{stack}'.format(
ex=ex, stack=traceback.format_exc()),
_logger=self.logger)
exception = ex
return HttpResponseType(code, output, exception)
def __send_http_request(self, path, http_type, data=None, headers=None):
"""Send an HTTP message.
Args:
path: [string] The URL path to send to (without network location)
http_type: [string] The HTTP message type (e.g. POST)
data: [string] Data payload to send, if any.
headers: [dict] Headers to write, if any.
Returns:
HttpResponseType
"""
if headers is None:
all_headers = self.__headers
else:
all_headers = self.__headers.copy()
all_headers.update(headers)
if path[0] == '/':
path = path[1:]
url = '{0}/{1}'.format(self.__base_url, path)
encoded_data = str.encode(data) if data is not None else None
req = Request(url=url, data=encoded_data, headers=all_headers)
req.get_method = lambda: http_type
scrubbed_url = self.__http_scrubber.scrub_url(url)
scrubbed_data = self.__http_scrubber.scrub_request(data)
if data is not None:
JournalLogger.journal_or_log_detail(
'{type} {url}'.format(type=http_type, url=scrubbed_url),
scrubbed_data,
_logger=self.logger,
_context='request')
else:
JournalLogger.journal_or_log(
'{type} {url}'.format(type=http_type, url=scrubbed_url),
_logger=self.logger,
_context='request')
if self.__ignore_ssl_cert_verification:
context = ssl._create_unverified_context()
opener = build_opener(HTTPSHandler(context=context), HTTPCookieProcessor())
else:
opener = build_opener(HTTPCookieProcessor())
code = None
output = None
exception = None
headers = None
try:
response = opener.open(req)
code = response.getcode()
output = bytes.decode(response.read())
if sys.version_info[0] > 2:
headers = dict(response.headers.items())
else:
headers = response.info().headers
scrubbed_output = self.__http_scrubber.scrub_response(output)
JournalLogger.journal_or_log_detail(
'HTTP {code}'.format(code=code),
scrubbed_output,
_logger=self.logger,
_context='response')
except HTTPError as ex:
code = ex.getcode()
output = bytes.decode(ex.read())
scrubbed_error = self.__http_scrubber.scrub_response(output)
JournalLogger.journal_or_log_detail(
'HTTP {code}'.format(code=code), scrubbed_error,
_logger=self.logger,
_context='response')
except URLError as ex:
JournalLogger.journal_or_log(
'Caught exception: {ex}\n{stack}'.format(
ex=ex, stack=traceback.format_exc()),
_logger=self.logger)
exception = ex
return HttpResponseType(http_code=code, output=output,
exception=exception, headers=headers)