def __do_invoke_resource(self, context, method, resource_type, resource_id=None, **kwargs): """Implements invoke_resource().""" variables = self.resource_method_to_variables(method, resource_type, resource_id=resource_id, **kwargs) variables = context.eval(variables) resource_obj = self.resource_type_to_resource_obj(resource_type) logging.debug('Calling %s.%s', resource_type, method) JournalLogger.journal_or_log( 'Requesting {type} {method} {vars}'.format(type=resource_type, method=method, vars=variables), _logging=self.logger.name, _context='request') request = getattr(resource_obj(), method)(**variables) response = request.execute() JournalLogger.journal_or_log( json.JSONEncoder(separators=(',', ': ')).encode(response), _logger=self.logger, _context='response', format='json') return response
def _log_call_method_response(self, method, response): # pylint: disable=unused-argument JournalLogger.journal_or_log( json.JSONEncoder(separators=(',', ': ')).encode(response), _logger=self.logger, _context='response', format='json')
def __do_list_resource(self, context, resource_type, method_variant='list', item_list_transform=None, **kwargs): """Helper function implementing list_resource().""" resource_obj = self.resource_type_to_resource_obj(resource_type) method_container = resource_obj() variables = self.resource_method_to_variables(method_variant, resource_type, **kwargs) variables = context.eval(variables) request = getattr(method_container, method_variant)(**variables) all_objects = [] more = '' while request: logging.debug('Calling %s.%s', resource_type, method_variant, extra={'citest_journal': { 'nojournal': True }}) JournalLogger.journal_or_log('Listing {0}{1}'.format( more, resource_type), _logger=self.logger, _context='request') response = request.execute() JournalLogger.journal_or_log( json.JSONEncoder(separators=(',', ': ')).encode(response), _logger=self.logger, _context='response', format='json') response_items = response.get('items', None) if response_items is None: # Assume item reponse is named by the type being listed. response_items = response.get(resource_type.split('.')[-1], []) all_items = (item_list_transform(response_items) if item_list_transform else response_items) if not isinstance(all_items, list): all_items = [all_items] all_objects.extend(all_items) try: request = method_container.list_next(request, response) if request: logging.debug( 'Iterate over another page of %s', resource_type, extra={'citest_journal': { 'nojournal': True }}) except AttributeError: request = None more = ' more ' self.logger.debug('Found total=%d %s', len(all_objects), resource_type) return all_objects
def __do_call_method(self, method_name, method, context, *pos_args, **kwargs): """Helper function implementing call_method().""" eval_pos_args = context.eval(pos_args) eval_kwargs = context.eval(kwargs) arg_text_list = [repr(arg) for arg in eval_pos_args] arg_text_list.extend(['{0}={1!r}'.format(key, value) for key, value in eval_kwargs.items()]) arg_text = ','.join(arg_text_list) JournalLogger.journal_or_log( '{0}({1})'.format(method_name, arg_text), _logger=self.logger, _context='request') response = method(*eval_pos_args, **eval_kwargs) self._log_call_method_response(method, response) return response
def __do_list_resource(self, context, resource_type, method_variant='list', item_list_transform=None, **kwargs): """Helper function implementing list_resource().""" resource_obj = self.resource_type_to_resource_obj(resource_type) method_container = resource_obj() variables = self.resource_method_to_variables( method_variant, resource_type, **kwargs) variables = context.eval(variables) request = getattr(method_container, method_variant)(**variables) all_objects = [] more = '' while request: logging.info('Calling %s.%s', resource_type, method_variant, extra={'citest_journal':{'nojournal':True}}) JournalLogger.journal_or_log( 'Listing {0}{1}'.format(more, resource_type), _logger=self.logger, _context='request') response = request.execute() JournalLogger.journal_or_log( json.JSONEncoder( encoding='utf-8', separators=(',', ': ')).encode(response), _logger=self.logger, _context='response', format='json') response_items = response.get('items', None) if response_items is None: # Assume item reponse is named by the type being listed. response_items = response.get(resource_type.split('.')[-1], []) all_items = (item_list_transform(response_items) if item_list_transform else response_items) if not isinstance(all_items, list): all_items = [all_items] all_objects.extend(all_items) try: request = method_container.list_next(request, response) if request: logging.debug('Iterate over another page of %s', resource_type, extra={'citest_journal':{'nojournal':True}}) except AttributeError: request = None more = ' more ' self.logger.info('Found total=%d %s', len(all_objects), resource_type) return all_objects
def __do_call_method(self, method_name, method, context, *pos_args, **kwargs): """Helper function implementing call_method().""" eval_pos_args = context.eval(pos_args) eval_kwargs = context.eval(kwargs) arg_text_list = [repr(arg) for arg in eval_pos_args] arg_text_list.extend([ '{0}={1!r}'.format(key, value) for key, value in eval_kwargs.items() ]) arg_text = ','.join(arg_text_list) JournalLogger.journal_or_log('{0}({1})'.format(method_name, arg_text), _logger=self.logger, _context='request') response = method(*eval_pos_args, **eval_kwargs) self._log_call_method_response(method, response) return response
def __do_invoke_resource(self, context, method, resource_type, resource_id=None, **kwargs): """Implements invoke_resource().""" variables = self.resource_method_to_variables( method, resource_type, resource_id=resource_id, **kwargs) variables = context.eval(variables) resource_obj = self.resource_type_to_resource_obj(resource_type) logging.debug('Calling %s.%s', resource_type, method) JournalLogger.journal_or_log( 'Requesting {type} {method} {vars}'.format( type=resource_type, method=method, vars=variables), _logging=self.logger.name, _context='request') request = getattr(resource_obj(), method)(**variables) response = request.execute() JournalLogger.journal_or_log( json.JSONEncoder(separators=(',', ': ')).encode(response), _logger=self.logger, _context='response', format='json') return response
def run(self, args, trace=True, output_scrubber=None): """Run the specified command. Args: args: The list of command-line arguments for self.__program. trace: If True then we should trace the call/response. Returns: CliResponseType tuple containing program execution results. """ command = self._args_to_full_commandline(args) log_msg = 'spawn {0} "{1}"'.format(command[0], '" "'.join(command[1:])) JournalLogger.journal_or_log(log_msg, _module=self.logger.name, _alwayslog=trace, _context='request') process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) stdout, stderr = process.communicate() scrubber = output_scrubber or self.__output_scrubber if scrubber: log_msg = 'Scrubbing output with {0}'.format( scrubber.__class__.__name__) JournalLogger.journal_or_log(log_msg, _module=self.logger.name, _alwayslog=trace) stdout = scrubber(stdout) # Strip leading/trailing eolns that program may add to errors and output. stderr = stderr.strip() stdout = stdout.strip() code = process.returncode # Always log to journal if stdout and stderr: which = 'both stdout and stderr' output_json = {'stdout': stdout, 'stderr': stderr} else: which = 'stderr' if stderr else 'stdout' output_json = stderr if stderr else stdout if output_json: JournalLogger.journal_or_log_detail('Result Code {0} / {1}'.format( code, which), output_json, _module=self.logger.name, _alwayslog=trace, _context='response') else: JournalLogger.journal_or_log( 'Result Code {0} / no ouptut'.format(code), _module=self.logger.name, _alwayslog=trace, _context='response') return CliResponseType(code, stdout, stderr)
def run(self, args, output_scrubber=None): """Run the specified command. Args: args: The list of command-line arguments for self.__program. Returns: CliResponseType tuple containing program execution results. """ command = self._args_to_full_commandline(args) log_msg = 'spawn {0} "{1}"'.format(command[0], '" "'.join(command[1:])) JournalLogger.journal_or_log(log_msg, _logger=self.logger, _context='request') process = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) stdout, stderr = process.communicate() if stdout is not None: stdout = bytes.decode(stdout) if stderr is not None: stderr = bytes.decode(stderr) scrubber = output_scrubber or self.__output_scrubber if scrubber: log_msg = 'Scrubbing output with {0}'.format(scrubber.__class__.__name__) JournalLogger.journal_or_log(log_msg, _logger=self.logger) stdout = scrubber(stdout) # Strip leading/trailing eolns that program may add to errors and output. stderr = stderr.strip() stdout = stdout.strip() code = process.returncode # Always log to journal if stdout and stderr: which = 'both stdout and stderr' output_json = {'stdout':stdout, 'stderr':stderr} else: which = 'stderr' if stderr else 'stdout' output_json = stderr if stderr else stdout if output_json: JournalLogger.journal_or_log_detail( 'Result Code {0} / {1}'.format(code, which), output_json, _logger=self.logger, _context='response') else: JournalLogger.journal_or_log( 'Result Code {0} / no ouptut'.format(code), _logger=self.logger, _context='response') return CliResponseType(code, stdout, stderr)
def _log_call_method_response(self, method, response): JournalLogger.journal_or_log( AwsJsonEncoder(separators=(',', ':')).encode(response), _logger=self.logger, _context='response', format='json')
def __send_http_request(self, path, http_type, data=None, headers=None): """Send an HTTP message. Args: path: [string] The URL path to send to (without network location) http_type: [string] The HTTP message type (e.g. POST) data: [string] Data payload to send, if any. headers: [dict] Headers to write, if any. Returns: HttpResponseType """ if headers is None: all_headers = self.__headers else: all_headers = self.__headers.copy() all_headers.update(headers) if path[0] == '/': path = path[1:] url = '{0}/{1}'.format(self.__base_url, path) encoded_data = str.encode(data) if data is not None else None req = Request(url=url, data=encoded_data, headers=all_headers) req.get_method = lambda: http_type scrubbed_url = self.__http_scrubber.scrub_url(url) scrubbed_data = self.__http_scrubber.scrub_request(data) if data is not None: JournalLogger.journal_or_log_detail( '{type} {url}'.format(type=http_type, url=scrubbed_url), scrubbed_data, _logger=self.logger, _context='request') else: JournalLogger.journal_or_log( '{type} {url}'.format(type=http_type, url=scrubbed_url), _logger=self.logger, _context='request') if self.__ignore_ssl_cert_verification: context = ssl._create_unverified_context() opener = build_opener(HTTPSHandler(context=context), HTTPCookieProcessor()) else: opener = build_opener(HTTPCookieProcessor()) code = None output = None exception = None headers = None try: response = opener.open(req) code = response.getcode() output = bytes.decode(response.read()) if sys.version_info[0] > 2: headers = dict(response.headers.items()) else: headers = response.info().headers scrubbed_output = self.__http_scrubber.scrub_response(output) JournalLogger.journal_or_log_detail( 'HTTP {code}'.format(code=code), scrubbed_output, _logger=self.logger, _context='response') except HTTPError as ex: code = ex.getcode() output = bytes.decode(ex.read()) scrubbed_error = self.__http_scrubber.scrub_response(output) JournalLogger.journal_or_log_detail( 'HTTP {code}'.format(code=code), scrubbed_error, _logger=self.logger, _context='response') except URLError as ex: JournalLogger.journal_or_log( 'Caught exception: {ex}\n{stack}'.format( ex=ex, stack=traceback.format_exc()), _logger=self.logger) exception = ex return HttpResponseType(http_code=code, output=output, exception=exception, headers=headers)
def _log_call_method_response(self, method, response): # pylint: disable=unused-argument JournalLogger.journal_or_log( json.JSONEncoder(separators=(',', ': ')).encode(response), _logger=self.logger, _context='response', format='json')
def _log_call_method_response(self, method, response): JournalLogger.journal_or_log( AwsJsonEncoder(separators=(',', ':')).encode(response), _logger=self.logger, _context='response', format='json')
def __send_http_request(self, path, http_type, data=None, headers=None): """Send an HTTP message. Args: path: [string] The URL path to send to (without network location) http_type: [string] The HTTP message type (e.g. POST) data: [string] Data payload to send, if any. headers: [dict] Headers to write, if any. Returns: HttpResponseType """ if headers is None: all_headers = self.__headers else: all_headers = self.__headers.copy() all_headers.update(headers) if path[0] == '/': path = path[1:] url = '{0}/{1}'.format(self.__base_url, path) req = urllib2.Request(url=url, data=data, headers=all_headers) req.get_method = lambda: http_type scrubbed_url = self.__http_scrubber.scrub_url(url) scrubbed_data = self.__http_scrubber.scrub_request(data) if data is not None: JournalLogger.journal_or_log_detail( '{type} {url}'.format(type=http_type, url=scrubbed_url), scrubbed_data, _logger=self.logger, _context='request') else: JournalLogger.journal_or_log( '{type} {url}'.format(type=http_type, url=scrubbed_url), _logger=self.logger, _context='request') code = None output = None exception = None try: response = urllib2.urlopen(req) code = response.getcode() output = response.read() scrubbed_output = self.__http_scrubber.scrub_response(output) JournalLogger.journal_or_log_detail( 'HTTP {code}'.format(code=code), scrubbed_output, _logger=self.logger, _context='response') except urllib2.HTTPError as ex: code = ex.getcode() output = ex.read() scrubbed_error = self.__http_scrubber.scrub_response(output) JournalLogger.journal_or_log_detail( 'HTTP {code}'.format(code=code), scrubbed_error, _logger=self.logger, _context='response') except urllib2.URLError as ex: JournalLogger.journal_or_log( 'Caught exception: {ex}\n{stack}'.format( ex=ex, stack=traceback.format_exc()), _logger=self.logger) exception = ex return HttpResponseType(code, output, exception)
def _log_call_method_response(self, method, response): JournalLogger.journal_or_log(AwsJsonEncoder( encoding='utf-8', separators=(',', ':')).encode(response), _module=self.logger.name, _context='response', format='json')
def __send_http_request(self, path, http_type, data=None, headers=None): """Send an HTTP message. Args: path: [string] The URL path to send to (without network location) http_type: [string] The HTTP message type (e.g. POST) data: [string] Data payload to send, if any. headers: [dict] Headers to write, if any. Returns: HttpResponseType """ if headers is None: all_headers = self.__headers else: all_headers = self.__headers.copy() all_headers.update(headers) if path[0] == '/': path = path[1:] url = '{0}/{1}'.format(self.__base_url, path) req = urllib2.Request(url=url, data=data, headers=all_headers) req.get_method = lambda: http_type scrubbed_url = self.__http_scrubber.scrub_url(url) scrubbed_data = self.__http_scrubber.scrub_request(data) if data is not None: JournalLogger.journal_or_log_detail('{type} {url}'.format( type=http_type, url=scrubbed_url), scrubbed_data, _logger=self.logger, _context='request') else: JournalLogger.journal_or_log('{type} {url}'.format( type=http_type, url=scrubbed_url), _logger=self.logger, _context='request') code = None output = None exception = None try: response = urllib2.urlopen(req) code = response.getcode() output = response.read() scrubbed_output = self.__http_scrubber.scrub_response(output) JournalLogger.journal_or_log_detail( 'HTTP {code}'.format(code=code), scrubbed_output, _logger=self.logger, _context='response') except urllib2.HTTPError as ex: code = ex.getcode() output = ex.read() scrubbed_error = self.__http_scrubber.scrub_response(output) JournalLogger.journal_or_log_detail( 'HTTP {code}'.format(code=code), scrubbed_error, _logger=self.logger, _context='response') except urllib2.URLError as ex: JournalLogger.journal_or_log( 'Caught exception: {ex}\n{stack}'.format( ex=ex, stack=traceback.format_exc()), _logger=self.logger) exception = ex return HttpResponseType(code, output, exception)
def __send_http_request(self, path, http_type, data=None, headers=None): """Send an HTTP message. Args: path: [string] The URL path to send to (without network location) http_type: [string] The HTTP message type (e.g. POST) data: [string] Data payload to send, if any. headers: [dict] Headers to write, if any. Returns: HttpResponseType """ if headers is None: all_headers = self.__headers else: all_headers = self.__headers.copy() all_headers.update(headers) if path[0] == '/': path = path[1:] url = '{0}/{1}'.format(self.__base_url, path) encoded_data = str.encode(data) if data is not None else None req = Request(url=url, data=encoded_data, headers=all_headers) req.get_method = lambda: http_type scrubbed_url = self.__http_scrubber.scrub_url(url) scrubbed_data = self.__http_scrubber.scrub_request(data) if data is not None: JournalLogger.journal_or_log_detail( '{type} {url}'.format(type=http_type, url=scrubbed_url), scrubbed_data, _logger=self.logger, _context='request') else: JournalLogger.journal_or_log( '{type} {url}'.format(type=http_type, url=scrubbed_url), _logger=self.logger, _context='request') if self.__ignore_ssl_cert_verification: context = ssl._create_unverified_context() opener = build_opener(HTTPSHandler(context=context), HTTPCookieProcessor()) else: opener = build_opener(HTTPCookieProcessor()) code = None output = None exception = None headers = None try: response = opener.open(req) code = response.getcode() output = bytes.decode(response.read()) if sys.version_info[0] > 2: headers = dict(response.headers.items()) else: headers = response.info().headers scrubbed_output = self.__http_scrubber.scrub_response(output) JournalLogger.journal_or_log_detail( 'HTTP {code}'.format(code=code), scrubbed_output, _logger=self.logger, _context='response') except HTTPError as ex: code = ex.getcode() output = bytes.decode(ex.read()) scrubbed_error = self.__http_scrubber.scrub_response(output) JournalLogger.journal_or_log_detail( 'HTTP {code}'.format(code=code), scrubbed_error, _logger=self.logger, _context='response') except URLError as ex: JournalLogger.journal_or_log( 'Caught exception: {ex}\n{stack}'.format( ex=ex, stack=traceback.format_exc()), _logger=self.logger) exception = ex return HttpResponseType(http_code=code, output=output, exception=exception, headers=headers)