def _fetch(self, url): """ Retrieves XML response from Amazon. In case of a timeout, it will try :const:`~RetryAPI.TRIES`` times before raising an error. """ attempts = 0 delay = self.DELAY while True: try: attempts += 1 return API._fetch(self, url) except URLError: e = sys.exc_info()[1] # Python 2/3 compatible # if a timeout occurred # wait for some time before trying again reason = getattr(e, 'reason', None) if isinstance(reason, socket.timeout) and attempts < self.TRIES: time.sleep(delay) delay *= self.BACKOFF continue # otherwise reraise the original error raise
def _fetch(self, url): """ Retrieves XML response from Amazon. In case of a timeout, it will try :const:`~RetryAPI.TRIES`` times before raising an error. """ attempts = 0 delay = self.DELAY while True: try: attempts += 1 return API._fetch(self, url) except urllib2.URLError, e: # if a timeout occurred # wait for some time before trying again reason = getattr(e, 'reason', None) if isinstance(reason, socket.timeout) and attempts < self.TRIES: time.sleep(delay) delay *= self.BACKOFF continue # otherwise reraise the original error raise
def _fetch(self, url): if self.cache: path = os.path.join(self.cache, "%s.xml" % self.get_hash(url)) # if response was fetched previously, use that one if os.path.isfile(path): return open(path) # fetch original response from Amazon resp = API._fetch(self, url) if self.cache: fp = open(path, "w+") fp.write(etree.tostring(etree.parse(resp), pretty_print=True)) fp.seek(0) return fp return resp
def _fetch(self, url): if self.cache: path = os.path.join(self.cache, '%s.xml' % self.get_hash(url)) # if response was fetched previously, use that one if os.path.isfile(path): return open(path) # fetch original response from Amazon resp = API._fetch(self, url) if self.cache: fp = open(path, 'w+') fp.write(etree.tostring(etree.parse(resp), pretty_print=True)) fp.seek(0) return fp return resp
def _fetch(self, url): if self.cache: path = os.path.join(self.cache, '%s.xml' % self.get_hash(url)) # if response was fetched previously, use that one if os.path.isfile(path): return open(path) # fetch original response from Amazon resp = API._fetch(self, url) if self.cache: fp = open(path, 'w+') node = xml.dom.minidom.parseString(resp.read()) fp.write(node.toprettyxml()) #from lxml import etree #fp.write(etree.tostring(etree.parse(resp), pretty_print=True)) fp.seek(0) return fp return resp
def pytest_funcarg__api(request): """ Initialises API for each test call (formerly done with ``setup_method()``). """ server = request.getfuncargvalue('server') url_reg = re.compile(r'^http://(?P<host>[\w\-\.]+)(?P<path>/onca/xml.*)$') # the following parameters are injected by pytest_generate_tests locale = request.param.locale version = request.param.version xml_response = request.param.xml_response processor = TESTABLE_PROCESSORS[request.param.processor] api = API(locale=locale, processor=processor) api.VERSION = version api.REQUESTS_PER_SECOND = 10000 # just for here! def counter(fnc): """ Wrapper function for ``_fetch`` which 1. keeps track of the times has been called and adjusts the path to the corresponding XML response 2. Fetches any response that has not been cached from the live servers """ api._count = 0 def wrapped(url): api._count += 1 path = xml_response if api._count > 1: root, ext = os.path.splitext(path) path = '%s-%i%s' % (root, api._count, ext) try: if request.config.option.fetch == 'all': raise ResponseRequired try: content = open(path, 'r').read() # If the XML response has been previously fetched compare # request arguments in order to see if there are any changes cached_params = utils.arguments_from_cached_xml(content) current_params = utils.arguments_from_url(url) if cached_params != current_params: raise ArgumentMismatch except IOError: if request.config.option.fetch in ('outdated', 'missing'): raise ResponseRequired raise pytest.skip('No cached XML response found!') except ArgumentMismatch: if request.config.option.fetch == 'outdated': raise ResponseRequired msg = ('Cached arguments in %s differ from the ones ' 'currently tested against!\ncached=%r\ncurrent=%r' % (path, cached_params, current_params)) return pytest.skip(msg) except AttributeError: # XML for error messages have no Argument elements! pass except ResponseRequired: # fetch XML via urllib2 rather than directly via # lxml.etree.parse() to avoid, for instance, problems with HTTP # 403 errors resp = requests.get(url, headers={'User-Agent': USER_AGENT}) root = lxml.etree.fromstring(resp.text) # overwrite sensitive information in XML document. for arg in root.xpath('//aws:Argument', namespaces={'aws': root.nsmap[None]}): if arg.get('Name') in ('Signature', 'AWSAccessKeyId', 'AssociateTag'): arg.set('Value', 'X'*15) content = lxml.etree.tostring(root, pretty_print=True) # complain loudly about missing credentials # UNLESS it was actually on purpose! if (six.b('MissingClientTokenId') in content and getattr(request.function, 'refetch', True)): raise pytest.fail('Cannot fetch XML response without credentials!') if not os.path.exists(os.path.dirname(path)): os.mkdir(os.path.dirname(path)) open(path, 'wb').write(content) # We simply exchange the real host with the local one now! # Note: Although strictly speaking it does not matter which URL is # called exactly, to appeal to one's sense of correctness, let's # keep at least the correct path! url = url_reg.sub(r'%s\g<path>' % server.url, url) server.serve_content(content) return fnc(url) return wrapped api._fetch = counter(api._fetch) return api
def pytest_funcarg__api(request): """ Initialises API for each test call (formerly done with ``setup_method()``). """ server = request.getfuncargvalue('server') url_reg = re.compile(r'^http://(?P<host>[\w\-\.]+)(?P<path>/onca/xml.*)$') # the following parameters are injected by pytest_generate_tests locale = request.param.locale version = request.param.version xml_response = request.param.xml_response processor = TESTABLE_PROCESSORS[request.param.processor] api = API(locale=locale, processor=processor) api.VERSION = version api.REQUESTS_PER_SECOND = 10000 # just for here! def counter(fnc): """ Wrapper function for ``_fetch`` which 1. keeps track of the times has been called and adjusts the path to the corresponding XML response 2. Fetches any response that has not been cached from the live servers """ api._count = 0 def wrapped(url): api._count += 1 path = xml_response if api._count > 1: root, ext = os.path.splitext(path) path = '%s-%i%s' % (root, api._count, ext) try: if request.config.option.fetch == 'all': raise ResponseRequired try: content = open(path, 'r').read() # If the XML response has been previously fetched compare # request arguments in order to see if there are any changes cached_params = utils.arguments_from_cached_xml(content) current_params = utils.arguments_from_url(url) if cached_params != current_params: raise ArgumentMismatch except IOError: if request.config.option.fetch in ('outdated', 'missing'): raise ResponseRequired raise pytest.skip('No cached XML response found!') except ArgumentMismatch: if request.config.option.fetch == 'outdated': raise ResponseRequired msg = ('Cached arguments in %s differ from the ones ' 'currently tested against!\ncached=%r\ncurrent=%r' % (path, cached_params, current_params)) return pytest.skip(msg) except AttributeError: # XML for error messages have no Argument elements! pass except ResponseRequired: # fetch XML via urllib2 rather than directly via # lxml.etree.parse() to avoid, for instance, problems with HTTP # 403 errors resp = requests.get(url, headers={'User-Agent': USER_AGENT}) root = lxml.etree.fromstring(resp.text) # overwrite sensitive information in XML document. for arg in root.xpath('//aws:Argument', namespaces={'aws': root.nsmap[None]}): if arg.get('Name') in ('Signature', 'AWSAccessKeyId', 'AssociateTag'): arg.set('Value', 'X' * 15) content = lxml.etree.tostring(root, pretty_print=True) # complain loudly about missing credentials # UNLESS it was actually on purpose! if (six.b('MissingClientTokenId') in content and getattr(request.function, 'refetch', True)): raise pytest.fail( 'Cannot fetch XML response without credentials!') if not os.path.exists(os.path.dirname(path)): os.mkdir(os.path.dirname(path)) open(path, 'wb').write(content) # We simply exchange the real host with the local one now! # Note: Although strictly speaking it does not matter which URL is # called exactly, to appeal to one's sense of correctness, let's # keep at least the correct path! url = url_reg.sub(r'%s\g<path>' % server.url, url) server.serve_content(content) return fnc(url) return wrapped api._fetch = counter(api._fetch) return api