def post_thread(self, queue, failures): while True: try: file_detail = queue.get_nowait() logging.debug("%s: processing job %s", threading.current_thread(), file_detail) retry_function(lambda: self._post_file(file_detail)) except requests.exceptions.RequestException as e: msg = "Error posting file after multiple attempts" # Do our best to attempt to upload all the files logging.exception(msg) failures.append({ "file": file_detail.filename, "error": "{}: {}".format(msg, e) }) continue except IOError as e: msg = "Error opening file" # Do our best to attempt to upload all the files logging.exception(msg) failures.append({ "file": file_detail.filename, "error": "{}: {}".format(msg, e) }) continue except queuelib.Empty: # No more work to do return
def post_thread(self, queue): while True: try: file_detail = queue.get_nowait() logging.debug("%s: processing job %s", threading.current_thread(), file_detail) retry_function(lambda: self._post_file(file_detail)) except IOError: # Do our best to attempt to upload all the files logging.exception("Error opening file") continue except queuelib.Empty: # No more work to do return
def __init__(self, cloud, container, prefix=None, delete_after=None, public=True, dry_run=False): self.dry_run = dry_run if dry_run: self.url = 'http://dry-run-url.com/a/path/' return self.cloud = cloud self.container = container self.prefix = prefix or '' self.delete_after = delete_after sess = self.cloud.config.get_session() adapter = requests.adapters.HTTPAdapter(pool_maxsize=100) sess.mount('https://', adapter) # If we're in Rackspace, there's some non-standard stuff we # need to do to get the public endpoint. try: cdn_endpoint = self.cloud.session.auth.get_endpoint( self.cloud.session, service_type='rax:object-cdn', region_name=self.cloud.config.region_name, interface=self.cloud.config.interface) cdn_url = os.path.join(cdn_endpoint, self.container) except keystoneauth1.exceptions.catalog.EndpointNotFound: cdn_url = None # We retry here because sometimes we get HTTP 401 errors in rax. # They seem to happen infrequently (on the order of once a day across # all jobs) so a retry is likely to work. container = retry_function( lambda: self.cloud.get_container(self.container)) if not container: retry_function( lambda: self.cloud.create_container( name=self.container, public=public)) headers = {'X-Container-Meta-Web-Index': 'index.html', 'X-Container-Meta-Access-Control-Allow-Origin': '*'} retry_function( lambda: self.cloud.update_container( name=self.container, headers=headers)) # 'X-Container-Meta-Web-Listings': 'true' # The ceph radosgw swift implementation requires an # index.html at the root in order for any other indexes to # work. index_headers = {'access-control-allow-origin': '*'} retry_function( lambda: self.cloud.create_object(self.container, name='index.html', data='', content_type='text/html', **index_headers)) # Enable the CDN in rax if cdn_url: retry_function(lambda: self.cloud.session.put(cdn_url)) if cdn_url: endpoint = retry_function( lambda: self.cloud.session.head( cdn_url).headers['X-Cdn-Ssl-Uri']) container = endpoint else: endpoint = self.cloud.object_store.get_endpoint() container = os.path.join(endpoint, self.container) self.url = os.path.join(container, self.prefix)