def post_thread(self, queue, failures):
while True:
try:
file_detail = queue.get_nowait()
logging.debug("%s: processing job %s",
threading.current_thread(),
file_detail)
retry_function(lambda: self._post_file(file_detail))
except requests.exceptions.RequestException as e:
msg = "Error posting file after multiple attempts"
# Do our best to attempt to upload all the files
logging.exception(msg)
failures.append({
"file": file_detail.filename,
"error": "{}: {}".format(msg, e)
})
continue
except IOError as e:
msg = "Error opening file"
# Do our best to attempt to upload all the files
logging.exception(msg)
failures.append({
"file": file_detail.filename,
"error": "{}: {}".format(msg, e)
})
continue
except queuelib.Empty:
# No more work to do
return
def post_thread(self, queue):
while True:
try:
file_detail = queue.get_nowait()
logging.debug("%s: processing job %s",
threading.current_thread(), file_detail)
retry_function(lambda: self._post_file(file_detail))
except IOError:
# Do our best to attempt to upload all the files
logging.exception("Error opening file")
continue
except queuelib.Empty:
# No more work to do
return
def __init__(self, cloud, container, prefix=None, delete_after=None,
public=True, dry_run=False):
self.dry_run = dry_run
if dry_run:
self.url = 'http://dry-run-url.com/a/path/'
return
self.cloud = cloud
self.container = container
self.prefix = prefix or ''
self.delete_after = delete_after
sess = self.cloud.config.get_session()
adapter = requests.adapters.HTTPAdapter(pool_maxsize=100)
sess.mount('https://', adapter)
# If we're in Rackspace, there's some non-standard stuff we
# need to do to get the public endpoint.
try:
cdn_endpoint = self.cloud.session.auth.get_endpoint(
self.cloud.session, service_type='rax:object-cdn',
region_name=self.cloud.config.region_name,
interface=self.cloud.config.interface)
cdn_url = os.path.join(cdn_endpoint, self.container)
except keystoneauth1.exceptions.catalog.EndpointNotFound:
cdn_url = None
# We retry here because sometimes we get HTTP 401 errors in rax.
# They seem to happen infrequently (on the order of once a day across
# all jobs) so a retry is likely to work.
container = retry_function(
lambda: self.cloud.get_container(self.container))
if not container:
retry_function(
lambda: self.cloud.create_container(
name=self.container, public=public))
headers = {'X-Container-Meta-Web-Index': 'index.html',
'X-Container-Meta-Access-Control-Allow-Origin': '*'}
retry_function(
lambda: self.cloud.update_container(
name=self.container,
headers=headers))
# 'X-Container-Meta-Web-Listings': 'true'
# The ceph radosgw swift implementation requires an
# index.html at the root in order for any other indexes to
# work.
index_headers = {'access-control-allow-origin': '*'}
retry_function(
lambda: self.cloud.create_object(self.container,
name='index.html',
data='',
content_type='text/html',
**index_headers))
# Enable the CDN in rax
if cdn_url:
retry_function(lambda: self.cloud.session.put(cdn_url))
if cdn_url:
endpoint = retry_function(
lambda: self.cloud.session.head(
cdn_url).headers['X-Cdn-Ssl-Uri'])
container = endpoint
else:
endpoint = self.cloud.object_store.get_endpoint()
container = os.path.join(endpoint, self.container)
self.url = os.path.join(container, self.prefix)
