def iterate(*args): for n in range(1, globals.num_retries): try: return fn(*args) except Exception, e: log.Warn("Attempt %s failed. %s: %s" % (n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback()) time.sleep(10) # wait a bit before trying again
def iterate(*args): for n in range(1, globals.num_retries): try: kwargs = {"raise_errors": True} return fn(*args, **kwargs) except Exception, e: log.Warn("Attempt %s failed: %s: %s" % (n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback()) if isinstance(e, TemporaryLoadException): time.sleep(30) # wait a bit before trying again
def iterate(*args): for n in range(1, globals.num_retries): try: kwargs = {"raise_errors": True} return fn(*args, **kwargs) except Exception, e: log.Warn(_("Attempt %s failed: %s: %s") % (n, e.__class__.__name__, util.uexc(e))) log.Debug(_("Backtrace of previous error: %s") % exception_traceback()) if isinstance(e, TemporaryLoadException): time.sleep(30) # wait longer before trying again else: time.sleep(10) # wait a bit before trying again
def list(self): for n in range(1, globals.num_retries+1): log.Info("Listing '%s'" % (self.container)) try: keys = self.container.list_objects() return keys except self.resp_exc, resperr: log.Warn("Listing of '%s' failed (attempt %s): CloudFiles returned: %s %s" % (self.container, n, resperr.status, resperr.reason)) except Exception, e: log.Warn("Listing of '%s' failed (attempt %s): %s: %s" % (self.container, n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback())
def inner_retry(self, *args): for n in range(1, globals.num_retries + 1): try: return fn(self, *args) except FatalBackendException as e: # die on fatal errors raise e except Exception as e: # retry on anything else log.Debug( _(u"Backtrace of previous error: %s") % exception_traceback()) at_end = n == globals.num_retries code = _get_code_from_exception(self.backend, operation, e) if code == log.ErrorCode.backend_not_found: # If we tried to do something, but the file just isn't there, # no need to retry. at_end = True if at_end and fatal: def make_filename(f): if isinstance(f, path.ROPath): return util.escape(f.uc_name) else: return util.escape(f) extra = u' '.join([operation] + [ make_filename(x) for x in args if (x and isinstance(x, str)) ]) raise log.FatalError( _(u"Giving up after %s attempts. %s: %s") % (n, e.__class__.__name__, util.uexc(e)), code=code, extra=extra) else: log.Warn( _(u"Attempt %s failed. %s: %s") % (n, e.__class__.__name__, util.uexc(e))) if not at_end: if isinstance(e, TemporaryLoadException): time.sleep(3 * globals.backend_retry_delay ) # wait longer before trying again else: time.sleep(globals.backend_retry_delay ) # wait a bit before trying again if hasattr(self.backend, u'_retry_cleanup'): self.backend._retry_cleanup()
def _retry_fatal(self, *args): try: n = 0 for n in range(1, globals.num_retries): try: self.retry_count = n return fn(self, *args) except FatalBackendError, e: # die on fatal errors raise e except Exception, e: # retry on anything else log.Warn(_("Attempt %s failed. %s: %s") % (n, e.__class__.__name__, util.uexc(e))) log.Debug(_("Backtrace of previous error: %s") % exception_traceback()) time.sleep(10) # wait a bit before trying again
def list(self): for n in range(1, globals.num_retries+1): log.Info("Listing '%s'" % (self.container)) try: # Cloud Files will return a max of 10,000 objects. We have # to make multiple requests to get them all. headers, objs = self.conn.get_container(self.container) return [ o['name'] for o in objs ] except self.resp_exc, resperr: log.Warn("Listing of '%s' failed (attempt %s): Swift server returned: %s %s" % (self.container, n, resperr.http_status, resperr.message)) except Exception, e: log.Warn("Listing of '%s' failed (attempt %s): %s: %s" % (self.container, n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback())
def _retry_fatal(self, *args): try: n = 0 for n in range(1, globals.num_retries): try: self.retry_count = n return fn(self, *args) except FatalBackendError, e: # die on fatal errors raise e except Exception, e: # retry on anything else log.Warn("Attempt %s failed. %s: %s" % (n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback()) time.sleep(10) # wait a bit before trying again
def put(self, source_path, remote_filename = None): if not remote_filename: remote_filename = source_path.get_filename() for n in range(1, globals.num_retries + 1): log.Info("Uploading '%s/%s' " % (self.container, remote_filename)) try: self.container.upload_file(source_path.name, remote_filename) return except self.client_exc, error: log.Warn("Upload of '%s' failed (attempt %d): pyrax returned: %s %s" % (remote_filename, n, error.__class__.__name__, error.message)) except Exception, e: log.Warn("Upload of '%s' failed (attempt %s): %s: %s" % (remote_filename, n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback())
def put(self, source_path, remote_filename = None): if not remote_filename: remote_filename = source_path.get_filename() for n in range(1, globals.num_retries+1): log.Info("Uploading '%s/%s' " % (self.container, remote_filename)) try: sobject = self.container.create_object(remote_filename) sobject.load_from_filename(source_path.name) return except self.resp_exc, error: log.Warn("Upload of '%s' failed (attempt %d): CloudFiles returned: %s %s" % (remote_filename, n, error.status, error.reason)) except Exception, e: log.Warn("Upload of '%s' failed (attempt %s): %s: %s" % (remote_filename, n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback())
def _list(self): if not self.bucket: raise BackendException("No connection to backend") for n in range(1, globals.num_retries+1): if n > 1: # sleep before retry time.sleep(30) log.Info("Listing %s" % self.straight_url) try: return self._list_filenames_in_bucket() except Exception, e: log.Warn("List %s failed (attempt #%d, reason: %s: %s)" "" % (self.straight_url, n, e.__class__.__name__, str(e)), 1) log.Debug("Backtrace of previous error: %s" % (exception_traceback(),))
def get(self, remote_filename, local_path): for n in range(1, globals.num_retries+1): log.Info("Downloading '%s/%s'" % (self.container, remote_filename)) try: sobject = self.container.create_object(remote_filename) f = open(local_path.name, 'w') for chunk in sobject.stream(): f.write(chunk) local_path.setdata() return except self.resp_exc, resperr: log.Warn("Download of '%s' failed (attempt %s): CloudFiles returned: %s %s" % (remote_filename, n, resperr.status, resperr.reason)) except Exception, e: log.Warn("Download of '%s' failed (attempt %s): %s: %s" % (remote_filename, n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback())
def list(self): for n in range(1, globals.num_retries + 1): log.Info("Listing '%s'" % (self.container)) try: # Cloud Files will return a max of 10,000 objects. We have # to make multiple requests to get them all. headers, objs = self.conn.get_container(self.container) return [o['name'] for o in objs] except self.resp_exc, resperr: log.Warn( "Listing of '%s' failed (attempt %s): Swift server returned: %s %s" % (self.container, n, resperr.http_status, resperr.message)) except Exception, e: log.Warn("Listing of '%s' failed (attempt %s): %s: %s" % (self.container, n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback())
def delete_one(self, remote_filename): for n in range(1, globals.num_retries + 1): log.Info("Deleting '%s/%s'" % (self.container, remote_filename)) try: self.container.delete_object(remote_filename) return except self.client_exc, resperr: if n > 1 and resperr.status == 404: # We failed on a timeout, but delete succeeded on the server log.Warn("Delete of '%s' missing after retry - must have succeded earler" % remote_filename) return log.Warn("Delete of '%s' failed (attempt %s): pyrax returned: %s %s" % (remote_filename, n, resperr.__class__.__name__, resperr.message)) except Exception, e: log.Warn("Delete of '%s' failed (attempt %s): %s: %s" % (remote_filename, n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback())
def get(self, remote_filename, local_path): for n in range(1, globals.num_retries + 1): log.Info("Downloading '%s/%s'" % (self.container, remote_filename)) try: sobject = self.container.get_object(remote_filename) f = open(local_path.name, 'w') f.write(sobject.get()) local_path.setdata() return except self.nso_exc: return except self.client_exc, resperr: log.Warn("Download of '%s' failed (attempt %s): pyrax returned: %s %s" % (remote_filename, n, resperr.__class__.__name__, resperr.message)) except Exception, e: log.Warn("Download of '%s' failed (attempt %s): %s: %s" % (remote_filename, n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback())
def pre_process_download(self, files_to_download, wait=False): # Used primarily to move files in Glacier to S3 if isinstance(files_to_download, basestring): files_to_download = [files_to_download] for remote_filename in files_to_download: success = False for n in range(1, globals.num_retries+1): if n > 1: # sleep before retry (new connection to a **hopeful** new host, so no need to wait so long) time.sleep(10) self.resetConnection() try: key_name = self.key_prefix + remote_filename if not self._listed_keys.get(key_name, False): self._listed_keys[key_name] = list(self.bucket.list(key_name))[0] key = self._listed_keys[key_name] if key.storage_class == "GLACIER": # We need to move the file out of glacier if not self.bucket.get_key(key.key).ongoing_restore: log.Info("File %s is in Glacier storage, restoring to S3" % remote_filename) key.restore(days=1) # Shouldn't need this again after 1 day if wait: log.Info("Waiting for file %s to restore from Glacier" % remote_filename) while self.bucket.get_key(key.key).ongoing_restore: time.sleep(60) self.resetConnection() log.Info("File %s was successfully restored from Glacier" % remote_filename) success = True break except Exception, e: log.Warn("Restoration from Glacier for file %s/%s failed (attempt #%d, reason: %s: %s)" "" % (self.straight_url, remote_filename, n, e.__class__.__name__, str(e)), 1) log.Debug("Backtrace of previous error: %s" % (exception_traceback(),)) if not success: log.Warn("Giving up trying to restore %s/%s after %d attempts" % (self.straight_url, remote_filename, globals.num_retries)) raise BackendException("Error restoring %s/%s from Glacier to S3" % (self.straight_url, remote_filename))
def get(self, remote_filename, local_path): for n in range(1, globals.num_retries+1): log.Info("Downloading '%s/%s'" % (self.container, remote_filename)) try: headers, body = self.conn.get_object(self.container, remote_filename) f = open(local_path.name, 'w') for chunk in body: f.write(chunk) local_path.setdata() return except self.resp_exc, resperr: log.Warn("Download of '%s' failed (attempt %s): Swift server returned: %s %s" % (remote_filename, n, resperr.http_status, resperr.message)) except Exception, e: log.Warn("Download of '%s' failed (attempt %s): %s: %s" % (remote_filename, n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback())
def _list(self): for n in range(1, globals.num_retries + 1): log.Info("Listing '%s'" % (self.container)) try: # Cloud Files will return a max of 10,000 objects. We have # to make multiple requests to get them all. objs = self.container.get_object_names() keys = objs while len(objs) == 10000: objs = self.container.get_object_names(marker = keys[-1]) keys += objs return keys except self.client_exc, resperr: log.Warn("Listing of '%s' failed (attempt %s): pyrax returned: %s %s" % (self.container, n, resperr.__class__.__name__, resperr.message)) except Exception, e: log.Warn("Listing of '%s' failed (attempt %s): %s: %s" % (self.container, n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback())
def retry_fatal(fn): def iterate(*args): for n in range(1, globals.num_retries): try: return fn(*args) except Exception, e: log.Warn("Attempt %s failed. %s: %s" % (n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback()) time.sleep(10) # wait a bit before trying again # final trial, die on exception try: return fn(*args) except Exception, e: log.FatalError("Giving up after %s attempts. %s: %s" % (globals.num_retries, e.__class__.__name__, str(e)), log.ErrorCode.backend_error) log.Debug("Backtrace of previous error: %s" % exception_traceback())
def get(self, remote_filename, local_path): for n in range(1, globals.num_retries + 1): log.Info("Downloading '%s/%s'" % (self.container, remote_filename)) try: headers, body = self.conn.get_object(self.container, remote_filename) f = open(local_path.name, 'w') for chunk in body: f.write(chunk) local_path.setdata() return except self.resp_exc, resperr: log.Warn( "Download of '%s' failed (attempt %s): Swift server returned: %s %s" % (remote_filename, n, resperr.http_status, resperr.message)) except Exception, e: log.Warn("Download of '%s' failed (attempt %s): %s: %s" % (remote_filename, n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback())
def get(self, remote_filename, local_path): key = self.key_class(self.bucket) key.key = self.key_prefix + remote_filename for n in range(1, globals.num_retries+1): if n > 1: # sleep before retry (new connection to a **hopeful** new host, so no need to wait so long) time.sleep(10) log.Info("Downloading %s/%s" % (self.straight_url, remote_filename)) try: key.get_contents_to_filename(local_path.name) local_path.setdata() self.resetConnection() return except Exception, e: log.Warn("Download %s/%s failed (attempt #%d, reason: %s: %s)" "" % (self.straight_url, remote_filename, n, e.__class__.__name__, str(e)), 1) log.Debug("Backtrace of previous error: %s" % (exception_traceback(),)) self.resetConnection()
def inner_retry(self, *args): for n in range(1, globals.num_retries + 1): try: return fn(self, *args) except FatalBackendException as e: # die on fatal errors raise e except Exception as e: # retry on anything else log.Debug(_("Backtrace of previous error: %s") % exception_traceback()) at_end = n == globals.num_retries code = _get_code_from_exception(self.backend, operation, e) if code == log.ErrorCode.backend_not_found: # If we tried to do something, but the file just isn't there, # no need to retry. at_end = True if at_end and fatal: def make_filename(f): if isinstance(f, path.ROPath): return util.escape(f.name) else: return util.escape(f) extra = ' '.join([operation] + [make_filename(x) for x in args if x]) log.FatalError(_("Giving up after %s attempts. %s: %s") % (n, e.__class__.__name__, util.uexc(e)), code=code, extra=extra) else: log.Warn(_("Attempt %s failed. %s: %s") % (n, e.__class__.__name__, util.uexc(e))) if not at_end: if isinstance(e, TemporaryLoadException): time.sleep(3 * globals.backend_retry_delay) # wait longer before trying again else: time.sleep(globals.backend_retry_delay) # wait a bit before trying again if hasattr(self.backend, '_retry_cleanup'): self.backend._retry_cleanup()
class BotoBackend(duplicity.backend.Backend): """ Backend for Amazon's Simple Storage System, (aka Amazon S3), though the use of the boto module, (http://code.google.com/p/boto/). To make use of this backend you must set aws_access_key_id and aws_secret_access_key in your ~/.boto or /etc/boto.cfg with your Amazon Web Services key id and secret respectively. Alternatively you can export the environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY. """ def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) import boto assert boto.Version >= BOTO_MIN_VERSION # This folds the null prefix and all null parts, which means that: # //MyBucket/ and //MyBucket are equivalent. # //MyBucket//My///My/Prefix/ and //MyBucket/My/Prefix are equivalent. self.url_parts = filter(lambda x: x != '', parsed_url.path.split('/')) if self.url_parts: self.bucket_name = self.url_parts.pop(0) else: # Duplicity hangs if boto gets a null bucket name. # HC: Caught a socket error, trying to recover raise BackendException('Boto requires a bucket name.') self.scheme = parsed_url.scheme if self.url_parts: self.key_prefix = '%s/' % '/'.join(self.url_parts) else: self.key_prefix = '' self.straight_url = duplicity.backend.strip_auth_from_url(parsed_url) self.parsed_url = parsed_url # duplicity and boto.storage_uri() have different URI formats. # boto uses scheme://bucket[/name] and specifies hostname on connect() self.boto_uri_str = '://'.join((parsed_url.scheme[:2], parsed_url.path.lstrip('/'))) self.storage_uri = boto.storage_uri(self.boto_uri_str) self.resetConnection() def resetConnection(self): self.bucket = None self.conn = None try: from boto.s3.connection import S3Connection from boto.s3.key import Key assert hasattr(S3Connection, 'lookup') # Newer versions of boto default to using # virtual hosting for buckets as a result of # upstream deprecation of the old-style access # method by Amazon S3. This change is not # backwards compatible (in particular with # respect to upper case characters in bucket # names); so we default to forcing use of the # old-style method unless the user has # explicitly asked us to use new-style bucket # access. # # Note that if the user wants to use new-style # buckets, we use the subdomain calling form # rather than given the option of both # subdomain and vhost. The reason being that # anything addressable as a vhost, is also # addressable as a subdomain. Seeing as the # latter is mostly a convenience method of # allowing browse:able content semi-invisibly # being hosted on S3, the former format makes # a lot more sense for us to use - being # explicit about what is happening (the fact # that we are talking to S3 servers). try: from boto.s3.connection import OrdinaryCallingFormat from boto.s3.connection import SubdomainCallingFormat cfs_supported = True calling_format = OrdinaryCallingFormat() except ImportError: cfs_supported = False calling_format = None if globals.s3_use_new_style: if cfs_supported: calling_format = SubdomainCallingFormat() else: log.FatalError("Use of new-style (subdomain) S3 bucket addressing was" "requested, but does not seem to be supported by the " "boto library. Either you need to upgrade your boto " "library or duplicity has failed to correctly detect " "the appropriate support.", log.ErrorCode.boto_old_style) else: if cfs_supported: calling_format = OrdinaryCallingFormat() else: calling_format = None except ImportError: log.FatalError("This backend (s3) requires boto library, version %s or later, " "(http://code.google.com/p/boto/)." % BOTO_MIN_VERSION, log.ErrorCode.boto_lib_too_old) if not self.parsed_url.hostname: # Use the default host. self.conn = self.storage_uri.connect( is_secure=(not globals.s3_unencrypted_connection)) else: assert self.scheme == 's3' self.conn = self.storage_uri.connect( host=self.parsed_url.hostname, is_secure=(not globals.s3_unencrypted_connection)) if hasattr(self.conn, 'calling_format'): if calling_format is None: log.FatalError("It seems we previously failed to detect support for calling " "formats in the boto library, yet the support is there. This is " "almost certainly a duplicity bug.", log.ErrorCode.boto_calling_format) else: self.conn.calling_format = calling_format else: # Duplicity hangs if boto gets a null bucket name. # HC: Caught a socket error, trying to recover raise BackendException('Boto requires a bucket name.') self.bucket = self.conn.lookup(self.bucket_name) def put(self, source_path, remote_filename=None): from boto.s3.connection import Location if globals.s3_european_buckets: if not globals.s3_use_new_style: log.FatalError("European bucket creation was requested, but not new-style " "bucket addressing (--s3-use-new-style)", log.ErrorCode.s3_bucket_not_style) #Network glitch may prevent first few attempts of creating/looking up a bucket for n in range(1, globals.num_retries+1): if self.bucket: break if n > 1: time.sleep(30) try: try: self.bucket = self.conn.get_bucket(self.bucket_name, validate=True) except Exception, e: if "NoSuchBucket" in str(e): if globals.s3_european_buckets: self.bucket = self.conn.create_bucket(self.bucket_name, location=Location.EU) else: self.bucket = self.conn.create_bucket(self.bucket_name) else: raise e except Exception, e: log.Warn("Failed to create bucket (attempt #%d) '%s' failed (reason: %s: %s)" "" % (n, self.bucket_name, e.__class__.__name__, str(e))) self.resetConnection() if not remote_filename: remote_filename = source_path.get_filename() key = self.bucket.new_key(self.key_prefix + remote_filename) for n in range(1, globals.num_retries+1): if n > 1: # sleep before retry (new connection to a **hopeful** new host, so no need to wait so long) time.sleep(10) if globals.s3_use_rrs: storage_class = 'REDUCED_REDUNDANCY' else: storage_class = 'STANDARD' log.Info("Uploading %s/%s to %s Storage" % (self.straight_url, remote_filename, storage_class)) try: key.set_contents_from_filename(source_path.name, {'Content-Type': 'application/octet-stream', 'x-amz-storage-class': storage_class}, cb=progress.report_transfer, num_cb=(max(2, 8 * globals.volsize / (1024 * 1024))) ) # Max num of callbacks = 8 times x megabyte key.close() self.resetConnection() return except Exception, e: log.Warn("Upload '%s/%s' failed (attempt #%d, reason: %s: %s)" "" % (self.straight_url, remote_filename, n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % (exception_traceback(),)) self.resetConnection()
except FatalBackendError, e: # die on fatal errors raise e except Exception, e: # retry on anything else log.Warn("Attempt %s failed. %s: %s" % (n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback()) time.sleep(10) # wait a bit before trying again # final trial, die on exception self.retry_count = n + 1 return fn(self, *args) except Exception, e: log.Debug("Backtrace of previous error: %s" % exception_traceback()) log.FatalError( "Giving up after %s attempts. %s: %s" % (self.retry_count, e.__class__.__name__, str(e)), log.ErrorCode.backend_error) self.retry_count = 0 return _retry_fatal class Backend: """ Represents a generic duplicity backend, capable of storing and retrieving files. Concrete sub-classes are expected to implement:
class BotoBackend(duplicity.backend.Backend): """ Backend for Amazon's Simple Storage System, (aka Amazon S3), though the use of the boto module, (http://code.google.com/p/boto/). To make use of this backend you must set aws_access_key_id and aws_secret_access_key in your ~/.boto or /etc/boto.cfg with your Amazon Web Services key id and secret respectively. Alternatively you can export the environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY. """ def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) from boto.s3.key import Key from boto.s3.multipart import MultiPartUpload # This folds the null prefix and all null parts, which means that: # //MyBucket/ and //MyBucket are equivalent. # //MyBucket//My///My/Prefix/ and //MyBucket/My/Prefix are equivalent. self.url_parts = filter(lambda x: x != '', parsed_url.path.split('/')) if self.url_parts: self.bucket_name = self.url_parts.pop(0) else: # Duplicity hangs if boto gets a null bucket name. # HC: Caught a socket error, trying to recover raise BackendException('Boto requires a bucket name.') self.scheme = parsed_url.scheme self.key_class = Key if self.url_parts: self.key_prefix = '%s/' % '/'.join(self.url_parts) else: self.key_prefix = '' self.straight_url = duplicity.backend.strip_auth_from_url(parsed_url) self.parsed_url = parsed_url self.resetConnection() def resetConnection(self): self.bucket = None self.conn = get_connection(self.scheme, self.parsed_url) self.bucket = self.conn.lookup(self.bucket_name) def put(self, source_path, remote_filename=None): from boto.s3.connection import Location if globals.s3_european_buckets: if not globals.s3_use_new_style: log.FatalError( "European bucket creation was requested, but not new-style " "bucket addressing (--s3-use-new-style)", log.ErrorCode.s3_bucket_not_style) #Network glitch may prevent first few attempts of creating/looking up a bucket for n in range(1, globals.num_retries + 1): if self.bucket: break if n > 1: time.sleep(30) try: try: self.bucket = self.conn.get_bucket(self.bucket_name, validate=True) except Exception, e: if "NoSuchBucket" in str(e): if globals.s3_european_buckets: self.bucket = self.conn.create_bucket( self.bucket_name, location=Location.EU) else: self.bucket = self.conn.create_bucket( self.bucket_name) else: raise e except Exception, e: log.Warn( "Failed to create bucket (attempt #%d) '%s' failed (reason: %s: %s)" "" % (n, self.bucket_name, e.__class__.__name__, str(e))) self.resetConnection() if not remote_filename: remote_filename = source_path.get_filename() key = self.key_prefix + remote_filename for n in range(1, globals.num_retries + 1): if n > 1: # sleep before retry (new connection to a **hopeful** new host, so no need to wait so long) time.sleep(10) if globals.s3_use_rrs: storage_class = 'REDUCED_REDUNDANCY' else: storage_class = 'STANDARD' log.Info("Uploading %s/%s to %s Storage" % (self.straight_url, remote_filename, storage_class)) try: headers = { 'Content-Type': 'application/octet-stream', 'x-amz-storage-class': storage_class } self.upload(source_path.name, key, headers) self.resetConnection() return except Exception, e: log.Warn("Upload '%s/%s' failed (attempt #%d, reason: %s: %s)" "" % (self.straight_url, remote_filename, n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % (exception_traceback(), )) self.resetConnection()
try: self.retry_count = n return fn(self, *args) except FatalBackendError, e: # die on fatal errors raise e except Exception, e: # retry on anything else log.Warn(_("Attempt %s failed. %s: %s") % (n, e.__class__.__name__, util.uexc(e))) log.Debug(_("Backtrace of previous error: %s") % exception_traceback()) time.sleep(10) # wait a bit before trying again # final trial, die on exception self.retry_count = n + 1 return fn(self, *args) except Exception, e: log.Debug(_("Backtrace of previous error: %s") % exception_traceback()) log.FatalError( _("Giving up after %s attempts. %s: %s") % (self.retry_count, e.__class__.__name__, util.uexc(e)), log.ErrorCode.backend_error, ) self.retry_count = 0 return _retry_fatal class Backend: """ Represents a generic duplicity backend, capable of storing and retrieving files. Concrete sub-classes are expected to implement:
except FatalBackendError, e: # die on fatal errors raise e except Exception, e: # retry on anything else log.Warn("Attempt %s failed. %s: %s" % (n, e.__class__.__name__, str(e))) log.Debug("Backtrace of previous error: %s" % exception_traceback()) time.sleep(10) # wait a bit before trying again # final trial, die on exception self.retry_count = n+1 return fn(self, *args) except Exception, e: log.Debug("Backtrace of previous error: %s" % exception_traceback()) log.FatalError("Giving up after %s attempts. %s: %s" % (self.retry_count, e.__class__.__name__, str(e)), log.ErrorCode.backend_error) self.retry_count = 0 return _retry_fatal class Backend: """ Represents a generic duplicity backend, capable of storing and retrieving files. Concrete sub-classes are expected to implement: - put
raise e except Exception, e: # retry on anything else log.Warn( _("Attempt %s failed. %s: %s") % (n, e.__class__.__name__, util.uexc(e))) log.Debug( _("Backtrace of previous error: %s") % exception_traceback()) time.sleep(10) # wait a bit before trying again # final trial, die on exception self.retry_count = n + 1 return fn(self, *args) except Exception, e: log.Debug( _("Backtrace of previous error: %s") % exception_traceback()) log.FatalError( _("Giving up after %s attempts. %s: %s") % (self.retry_count, e.__class__.__name__, util.uexc(e)), log.ErrorCode.backend_error) self.retry_count = 0 return _retry_fatal class Backend: """ Represents a generic duplicity backend, capable of storing and retrieving files. Concrete sub-classes are expected to implement: