def __init__(self, *args, **kwargs):
try:
global socket, ssl
import socket
import ssl
except ImportError:
raise FatalBackendException("Missing socket or ssl libraries.")
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
self.cacert_file = globals.ssl_cacert_file
cacert_candidates = [
"~/.duplicity/cacert.pem", "~/duplicity_cacert.pem",
"/etc/duplicity/cacert.pem"
]
#
if not self.cacert_file:
for path in cacert_candidates:
path = os.path.expanduser(path)
if (os.path.isfile(path)):
self.cacert_file = path
break
# still no cacert file, inform user
if not self.cacert_file:
raise FatalBackendException("""\
For certificate verification a cacert database file is needed in one of these locations: %s
Hints:
Consult the man page, chapter 'SSL Certificate Verification'.
Consider using the options --ssl-cacert-file, --ssl-no-check-certificate ."""
% ", ".join(cacert_candidates))
# check if file is accessible (libssl errors are not very detailed)
if not os.access(self.cacert_file, os.R_OK):
raise FatalBackendException(
"Cacert database file '%s' is not readable." %
self.cacert_file)
def __init__(self, *args, **kwargs):
try:
global socket, ssl
import socket
import ssl
except ImportError:
raise FatalBackendException(
_("Missing socket or ssl python modules."))
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
self.cacert_file = globals.ssl_cacert_file
self.cacert_candidates = [
"~/.duplicity/cacert.pem", "~/duplicity_cacert.pem",
"/etc/duplicity/cacert.pem"
]
# if no cacert file was given search default locations
if not self.cacert_file:
for path in self.cacert_candidates:
path = os.path.expanduser(path)
if (os.path.isfile(path)):
self.cacert_file = path
break
# check if file is accessible (libssl errors are not very detailed)
if self.cacert_file and not os.access(self.cacert_file, os.R_OK):
raise FatalBackendException(
_("Cacert database file '%s' is not readable.") %
self.cacert_file)
def connect(self):
# create new socket
sock = socket.create_connection((self.host, self.port),
self.timeout)
if self._tunnel_host:
self.sock = sock
self.tunnel()
# python 2.7.9+ supports default system certs now
if u"create_default_context" in dir(ssl):
context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH,
cafile=self.cacert_file,
capath=config.ssl_cacert_path)
self.sock = context.wrap_socket(sock, server_hostname=self.host)
# the legacy way needing a cert file
else:
if config.ssl_cacert_path:
raise FatalBackendException(
_(u"Option '--ssl-cacert-path' is not supported "
u"with python 2.7.8 and below."))
if not self.cacert_file:
raise FatalBackendException(_(u"""\
For certificate verification with python 2.7.8 or earlier a cacert database
file is needed in one of these locations: %s
Hints:
Consult the man page, chapter 'SSL Certificate Verification'.
Consider using the options --ssl-cacert-file, --ssl-no-check-certificate .""") %
u", ".join(self.cacert_candidates))
# wrap the socket in ssl using verification
self.sock = ssl.wrap_socket(sock,
cert_reqs=ssl.CERT_REQUIRED,
ca_certs=self.cacert_file,
)
def request(self, method, path, data=None, redirected=0):
u"""
Wraps the connection.request method to retry once if authentication is
required
"""
self._close() # or we get previous request's data or exception
self.connect()
quoted_path = urllib.parse.quote(path, u"/:~")
if self.digest_challenge is not None:
self.headers[u'Authorization'] = self.get_digest_authorization(
path)
log.Info(
_(u"WebDAV %s %s request with headers: %s ") %
(method, quoted_path, self.headers))
log.Info(_(u"WebDAV data length: %s ") % len(str(data)))
self.conn.request(method, quoted_path, data, self.headers)
response = self.conn.getresponse()
log.Info(
_(u"WebDAV response status %s with reason '%s'.") %
(response.status, response.reason))
# resolve redirects and reset url on listing requests (they usually come before everything else)
if response.status in [301, 302] and method == u'PROPFIND':
redirect_url = response.getheader(u'location', None)
response.close()
if redirect_url:
log.Notice(
_(u"WebDAV redirect to: %s ") %
urllib.parse.unquote(redirect_url))
if redirected > 10:
raise FatalBackendException(
_(u"WebDAV redirected 10 times. Giving up."))
self.parsed_url = duplicity.backend.ParsedUrl(redirect_url)
self.directory = self.sanitize_path(self.parsed_url.path)
return self.request(method, self.directory, data,
redirected + 1)
else:
raise FatalBackendException(
_(u"WebDAV missing location header in redirect response."))
elif response.status == 401:
response.read()
response.close()
self.headers[u'Authorization'] = self.get_authorization(
response, quoted_path)
log.Info(_(u"WebDAV retry request with authentification headers."))
log.Info(
_(u"WebDAV %s %s request2 with headers: %s ") %
(method, quoted_path, self.headers))
log.Info(_(u"WebDAV data length: %s ") % len(str(data)))
self.conn.request(method, quoted_path, data, self.headers)
response = self.conn.getresponse()
log.Info(
_(u"WebDAV response2 status %s with reason '%s'.") %
(response.status, response.reason))
return response
def connect(self, forced=False):
"""
Connect or re-connect to the server, updates self.conn
# reconnect on errors as a precaution, there are errors e.g.
# "[Errno 32] Broken pipe" or SSl errors that render the connection unusable
"""
if not forced and self.conn \
and self.conn.host == self.parsed_url.hostname:
return
log.Info(
_("WebDAV create connection on '%s'") % (self.parsed_url.hostname))
self._close()
# http schemes needed for redirect urls from servers
if self.parsed_url.scheme in ['webdav', 'http']:
self.conn = httplib.HTTPConnection(self.parsed_url.hostname,
self.parsed_url.port)
elif self.parsed_url.scheme in ['webdavs', 'https']:
if globals.ssl_no_check_certificate:
self.conn = httplib.HTTPSConnection(self.parsed_url.hostname,
self.parsed_url.port)
else:
self.conn = VerifiedHTTPSConnection(self.parsed_url.hostname,
self.parsed_url.port)
else:
raise FatalBackendException(
_("WebDAV Unknown URI scheme: %s") % (self.parsed_url.scheme))
def __init__(self, parsed_url):
u"""
Authorize to B2 api and set up needed variables
"""
duplicity.backend.Backend.__init__(self, parsed_url)
global DownloadDestLocalFile, FileVersionInfoFactory
try: # try to import the new b2sdk if available
from b2sdk.api import B2Api
from b2sdk.account_info import InMemoryAccountInfo
from b2sdk.download_dest import DownloadDestLocalFile
from b2sdk.exception import NonExistentBucket
from b2sdk.file_version import FileVersionInfoFactory
except ImportError as e:
if u'b2sdk' in getattr(e, u'name', u'b2sdk'):
raise
try: # fall back to import the old b2 client
from b2.api import B2Api
from b2.account_info import InMemoryAccountInfo
from b2.download_dest import DownloadDestLocalFile
from b2.exception import NonExistentBucket
from b2.file_version import FileVersionInfoFactory
except ImportError:
if u'b2' in getattr(e, u'name', u'b2'):
raise
raise BackendException(
u'B2 backend requires B2 Python SDK (pip install b2sdk)')
self.service = B2Api(InMemoryAccountInfo())
self.parsed_url.hostname = u'B2'
account_id = parsed_url.username
account_key = self.get_password()
self.url_parts = [
x for x in parsed_url.path.replace(u"@", u"/").split(u'/')
if x != u''
]
if self.url_parts:
self.username = self.url_parts.pop(0)
bucket_name = self.url_parts.pop(0)
else:
raise BackendException(u"B2 requires a bucket name")
self.path = u"".join([url_part + u"/" for url_part in self.url_parts])
self.service.authorize_account(u'production', account_id, account_key)
log.Log(
u"B2 Backend (path= %s, bucket= %s, minimum_part_size= %s)" %
(self.path, bucket_name,
self.service.account_info.get_minimum_part_size()), log.INFO)
try:
self.bucket = self.service.get_bucket_by_name(bucket_name)
log.Log(u"Bucket found", log.INFO)
except NonExistentBucket:
try:
log.Log(u"Bucket not found, creating one", log.INFO)
self.bucket = self.service.create_bucket(
bucket_name, u'allPrivate')
except:
raise FatalBackendException(u"Bucket cannot be created")
def __init__(self, parsed_url):
"""
Authorize to B2 api and set up needed variables
"""
duplicity.backend.Backend.__init__(self, parsed_url)
# for prettier password prompt only
self.parsed_url.hostname = 'B2'
self.account_id = parsed_url.username
account_key = self.get_password()
self.url_parts = [
x for x in parsed_url.path.replace("@", "/").split('/') if x != ''
]
if self.url_parts:
self.username = self.url_parts.pop(0)
self.bucket_name = self.url_parts.pop(0)
else:
raise BackendException("B2 requires a bucket name")
self.path = "/".join(self.url_parts)
self.id_and_key = self.account_id + ":" + account_key
self._authorize()
try:
self.find_or_create_bucket(self.bucket_name)
except urllib2.HTTPError:
raise FatalBackendException("Bucket cannot be created")
def __init__(self, parsed_url):
u"""
Authorize to B2 api and set up needed variables
"""
duplicity.backend.Backend.__init__(self, parsed_url)
# Import B2 API
try:
global b2
import b2
global b2sdk
import b2sdk
import b2.api
import b2.account_info
import b2.download_dest
import b2.file_version
except ImportError:
raise BackendException(
u'B2 backend requires B2 Python APIs (pip install b2)')
self.service = b2.api.B2Api(b2.account_info.InMemoryAccountInfo())
self.parsed_url.hostname = u'B2'
account_id = parsed_url.username
account_key = self.get_password()
self.url_parts = [
x for x in parsed_url.path.replace(u"@", u"/").split(u'/')
if x != u''
]
if self.url_parts:
self.username = self.url_parts.pop(0)
bucket_name = self.url_parts.pop(0)
else:
raise BackendException(u"B2 requires a bucket name")
self.path = u"".join([url_part + u"/" for url_part in self.url_parts])
self.service.authorize_account(u'production', account_id, account_key)
log.Log(
u"B2 Backend (path= %s, bucket= %s, minimum_part_size= %s)" %
(self.path, bucket_name,
self.service.account_info.get_minimum_part_size()), log.INFO)
try:
self.bucket = self.service.get_bucket_by_name(bucket_name)
log.Log(u"Bucket found", log.INFO)
except b2.exception.NonExistentBucket:
try:
log.Log(u"Bucket not found, creating one", log.INFO)
self.bucket = self.service.create_bucket(
bucket_name, u'allPrivate')
except:
raise FatalBackendException(u"Bucket cannot be created")
def _put(self, local_source_path, remote_filename):
from boto3.s3.transfer import TransferConfig # pylint: disable=import-error
if not self.s3:
self.reset_connection()
remote_filename = util.fsdecode(remote_filename)
key = self.key_prefix + remote_filename
if config.s3_use_rrs:
storage_class = u'REDUCED_REDUNDANCY'
elif config.s3_use_ia:
storage_class = u'STANDARD_IA'
elif config.s3_use_onezone_ia:
storage_class = u'ONEZONE_IA'
elif config.s3_use_glacier and u"manifest" not in remote_filename:
storage_class = u'GLACIER'
elif config.s3_use_glacier_ir and u"manifest" not in remote_filename:
storage_class = u'GLACIER_IR'
elif config.s3_use_deep_archive and u"manifest" not in remote_filename:
storage_class = u'DEEP_ARCHIVE'
else:
storage_class = u'STANDARD'
extra_args = {u'StorageClass': storage_class}
if config.s3_use_sse:
extra_args[u'ServerSideEncryption'] = u'AES256'
elif config.s3_use_sse_kms:
if config.s3_kms_key_id is None:
raise FatalBackendException(u"S3 USE SSE KMS was requested, but key id not provided "
u"require (--s3-kms-key-id)",
code=log.ErrorCode.s3_kms_no_id)
extra_args[u'ServerSideEncryption'] = u'aws:kms'
extra_args[u'SSEKMSKeyId'] = config.s3_kms_key_id
if config.s3_kms_grant:
extra_args[u'GrantFullControl'] = config.s3_kms_grant
transfer_config = TransferConfig(multipart_chunksize=config.s3_multipart_chunk_size,
multipart_threshold=config.s3_multipart_chunk_size)
# Should the tracker be scoped to the put or the backend?
# The put seems right to me, but the results look a little more correct
# scoped to the backend. This brings up questions about knowing when
# it's proper for it to be reset.
# tracker = UploadProgressTracker() # Scope the tracker to the put()
tracker = self.tracker
log.Info(u"Uploading %s/%s to %s Storage" % (self.straight_url, remote_filename, storage_class))
self.s3.Object(self.bucket.name, key).upload_file(local_source_path.uc_name,
Callback=tracker.progress_cb,
Config=transfer_config,
ExtraArgs=extra_args)
def _put(self, source_path, remote_filename):
if globals.s3_european_buckets:
if not globals.s3_use_new_style:
raise FatalBackendException(
"European bucket creation was requested, but not new-style "
"bucket addressing (--s3-use-new-style)",
code=log.ErrorCode.s3_bucket_not_style)
if self.bucket is None:
try:
self.bucket = self.conn.get_bucket(self.bucket_name)
except Exception as e:
if "NoSuchBucket" in str(e):
self.bucket = self.conn.create_bucket(
self.bucket_name, location=self.my_location)
else:
raise
key = self.bucket.new_key(self.key_prefix + remote_filename)
if globals.s3_use_rrs:
storage_class = 'REDUCED_REDUNDANCY'
elif globals.s3_use_ia:
storage_class = 'STANDARD_IA'
elif globals.s3_use_onezone_ia:
storage_class = 'ONEZONE_IA'
else:
storage_class = 'STANDARD'
log.Info("Uploading %s/%s to %s Storage" %
(self.straight_url, remote_filename, storage_class))
if globals.s3_use_sse:
headers = {
'Content-Type': 'application/octet-stream',
'x-amz-storage-class': storage_class,
'x-amz-server-side-encryption': 'AES256'
}
else:
headers = {
'Content-Type': 'application/octet-stream',
'x-amz-storage-class': storage_class
}
upload_start = time.time()
self.upload(source_path.name, key, headers)
upload_end = time.time()
total_s = abs(upload_end - upload_start) or 1 # prevent a zero value!
rough_upload_speed = os.path.getsize(source_path.name) / total_s
log.Debug("Uploaded %s/%s to %s Storage at roughly %f bytes/second" %
(self.straight_url, remote_filename, storage_class,
rough_upload_speed))
def __init__(self, parsed_url):
"""
Authorize to B2 api and set up needed variables
"""
duplicity.backend.Backend.__init__(self, parsed_url)
# for prettier password prompt only
self.parsed_url.hostname = 'B2'
self.account_id = parsed_url.username
account_key = self.get_password()
self.url_parts = [
x for x in parsed_url.path.replace("@", "/").split('/') if x != ''
]
if self.url_parts:
self.username = self.url_parts.pop(0)
self.bucket_name = self.url_parts.pop(0)
else:
raise BackendException("B2 requires a bucket name")
self.path = "/".join(self.url_parts)
id_and_key = self.account_id + ":" + account_key
basic_auth_string = 'Basic ' + base64.b64encode(id_and_key)
headers = {'Authorization': basic_auth_string}
request = urllib2.Request(
'https://api.backblaze.com/b2api/v1/b2_authorize_account',
headers=headers
)
response = urllib2.urlopen(request)
response_data = json.loads(response.read())
response.close()
self.auth_token = response_data['authorizationToken']
self.api_url = response_data['apiUrl']
self.download_url = response_data['downloadUrl']
try:
self.find_or_create_bucket(self.bucket_name)
except urllib2.HTTPError:
raise FatalBackendException("Bucket cannot be created")
def reset_connection(self):
import boto3 # pylint: disable=import-error
import botocore # pylint: disable=import-error
from botocore.exceptions import ClientError # pylint: disable=import-error
self.bucket = None
self.s3 = boto3.resource(u's3', region_name=config.s3_region_name, endpoint_url=config.s3_endpoint_url)
try:
self.s3.meta.client.head_bucket(Bucket=self.bucket_name)
except botocore.exceptions.ClientError as bce:
error_code = bce.response[u'Error'][u'Code']
if error_code == u'404':
raise FatalBackendException(u'S3 bucket "%s" does not exist' % self.bucket_name,
code=log.ErrorCode.backend_not_found)
else:
raise
self.bucket = self.s3.Bucket(self.bucket_name) # only set if bucket is thought to exist.
def _put(self, source_path, remote_filename):
remote_filename = util.fsdecode(remote_filename)
if config.s3_european_buckets:
if not config.s3_use_new_style:
raise FatalBackendException(
u"European bucket creation was requested, but not new-style "
u"bucket addressing (--s3-use-new-style)",
code=log.ErrorCode.s3_bucket_not_style)
if self.bucket is None:
try:
self.bucket = self.conn.get_bucket(self.bucket_name)
except Exception as e:
if u"NoSuchBucket" in str(e):
self.bucket = self.conn.create_bucket(
self.bucket_name, location=self.my_location)
else:
raise
key = self.bucket.new_key(self.key_prefix + remote_filename)
if config.s3_use_rrs:
storage_class = u'REDUCED_REDUNDANCY'
elif config.s3_use_ia:
storage_class = u'STANDARD_IA'
elif config.s3_use_onezone_ia:
storage_class = u'ONEZONE_IA'
elif config.s3_use_glacier and u"manifest" not in remote_filename:
storage_class = u'GLACIER'
else:
storage_class = u'STANDARD'
log.Info(u"Uploading %s/%s to %s Storage" %
(self.straight_url, remote_filename, storage_class))
if config.s3_use_sse:
headers = {
u'Content-Type': u'application/octet-stream',
u'x-amz-storage-class': storage_class,
u'x-amz-server-side-encryption': u'AES256'
}
elif config.s3_use_sse_kms:
if config.s3_kms_key_id is None:
raise FatalBackendException(
u"S3 USE SSE KMS was requested, but key id not provided "
u"require (--s3-kms-key-id)",
code=log.ErrorCode.s3_kms_no_id)
headers = {
u'Content-Type':
u'application/octet-stream',
u'x-amz-storage-class':
storage_class,
u'x-amz-server-side-encryption':
u'aws:kms',
u'x-amz-server-side-encryption-aws-kms-key-id':
config.s3_kms_key_id
}
if config.s3_kms_grant is not None:
headers[u'x-amz-grant-full-control'] = config.s3_kms_grant
else:
headers = {
u'Content-Type': u'application/octet-stream',
u'x-amz-storage-class': storage_class
}
upload_start = time.time()
self.upload(source_path.name, key, headers)
upload_end = time.time()
total_s = abs(upload_end - upload_start) or 1 # prevent a zero value!
rough_upload_speed = os.path.getsize(source_path.name) / total_s
log.Debug(u"Uploaded %s/%s to %s Storage at roughly %f bytes/second" %
(self.straight_url, remote_filename, storage_class,
rough_upload_speed))
