def _delete(self, filename):
# Store an indication on whether any passed
passed = False
stores = self._eligible_stores(filename)
# since the backend operations will be retried, we can't
# simply try to get from the store, if not found, move to the
# next store (since each failure will be retried n times
# before finally giving up). So we need to get the list first
# before we try to delete
# ENHANCEME: maintain a cached list for each store
for s in stores:
list = s.list()
if filename in list:
s._do_delete(filename)
passed = True
# In stripe mode, only one item will have the file
if self.__mode == 'stripe':
return
log.Log(
_("MultiBackend: failed to delete %s from %s") %
(filename, s.backend.parsed_url.url_string), log.INFO)
if not passed:
log.Log(
_("MultiBackend: failed to delete %s. Tried all backing stores and none succeeded"
) % (filename), log.ERROR)
def __init__(self, parsed_url):
u"""
Authorize to B2 api and set up needed variables
"""
duplicity.backend.Backend.__init__(self, parsed_url)
global DownloadDestLocalFile, FileVersionInfoFactory
try: # try to import the new b2sdk if available
from b2sdk.api import B2Api
from b2sdk.account_info import InMemoryAccountInfo
from b2sdk.download_dest import DownloadDestLocalFile
from b2sdk.exception import NonExistentBucket
from b2sdk.file_version import FileVersionInfoFactory
except ImportError as e:
if u'b2sdk' in getattr(e, u'name', u'b2sdk'):
raise
try: # fall back to import the old b2 client
from b2.api import B2Api
from b2.account_info import InMemoryAccountInfo
from b2.download_dest import DownloadDestLocalFile
from b2.exception import NonExistentBucket
from b2.file_version import FileVersionInfoFactory
except ImportError:
if u'b2' in getattr(e, u'name', u'b2'):
raise
raise BackendException(
u'B2 backend requires B2 Python SDK (pip install b2sdk)')
self.service = B2Api(InMemoryAccountInfo())
self.parsed_url.hostname = u'B2'
account_id = parsed_url.username
account_key = self.get_password()
self.url_parts = [
x for x in parsed_url.path.replace(u"@", u"/").split(u'/')
if x != u''
]
if self.url_parts:
self.username = self.url_parts.pop(0)
bucket_name = self.url_parts.pop(0)
else:
raise BackendException(u"B2 requires a bucket name")
self.path = u"".join([url_part + u"/" for url_part in self.url_parts])
self.service.authorize_account(u'production', account_id, account_key)
log.Log(
u"B2 Backend (path= %s, bucket= %s, minimum_part_size= %s)" %
(self.path, bucket_name,
self.service.account_info.get_minimum_part_size()), log.INFO)
try:
self.bucket = self.service.get_bucket_by_name(bucket_name)
log.Log(u"Bucket found", log.INFO)
except NonExistentBucket:
try:
log.Log(u"Bucket not found, creating one", log.INFO)
self.bucket = self.service.create_bucket(
bucket_name, u'allPrivate')
except:
raise FatalBackendException(u"Bucket cannot be created")
def addtotree(root, path, perm, size, mtime, uid, gid, type):
if len(path) == 1:
c = path[0]
ec = pathencode(c)
e = None
for f in root.getchildren():
if f.tag == ec:
e = f
break
if e is None:
e = SubElement(root, ec)
log.Log("add " + c + "(" + ec + ") to " + str(root), 5)
else:
log.Log("found " + c + "(" + ec + ") in " + str(root), 7)
e.set("perm", perm)
e.set("size", -1)
e.set("mtime", mtime)
e.set("uid", uid)
e.set("gid", gid)
e.set("type", type)
e.set("name", c)
else:
c = path[0]
ec = pathencode(c)
for f in root.getchildren():
if f.tag == ec:
log.Log(
"adding " + path[1] + " to " + c + "(" + ec + ") in " +
str(root), 7)
addtotree(f, path[1:], perm, size, mtime, uid, gid, type)
return
f = SubElement(root, ec)
log.Log("new " + c + "(" + ec + ") in " + str(root), 5)
addtotree(f, path[1:], perm, size, mtime, uid, gid, type)
def get_query_params(parsed_url):
# Reparse so the query string is available
reparsed_url = urlparse.urlparse(parsed_url.geturl())
if len(reparsed_url.query) == 0:
return dict()
try:
queryMultiDict = urlparse.parse_qs(reparsed_url.query,
strict_parsing=True)
except ValueError as e:
log.Log(
_("MultiBackend: Could not parse query string %s: %s ") %
(reparsed_url.query, e), log.ERROR)
raise BackendException('Could not parse query string')
queryDict = dict()
# Convert the multi-dict to a single dictionary
# while checking to make sure that no unrecognized values are found
for name, valueList in queryMultiDict.items():
if len(valueList) != 1:
log.Log(
_("MultiBackend: Invalid query string %s: more than one value for %s"
) % (reparsed_url.query, name), log.ERROR)
raise BackendException('Invalid query string')
if name not in MultiBackend.__knownQueryParameters:
log.Log(
_("MultiBackend: Invalid query string %s: unknown parameter %s"
) % (reparsed_url.query, name), log.ERROR)
raise BackendException('Invalid query string')
queryDict[name] = valueList[0]
return queryDict
def _list(self):
"""
List files on remote server
"""
log.Log("Listing files", 9)
endpoint = 'b2_list_file_names'
url = self.formatted_url(endpoint)
params = {
'bucketId': self.bucket_id,
'maxFileCount': 1000,
}
try:
resp = self.get_or_post(url, params)
except urllib2.HTTPError:
return []
files = [x['fileName'].split('/')[-1] for x in resp['files']
if os.path.dirname(x['fileName']) == self.path]
next_file = resp['nextFileName']
while next_file:
log.Log("There are still files, getting next list", 9)
params['startFileName'] = next_file
try:
resp = self.get_or_post(url, params)
except urllib2.HTTPError:
return files
files += [x['fileName'].split('/')[-1] for x in resp['files']
if os.path.dirname(x['fileName']) == self.path]
next_file = resp['nextFileName']
return files
def _put(self, source_path, remote_filename):
first = self.__write_cursor
while True:
store = self.__stores[self.__write_cursor]
try:
next = self.__write_cursor + 1
if (next > len(self.__stores) - 1):
next = 0
log.Log(_("MultiBackend: _put: write to store #%s (%s)")
% (self.__write_cursor, store.backend.parsed_url.url_string),
log.DEBUG)
store.put(source_path, remote_filename)
self.__write_cursor = next
break
except Exception as e:
log.Log(_("MultiBackend: failed to write to store #%s (%s), try #%s, Exception: %s")
% (self.__write_cursor, store.backend.parsed_url.url_string, next, e),
log.INFO)
self.__write_cursor = next
if (self.__write_cursor == first):
log.Log(_("MultiBackend: failed to write %s. Tried all backing stores and none succeeded")
% (source_path),
log.ERROR)
raise BackendException("failed to write")
def getattr(self, path):
log.Log("getattr " + path, 5)
st = DuplicityStat()
p = path[1:].split(os.path.sep)
if path == '/':
st.st_mode = stat.S_IFDIR | 0o755
st.st_nlink = 1 + len(self.dircache.keys())
return st
if len(p) == 1:
if not self.dircache.has_key(p[0]):
return -errno.ENOENT
st.st_mode = stat.S_IFDIR | 0o755
st.st_nlink = 2
return st
e = findpath(self.fillcache(p[0]), p[1:])
if e is None:
return -errno.ENOENT
mode = int(
(3 * '{:b}').format(*[int(x) for x in e.get("perm").split()[-1]]),
base=2)
if e.get("type") == 'dir':
st.st_mode = stat.S_IFDIR | mode
e.set("size", 0)
else:
st.st_mode = stat.S_IFREG | mode
if int(self.filemode) == 0 and e.get(
"size"
) < 0: # need to read size from filearch? not in signature?
ds = [d[0] for d in self.date_types if date2str(d[0]) in p[0]]
np = os.path.join(*p[1:])
files = restore_get_patched_rop_iter(self.col_stats,
date2num(ds[0]), tuple(p[1:]))
for x in files[0]:
lp = os.path.join(*[
y for y in (np + os.path.sep +
x.get_relative_path()).split(os.path.sep)
if y != '.'
])
log.Log("looking at %s for %s" % (lp, np), 5)
le = findpath(self.dircache[p[0]], lp.split(os.path.sep))
if le is None:
log.Log("not found in dircache: " + str(le), 7)
continue
if le.get("size") < 0:
le.set("size", x.getsize())
if lp == np:
log.Log("found " + np, 5)
break
for x in files[1]:
x.close()
elif e.get("size") < 0:
e.set("size", 0)
st.st_size = e.get("size")
st.st_uid = e.get("uid")
st.st_gid = e.get("gid")
st.st_mtime = e.get("mtime")
st.st_nlink = 1 + len(e.getchildren())
return st
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
# Init each of the wrapped stores
#
# config file is a json formatted collection of values, one for
# each backend. We will 'stripe' data across all the given stores:
#
# 'url' - the URL used for the backend store
# 'env' - an optional list of enviroment variable values to set
# during the intialization of the backend
#
# Example:
#
# [
# {
# "url": "abackend://[email protected]/backup",
# "env": [
# {
# "name" : "MYENV",
# "value" : "xyz"
# },
# {
# "name" : "FOO",
# "value" : "bar"
# }
# ]
# },
# {
# "url": "file:///path/to/dir"
# }
# ]
try:
with open(parsed_url.path) as f:
configs = json.load(f)
except IOError as e:
log.Log(_("MultiBackend: Could not load config file %s: %s ")
% (parsed_url.path, e),
log.ERROR)
raise BackendException('Could not load config file')
for config in configs:
url = config['url']
log.Log(_("MultiBackend: use store %s")
% (url),
log.INFO)
if 'env' in config:
for env in config['env']:
log.Log(_("MultiBackend: set env %s = %s")
% (env['name'], env['value']),
log.INFO)
os.environ[env['name']] = env['value']
store = duplicity.backend.get_backend(url)
self.__stores.append(store)
def fillcache(self, p):
if self.dircache[p] is None:
log.Log("filling cache "+p, 5)
for s in xrange(0, len(self.date_types)):
if date2str(self.date_types[s][0]) in p:
log.Log("filling cache for "+str(s)+" "+p, 5)
t = date2num(self.date_types[s][0])
self.dircache[p] = getfiletree(p, self.col_stats.get_signature_chain_at_time(t).get_fileobjs(t))
break
return self.dircache[p]
def _list(self):
lists = []
for s in self.__stores:
l = s.list()
log.Log(
_("MultiBackend: list from %s: %s") %
(s.backend.parsed_url.url_string, l), log.DEBUG)
lists.append(s.list())
# combine the lists into a single flat list w/o duplicates via set:
result = list({item for sublist in lists for item in sublist})
log.Log(_("MultiBackend: combined list: %s") % (result), log.DEBUG)
return result
def _put(self, source_path, remote_filename):
# Store an indication of whether any of these passed
passed = False
# Eligibile stores for this action
stores = self._eligible_stores(remote_filename)
# Mirror mode always starts at zero
if self.__mode == 'mirror':
self.__write_cursor = 0
first = self.__write_cursor
while True:
store = stores[self.__write_cursor]
try:
next = self.__write_cursor + 1
if (next > len(stores) - 1):
next = 0
log.Log(
_("MultiBackend: _put: write to store #%s (%s)") %
(self.__write_cursor, store.backend.parsed_url.url_string),
log.DEBUG)
store.put(source_path, remote_filename)
passed = True
self.__write_cursor = next
# No matter what, if we loop around, break this loop
if next == 0:
break
# If in stripe mode, don't continue to the next
if self.__mode == 'stripe':
break
except Exception as e:
log.Log(
_("MultiBackend: failed to write to store #%s (%s), try #%s, Exception: %s"
) % (self.__write_cursor,
store.backend.parsed_url.url_string, next, e),
log.INFO)
self.__write_cursor = next
# If we consider write failure as abort, abort
if self.__onfail_mode == 'abort':
log.Log(
_("MultiBackend: failed to write %s. Aborting process."
) % (source_path), log.ERROR)
raise BackendException("failed to write")
# If we've looped around, and none of them passed, fail
if (self.__write_cursor == first) and not passed:
log.Log(
_("MultiBackend: failed to write %s. Tried all backing stores and none succeeded"
) % (source_path), log.ERROR)
raise BackendException("failed to write")
def __init__(self, parsed_url):
u"""
Authorize to B2 api and set up needed variables
"""
duplicity.backend.Backend.__init__(self, parsed_url)
# Import B2 API
try:
global b2
import b2
global b2sdk
import b2sdk
import b2.api
import b2.account_info
import b2.download_dest
import b2.file_version
except ImportError:
raise BackendException(
u'B2 backend requires B2 Python APIs (pip install b2)')
self.service = b2.api.B2Api(b2.account_info.InMemoryAccountInfo())
self.parsed_url.hostname = u'B2'
account_id = parsed_url.username
account_key = self.get_password()
self.url_parts = [
x for x in parsed_url.path.replace(u"@", u"/").split(u'/')
if x != u''
]
if self.url_parts:
self.username = self.url_parts.pop(0)
bucket_name = self.url_parts.pop(0)
else:
raise BackendException(u"B2 requires a bucket name")
self.path = u"".join([url_part + u"/" for url_part in self.url_parts])
self.service.authorize_account(u'production', account_id, account_key)
log.Log(
u"B2 Backend (path= %s, bucket= %s, minimum_part_size= %s)" %
(self.path, bucket_name,
self.service.account_info.get_minimum_part_size()), log.INFO)
try:
self.bucket = self.service.get_bucket_by_name(bucket_name)
log.Log(u"Bucket found", log.INFO)
except b2.exception.NonExistentBucket:
try:
log.Log(u"Bucket not found, creating one", log.INFO)
self.bucket = self.service.create_bucket(
bucket_name, u'allPrivate')
except:
raise FatalBackendException(u"Bucket cannot be created")
def findpath(root, path):
if len(path) == 0:
return root
c = path[0]
ec = pathencode(c)
s = root.find(ec)
if s is None:
log.Log("node "+c+"("+ec+") in "+str(root)+" not found", 5)
return None
if len(path) == 1:
log.Log("node "+c+"("+ec+") in "+str(root)+" found", 5)
return s
log.Log("search "+path[1]+" in "+c+"("+ec+") in "+str(root), 5)
return findpath(s, path[1:])
def schedule_task(self, fn, params):
"""
Schedule the given task (callable, typically function) for
execution. Pass the given parameters to the function when
calling it. Returns a callable which can optionally be used
to wait for the task to complete, either by returning its
return value or by propagating any exception raised by said
task.
This method may block or return immediately, depending on the
configuration and state of the scheduler.
This method may also raise an exception in order to trigger
failures early, if the task (if run synchronously) or a previous
task has already failed.
NOTE: Pay particular attention to the scope in which this is
called. In particular, since it will execute concurrently in
the background, assuming fn is a closure, any variables used
most be properly bound in the closure. This is the reason for
the convenience feature of being able to give parameters to
the call, to avoid having to wrap the call itself in a
function in order to "fixate" variables in, for example, an
enclosing loop.
"""
assert fn is not None
# Note: It is on purpose that we keep track of concurrency in
# the front end and launch threads for each task, rather than
# keep a pool of workers. The overhead is not relevant in the
# situation this will be used, and it removes complexity in
# terms of ensuring the scheduler is garbage collected/shut
# down properly when no longer referenced/needed by calling
# code.
if self.__concurrency == 0:
# special case this to not require any platform support for
# threading at all
log.Log("%s: %s" % (self.__class__.__name__,
_("running task synchronously (asynchronicity disabled)")),
log.DEBUG, log.InfoCode.synchronous_upload_begin)
return self.__run_synchronously(fn, params)
else:
log.Log("%s: %s" % (self.__class__.__name__,
_("scheduling task for asynchronous execution")),
log.DEBUG,
log.InfoCode.asynchronous_upload_begin)
return self.__run_asynchronously(fn, params)
def __enter__(self):
request = urllib2.Request(self.url, self.data, self.headers)
self.file = urllib2.urlopen(request)
log.Log(
"Request of %s returned with status %s" %
(self.url, self.file.code), 9)
return self.file
def import_backends():
"""
Import files in the duplicity/backends directory where
the filename ends in 'backend.py' and ignore the rest.
@rtype: void
@return: void
"""
path = duplicity.backends.__path__[0]
assert path.endswith("duplicity/backends"), duplicity.backends.__path__
files = os.listdir(path)
for fn in files:
if fn.endswith("backend.py"):
fn = fn[:-3]
imp = "duplicity.backends.%s" % (fn, )
# ignore gio as it is explicitly loaded in commandline.parse_cmdline_options()
if fn == "giobackend": continue
try:
__import__(imp)
res = "Succeeded"
level = log.INFO
except Exception:
res = "Failed: " + str(sys.exc_info()[1])
level = log.WARNING
log.Log("Import of %s %s" % (imp, res), level)
else:
continue
def _delete_list(self, filenames):
# Store an indication on whether any passed
passed = False
stores = self.__stores
# since the backend operations will be retried, we can't
# simply try to get from the store, if not found, move to the
# next store (since each failure will be retried n times
# before finally giving up). So we need to get the list first
# before we try to delete
# ENHANCEME: maintain a cached list for each store
for s in stores:
flist = s.list()
cleaned = [f for f in filenames if f in flist]
if hasattr(s.backend, u'_delete_list'):
s._do_delete_list(cleaned)
elif hasattr(s.backend, u'_delete'):
for filename in cleaned:
s._do_delete(filename)
passed = True
# In stripe mode, only one item will have the file
if self.__mode == u'stripe':
return
if not passed:
log.Log(
_(u"MultiBackend: failed to delete %s. Tried all backing stores and none succeeded"
) % (filenames), log.ERROR)
def _delete(self, filename):
"""
Delete filename from remote server
"""
log.Log("Delete: %s" % self.path + filename, log.INFO)
file_version_info = self.file_info(self.path + filename)
self.bucket.delete_file_version(file_version_info.id_, file_version_info.file_name)
def import_backends():
"""
Import files in the duplicity/backends directory where
the filename ends in 'backend.py' and ignore the rest.
@rtype: void
@return: void
"""
path = duplicity.backends.__path__[0]
assert path.endswith("duplicity/backends"), duplicity.backends.__path__
files = os.listdir(path)
files.sort()
for fn in files:
if fn.endswith("backend.py"):
fn = fn[:-3]
imp = "duplicity.backends.%s" % (fn,)
try:
__import__(imp)
res = "Succeeded"
except Exception:
res = "Failed: " + str(sys.exc_info()[1])
log.Log(_("Import of %s %s") % (imp, res), log.INFO)
else:
continue
def old_globbing_filelist_deprecation(opt):
log.Log(_(
"Warning: Option %s is pending deprecation and will be removed in a future release.\n"
"--include-filelist and --exclude-filelist now accept globbing characters and should "
"be used instead.") % opt,
log.ERROR,
force_print=True)
def _get(self, remote_filename, local_path):
"""
Download remote_filename to local_path
"""
log.Log("Get: %s -> %s" % (self.path + remote_filename, local_path.name), log.INFO)
self.bucket.download_file_by_name(self.path + remote_filename,
b2.download_dest.DownloadDestLocalFile(local_path.name))
def _query(self, filename):
u"""
Get size info of filename
"""
log.Log(u"Query: %s" % self.path + util.fsdecode(filename), log.INFO)
file_version_info = self.file_info(quote_plus(self.path + util.fsdecode(filename), u'/'))
return {u'size': file_version_info.size
if file_version_info is not None and file_version_info.size is not None else -1}
def _put(self, source_path, remote_filename):
"""
Copy source_path to remote_filename
"""
log.Log("Put: %s -> %s" % (source_path.name, self.path + remote_filename), log.INFO)
self.bucket.upload_local_file(source_path.name, self.path + remote_filename,
content_type='application/pgp-encrypted',
progress_listener=B2ProgressListener())
def _query(self, filename):
"""
Get size info of filename
"""
log.Log("Query: %s" % self.path + filename, log.INFO)
file_version_info = self.file_info(self.path + filename)
return {'size': file_version_info.size
if file_version_info is not None and file_version_info.size is not None else -1}
def _delete(self, filename):
# since the backend operations will be retried, we can't
# simply try to get from the store, if not found, move to the
# next store (since each failure will be retried n times
# before finally giving up). So we need to get the list first
# before we try to delete
# ENHANCEME: maintain a cached list for each store
for s in self.__stores:
list = s.list()
if filename in list:
s._do_delete(filename)
return
log.Log(_("MultiBackend: failed to delete %s from %s")
% (filename, s.backend.parsed_url.url_string),
log.INFO)
log.Log(_("MultiBackend: failed to delete %s. Tried all backing stores and none succeeded")
% (filename),
log.ERROR)
def _get(self, remote_filename, local_path):
u"""
Download remote_filename to local_path
"""
log.Log(u"Get: %s -> %s" % (self.path + util.fsdecode(remote_filename),
util.fsdecode(local_path.name)),
log.INFO)
self.bucket.download_file_by_name(quote_plus(self.path + util.fsdecode(remote_filename), u'/'),
DownloadDestLocalFile(local_path.name))
def stdin_deprecation(opt):
# See https://bugs.launchpad.net/duplicity/+bug/1423367
# In almost all Linux distros stdin is a file represented by /dev/stdin,
# so --exclude-file=/dev/stdin will work as a substitute.
log.Log(_("Warning: Option %s is pending deprecation and will be removed in a future release.\n"
"On many GNU/Linux systems, stdin is represented by /dev/stdin and\n"
"--include-filelist=/dev/stdin or --exclude-filelist=/dev/stdin could\n"
"be used as a substitute.") % opt,
log.ERROR, force_print=True)
def _query(self, filename):
"""
Get size info of filename
"""
log.Log("Querying file %s" % filename, 9)
info = self.get_file_info(filename)
if not info:
return {'size': -1}
return {'size': info['size']}
def _get(self, remote_filename, local_path):
# since the backend operations will be retried, we can't
# simply try to get from the store, if not found, move to the
# next store (since each failure will be retried n times
# before finally giving up). So we need to get the list first
# before we try to fetch
# ENHANCEME: maintain a cached list for each store
for s in self.__stores:
list = s.list()
if remote_filename in list:
s.get(remote_filename, local_path)
return
log.Log(_("MultiBackend: failed to get %s to %s from %s")
% (remote_filename, local_path, s.backend.parsed_url.url_string),
log.INFO)
log.Log(_("MultiBackend: failed to get %s. Tried all backing stores and none succeeded")
% (remote_filename),
log.ERROR)
raise BackendException("failed to get")
def runduplicity(self):
if self.url is None:
return
log.setup()
log.setverbosity(int(self.debuglevel))
if self.passphrasefd:
self.passphrasefd = int(self.passphrasefd)
if self.passwordfd:
self.passwordfd = int(self.passwordfd)
if self.url.find("file:/") != 0:
get_backendpassphrase(self.passwordfd)
opts = []
for i in self.options:
try:
v = eval("self." + i.replace("-", ""))
if v:
opts.append("--%s=%s" % (i, v))
except:
pass
for i in self.no_options:
try:
v = eval("self." + i.replace("-", ""))
if v:
opts.append("--%s" % (i))
except:
pass
self.options = []
parameter = ["list-current-files", "--ssh-askpass"] + opts + [self.url]
log.Log("processing %s" % (" ".join(parameter)), 5)
sys.argv = ["duplicity"] + parameter
action = commandline.ProcessCommandLine(parameter)
log.Log("running action %s" % (action), 5)
globals.gpg_profile.passphrase = get_passphrase(self.passphrasefd)
self.col_stats = collections.CollectionsStatus(
globals.backend, globals.archive_dir,
"collection-status").set_values()
self.date_types = []
for chain in self.col_stats.all_backup_chains:
for s in chain.get_all_sets():
self.date_types.append(
(datetime.fromtimestamp(s.get_time()), s.type))
for s in self.date_types:
self.dircache[date2str(s[0]) + '_' + s[1]] = None
