def log_exception(e):
log.Error('Exception [%s]:' % (e, ))
f = StringIO.StringIO()
traceback.print_exc(file=f)
f.seek(0)
for s in f.readlines():
log.Error('| ' + s.rstrip())
f.close()
def command(login_required=True):
"""a decorator for handling authentication and exceptions"""
def decorate(f):
def wrapper(self, *args):
if login_required and not self.sess.is_linked():
log.FatalError("dpbx Cannot login: check your credentials",
log.ErrorCode.dpbx_nologin)
return
try:
return f(self, *args)
except TypeError, e:
log_exception(e)
log.FatalError('dpbx type error "%s"' % (e, ),
log.ErrorCode.backend_code_error)
except rest.ErrorResponse, e:
msg = e.user_error_msg or str(e)
log.Error('dpbx error: %s' % (msg, ),
log.ErrorCode.backend_command_error)
raise e
except Exception, e:
log_exception(e)
log.Error('dpbx code error "%s"' % (e, ),
log.ErrorCode.backend_code_error)
raise e
def token_updater(token):
try:
with open(self.OAUTH_TOKEN_PATH, u'w') as f:
json.dump(token, f)
except Exception as e:
log.Error((u'Could not save the OAuth2 token to %s. '
u'This means you may need to do the OAuth2 '
u'authorization process again soon. '
u'Original error: %s' % (self.OAUTH_TOKEN_PATH, e)))
def token_updater(token):
u"""Stores oauth2 token on disk"""
try:
with open(self.OAUTH_TOKEN_PATH, u'w') as f:
json.dump(token, f)
except Exception as err:
log.Error(u'Could not save the OAuth2 token to %s. This means '
u'you may need to do the OAuth2 authorization '
u'process again soon. Original error: %s' %
(self.OAUTH_TOKEN_PATH, err))
def wrapper(self, *args):
try:
return f(self, *args)
except ApiError as e:
log_exception(e)
raise BackendException(u'dpbx api error "%s"' % (e,))
except Exception as e:
log_exception(e)
log.Error(u'dpbx code error "%s"' % (e,), log.ErrorCode.backend_code_error)
raise
def get(self, remote_filename, local_path):
u"""transfer remote_filename and the related .par2 file into
a temp-dir. remote_filename will be renamed into local_path before
finishing.
If "par2 verify" detect an error transfer the Par2-volumes into the
temp-dir and try to repair.
"""
par2temp = local_path.get_temp_in_same_dir()
par2temp.mkdir()
local_path_temp = par2temp.append(remote_filename)
self.wrapped_backend._get(remote_filename, local_path_temp)
try:
par2file = par2temp.append(remote_filename + b'.par2')
self.wrapped_backend._get(par2file.get_filename(), par2file)
par2verify = u'par2 v %s %s "%s"' % (
self.common_options, util.fsdecode(par2file.get_canonical()),
util.fsdecode(local_path_temp.get_canonical()))
out, returncode = pexpect.run(par2verify, None, True)
if returncode:
log.Warn(u"File is corrupt. Try to repair %s" %
remote_filename)
c = re.compile(u'%s\\.vol[\\d+]*\\.par2' %
remote_filename.decode())
par2volumes = [
f for f in self.wrapped_backend._list()
if c.match(util.fsdecode(f))
]
for filename in par2volumes:
file = par2temp.append(filename)
self.wrapped_backend._get(filename, file)
par2repair = u'par2 r %s %s "%s"' % (
self.common_options, util.fsdecode(
par2file.get_canonical()),
util.fsdecode(local_path_temp.get_canonical()))
out, returncode = pexpect.run(par2repair, None, True)
if returncode:
log.Error(u"Failed to repair %s" % remote_filename)
else:
log.Warn(u"Repair successful %s" % remote_filename)
except BackendException:
# par2 file not available
pass
finally:
local_path_temp.rename(local_path)
par2temp.deltree()
def release_lockfile():
if config.lockfile:
log.Debug(_(u"Releasing lockfile %s") % config.lockpath)
try:
config.lockfile.release()
config.lockfile = None
os.remove(config.lockpath)
config.lockpath = u""
except Exception as e:
log.Error(u"Could not release lockfile: %s" % str(e))
pass
def list_filenames_in_bucket(self):
import oss2
filename_list = []
for item in oss2.ObjectIterator(self.bucket, prefix=self.key_prefix):
try:
filename = item.key.replace(self.key_prefix, '', 1)
filename_list.append(filename)
log.Debug("Listed %s/%s" % (self.straight_url, filename))
except AttributeError:
log.Error("List AttributeError")
return filename_list
def wrapper(self, *args):
if login_required and not self.sess.is_linked():
raise BackendException(
"dpbx Cannot login: check your credentials",
log.ErrorCode.dpbx_nologin)
return
try:
return f(self, *args)
except TypeError as e:
log_exception(e)
raise BackendException('dpbx type error "%s"' % (e, ))
except rest.ErrorResponse as e:
msg = e.user_error_msg or util.uexc(e)
log.Error('dpbx error: %s' % (msg, ),
log.ErrorCode.backend_command_error)
raise e
except Exception as e:
log_exception(e)
log.Error('dpbx code error "%s"' % (e, ),
log.ErrorCode.backend_code_error)
raise e
def get_remote_manifest(self):
"""
Return manifest by reading remote manifest on backend
"""
assert self.remote_manifest_name
try:
manifest_buffer = self.backend.get_data(self.remote_manifest_name)
except GPGError as message:
log.Error(_("Error processing remote manifest (%s): %s") %
(util.ufn(self.remote_manifest_name), util.uexc(message)))
return None
log.Info(_("Processing remote manifest %s (%s)") % (
util.ufn(self.remote_manifest_name), len(manifest_buffer)))
return manifest.Manifest().from_string(manifest_buffer)
def transfer(self, method, source_path, remote_filename):
u"""create Par2 files and transfer the given file and the Par2 files
with the wrapped backend.
Par2 must run on the real filename or it would restore the
temp-filename later on. So first of all create a tempdir and symlink
the soure_path with remote_filename into this.
"""
par2temp = source_path.get_temp_in_same_dir()
par2temp.mkdir()
source_symlink = par2temp.append(remote_filename)
source_target = source_path.get_canonical()
if not os.path.isabs(source_target):
source_target = os.path.join(util.fsencode(os.getcwd()),
source_target)
os.symlink(source_target, source_symlink.get_canonical())
source_symlink.setdata()
log.Info(u"Create Par2 recovery files")
par2create = u'par2 c -r%d -n%d %s "%s"' % (
self.redundancy, self.volumes, self.common_options,
util.fsdecode(source_symlink.get_canonical()))
out, returncode = pexpect.run(par2create, None, True)
if returncode:
log.Warn(
u"Failed to create par2 file with requested options, retrying with -n1"
)
par2create = u'par2 c -r%d -n1 %s "%s"' % (
self.redundancy, self.common_options,
util.fsdecode(source_symlink.get_canonical()))
out, returncode = pexpect.run(par2create, None, True)
if not returncode:
log.Warn(u"Successfully created par2 file with -n1")
source_symlink.delete()
files_to_transfer = []
if not returncode:
for file in par2temp.listdir():
files_to_transfer.append(par2temp.append(file))
else:
log.Error(u"FAILED to create par2 file with returncode %d" %
returncode)
method(source_path, remote_filename)
for file in files_to_transfer:
method(file, file.get_filename())
par2temp.deltree()
def from_string(self, s):
u"""
Initialize self from string s, return self
"""
def get_field(fieldname):
u"""
Return the value of a field by parsing s, or None if no field
"""
if not isinstance(fieldname, bytes):
fieldname = fieldname.encode()
m = re.search(b"(^|\\n)%s\\s(.*?)\n" % fieldname, s, re.I)
if not m:
return None
else:
return Unquote(m.group(2))
self.hostname = get_field(u"hostname")
if self.hostname is not None:
self.hostname = self.hostname.decode()
self.local_dirname = get_field(u"localdir")
highest_vol = 0
latest_vol = 0
vi_regexp = re.compile(
b"(?:^|\\n)(volume\\s.*(?:\\n.*)*?)(?=\\nvolume\\s|$)", re.I)
vi_iterator = vi_regexp.finditer(s)
for match in vi_iterator:
vi = VolumeInfo().from_string(match.group(1))
self.add_volume_info(vi)
latest_vol = vi.volume_number
highest_vol = max(highest_vol, latest_vol)
log.Debug(_(u"Found manifest volume %s") % latest_vol)
# If we restarted after losing some remote volumes, the highest volume
# seen may be higher than the last volume recorded. That is, the
# manifest could contain "vol1, vol2, vol3, vol2." If so, we don't
# want to keep vol3's info.
for i in range(latest_vol + 1, highest_vol + 1):
self.del_volume_info(i)
log.Info(_(u"Found %s volumes in manifest") % latest_vol)
# Get file changed list - not needed if --file-changed and
# --show-changes-in-set are not present
filecount = 0
if config.file_changed is not None or config.show_changes_in_set is not None:
filelist_regexp = re.compile(
b"(^|\\n)filelist\\s([0-9]+)\\n(.*?)(\\nvolume\\s|$)",
re.I | re.S)
match = filelist_regexp.search(s)
if match:
filecount = int(match.group(2))
if filecount > 0:
def parse_fileinfo(line):
fileinfo = line.strip().split()
return (fileinfo[0], b''.join(fileinfo[1:]))
self.files_changed = list(
map(parse_fileinfo,
match.group(3).split(b'\n')))
if filecount != len(self.files_changed):
log.Error(
_(u"Manifest file '%s' is corrupt: File count says %d, File list contains %d"
% (self.fh.base if self.fh else u"", filecount,
len(self.files_changed))))
self.corrupt_filelist = True
return self
def initialize_oauth2_session(self):
def token_updater(token):
try:
with open(self.OAUTH_TOKEN_PATH, u'w') as f:
json.dump(token, f)
except Exception as e:
log.Error((u'Could not save the OAuth2 token to %s. '
u'This means you may need to do the OAuth2 '
u'authorization process again soon. '
u'Original error: %s' % (self.OAUTH_TOKEN_PATH, e)))
token = None
try:
with open(self.OAUTH_TOKEN_PATH) as f:
token = json.load(f)
except IOError as e:
log.Error(
(u'Could not load OAuth2 token. '
u'Trying to create a new one. (original error: %s)' % e))
self.http_client = OAuth2Session(self.CLIENT_ID,
scope=self.OAUTH_SCOPE,
redirect_uri=self.OAUTH_REDIRECT_URI,
token=token,
auto_refresh_kwargs={
u'client_id': self.CLIENT_ID,
},
auto_refresh_url=self.OAUTH_TOKEN_URI,
token_updater=token_updater)
# We have to refresh token manually because it's not working "under the covers"
if token is not None:
self.http_client.refresh_token(self.OAUTH_TOKEN_URI)
# Send a request to make sure the token is valid (or could at least be
# refreshed successfully, which will happen under the covers). In case
# this request fails, the provided token was too old (i.e. expired),
# and we need to get a new token.
user_info_response = self.http_client.get(self.API_URI + u'me')
if user_info_response.status_code != requests.codes.ok:
token = None
if token is None:
if not sys.stdout.isatty() or not sys.stdin.isatty():
log.FatalError(
(u'The OAuth2 token could not be loaded from %s '
u'and you are not running duplicity '
u'interactively, so duplicity cannot possibly '
u'access OneDrive.' % self.OAUTH_TOKEN_PATH))
authorization_url, state = self.http_client.authorization_url(
self.OAUTH_AUTHORIZE_URI, display=u'touch')
print()
print(u'In order to authorize duplicity to access your OneDrive, '
u'please open %s in a browser and copy the URL of the blank '
u'page the dialog leads to.' % authorization_url)
print()
redirected_to = input(u'URL of the blank page: ').strip()
token = self.http_client.fetch_token(
self.OAUTH_TOKEN_URI, authorization_response=redirected_to)
user_info_response = self.http_client.get(self.API_URI + u'me')
user_info_response.raise_for_status()
try:
with open(self.OAUTH_TOKEN_PATH, u'w') as f:
json.dump(token, f)
except Exception as e:
log.Error(
(u'Could not save the OAuth2 token to %s. '
u'This means you need to do the OAuth2 authorization '
u'process on every start of duplicity. '
u'Original error: %s' % (self.OAUTH_TOKEN_PATH, e)))