def connect(self):
# get the path to the command executable
path = os.environ.get(u"IDEVSPATH")
if path is None:
log.Warn(u"-" * 72)
log.Warn(
u"WARNING: No path to 'idevsutil_dedup' has been set. Download module from"
)
log.Warn(
u" https://www.idrivedownloads.com/downloads/linux/download-options/IDrive_linux_64bit.zip"
)
log.Warn(u"or")
log.Warn(
u" https://www.idrivedownloads.com/downloads/linux/download-options/IDrive_linux_32bit.zip"
)
log.Warn(
u"and place anywhere with exe rights. Then creat env var 'IDEVSPATH' with path to file"
)
log.Warn(u"-" * 72)
raise BackendException(
u"No IDEVSPATH env var set. Should contain folder to idevsutil_dedup"
)
self.cmd = os.path.join(path, u"idevsutil_dedup")
log.Debug(u"IDrive command base: %s" % (self.cmd))
# get the account-id
self.idriveid = os.environ.get(u"IDRIVEID")
if self.idriveid is None:
log.Warn(u"-" * 72)
log.Warn(u"WARNING: IDrive logon ID missing")
log.Warn(
u"Create an environment variable IDriveID with your IDrive logon ID"
)
log.Warn(u"-" * 72)
raise BackendException(
u"No IDRIVEID env var set. Should contain IDrive id")
log.Debug(u"IDrive id: %s" % (self.idriveid))
# Get the full-path to the account password file
filepath = os.environ.get(u"IDPWDFILE")
if filepath is None:
log.Warn(u"-" * 72)
log.Warn(u"WARNING: IDrive password file missging")
log.Warn(u"Please create a file with your IDrive logon password,")
log.Warn(
u"Then create an environment variable IDPWDFILE with path/filename of said file"
)
log.Warn(u"-" * 72)
raise BackendException(
u"No IDPWDFILE env var set. Should contain file with password")
log.Debug(u"IDrive pwdpath: %s" % (filepath))
self.auth_switch = u" --password-file={0}".format(filepath)
# fakeroot set? Create directory and mark for cleanup
if config.fakeroot is None:
self.cleanup = False
self.fakeroot = u''
else:
# Make sure fake root is created at root level!
self.fakeroot = os.path.join(u'/', config.fakeroot)
try:
os.mkdir(self.fakeroot)
except OSError as e:
self.cleanup = False
if e.errno == errno.EEXIST:
log.Debug(
u"Using existing directory {0} as fake-root".format(
self.fakeroot))
else:
log.Warn(u"-" * 72)
log.Warn(
u"WARNING: Creation of FAKEROOT {0} failed; backup will use system temp directory"
.format(self.fakeroot))
log.Warn(u"This might interfere with incremental backups")
log.Warn(u"-" * 72)
raise BackendException(
u"Creation of the directory {0} failed".format(
self.fakeroot))
else:
log.Debug(
u"Directory {0} created as fake-root (Will clean-up afterwards!)"
.format(self.fakeroot))
self.cleanup = True
# get the bucket
self.bucket = os.environ.get(u"IDBUCKET")
if self.bucket is None:
log.Warn(u"-" * 72)
log.Warn(u"WARNING: IDrive backup bucket missing")
log.Warn(
u"Create an environment variable IDBUCKET specifying the target bucket"
)
log.Warn(u"-" * 72)
raise BackendException(
u"No IDBUCKET env var set. Should contain IDrive backup bucket"
)
log.Debug(u"IDrive bucket: %s" % (self.bucket))
# check account / get config status and config type
el = self.request(
self.cmd + self.auth_switch +
u" --validate --user={0}".format(self.idriveid)).find(u'tree')
if el.attrib[u"message"] != u"SUCCESS":
raise BackendException(u"Protocol failure - " + el.attrib[u"desc"])
if el.attrib[u"desc"] != u"VALID ACCOUNT":
raise BackendException(u"IDrive account invalid")
if el.attrib[u"configstatus"] != u"SET":
raise BackendException(u"IDrive account not set")
# When private encryption enabled: get the full-path to a encription key file
if el.attrib[u"configtype"] == u"PRIVATE":
filepath = os.environ.get(u"IDKEYFILE")
if filepath is None:
log.Warn(u"-" * 72)
log.Warn(u"WARNING: IDrive encryption key file missging")
log.Warn(
u"Please create a file with your IDrive encryption key,")
log.Warn(
u"Then create an environment variable IDKEYFILE with path/filename of said file"
)
log.Warn(u"-" * 72)
raise BackendException(
u"No IDKEYFILE env var set. Should contain file with encription key"
)
log.Debug(u"IDrive keypath: %s" % (filepath))
self.auth_switch += u" --pvt-key={0}".format(filepath)
# get the server address
el = self.request(
self.cmd + self.auth_switch +
u" --getServerAddress {0}".format(self.idriveid)).find(u'tree')
self.idriveserver = el.attrib[u"cmdUtilityServer"]
# get the device list - primarely used to get device-id string
el = self.request(self.cmd + self.auth_switch +
u" --list-device {0}@{1}::home".format(
self.idriveid, self.idriveserver))
# scan all returned devices for requested device (== bucket)
self.idrivedevid = None
for item in el.findall(u'item'):
if item.attrib[u'nick_name'] == self.bucket:
# prefix and suffix reverse-engineered from Common.pl!
self.idrivedevid = u"5c0b" + item.attrib[u"device_id"] + u"4b5z"
if self.idrivedevid is None:
el = self.request(
self.cmd + self.auth_switch +
u" --create-bucket --bucket-type=D --nick-name={0} --os=Linux --uid=987654321 {1}@{2}::home/"
.format(self.bucket, self.idriveid, self.idriveserver)).find(
u'item')
# prefix and suffix reverse-engineered from Common.pl!
self.idrivedevid = u"5c0b" + el.attrib[u"device_id"] + u"4b5z"
# We're fully connected!
self.connected = True
log.Debug(u"User fully connected")
def upload(self, filename, key, headers=None):
import boto # pylint: disable=import-error
chunk_size = config.s3_multipart_chunk_size
# Check minimum chunk size for S3
if chunk_size < config.s3_multipart_minimum_chunk_size:
log.Warn(u"Minimum chunk size is %d, but %d specified." %
(config.s3_multipart_minimum_chunk_size, chunk_size))
chunk_size = config.s3_multipart_minimum_chunk_size
# Decide in how many chunks to upload
bytes = os.path.getsize(filename) # pylint: disable=redefined-builtin
if bytes < chunk_size:
chunks = 1
else:
chunks = bytes // chunk_size
if (bytes % chunk_size):
chunks += 1
log.Debug(u"Uploading %d bytes in %d chunks" % (bytes, chunks))
mp = self.bucket.initiate_multipart_upload(
key.key, headers, encrypt_key=config.s3_use_sse)
# Initiate a queue to share progress data between the pool
# workers and a consumer thread, that will collect and report
queue = None
if config.progress:
manager = multiprocessing.Manager()
queue = manager.Queue()
consumer = ConsumerThread(queue, bytes)
consumer.start()
tasks = []
for n in range(chunks):
storage_uri = boto.storage_uri(self.boto_uri_str)
params = [
self.scheme, self.parsed_url, storage_uri, self.bucket_name,
mp.id, filename, n, chunk_size, config.num_retries, queue
]
tasks.append(
self._pool.apply_async(multipart_upload_worker, params))
log.Debug(u"Waiting for the pool to finish processing %s tasks" %
len(tasks))
while tasks:
try:
tasks[0].wait(timeout=config.s3_multipart_max_timeout)
if tasks[0].ready():
if tasks[0].successful():
del tasks[0]
else:
log.Debug(
u"Part upload not successful, aborting multipart upload."
)
self._setup_pool()
break
else:
raise multiprocessing.TimeoutError
except multiprocessing.TimeoutError:
log.Debug(u"%s tasks did not finish by the specified timeout,"
u"aborting multipart upload and resetting pool." %
len(tasks))
self._setup_pool()
break
log.Debug(u"Done waiting for the pool to finish processing")
# Terminate the consumer thread, if any
if config.progress:
consumer.finish = True
consumer.join()
if len(tasks) > 0 or len(mp.get_all_parts()) < chunks:
mp.cancel_upload()
raise BackendException(u"Multipart upload failed. Aborted.")
return mp.complete_upload()
class BotoBackend(duplicity.backend.Backend):
"""
Backend for Amazon's Simple Storage System, (aka Amazon S3), though
the use of the boto module, (http://code.google.com/p/boto/).
To make use of this backend you must set aws_access_key_id
and aws_secret_access_key in your ~/.boto or /etc/boto.cfg
with your Amazon Web Services key id and secret respectively.
Alternatively you can export the environment variables
AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.
"""
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
import boto
assert boto.Version >= BOTO_MIN_VERSION
from boto.s3.key import Key
# This folds the null prefix and all null parts, which means that:
# //MyBucket/ and //MyBucket are equivalent.
# //MyBucket//My///My/Prefix/ and //MyBucket/My/Prefix are equivalent.
self.url_parts = filter(lambda x: x != '', parsed_url.path.split('/'))
if self.url_parts:
self.bucket_name = self.url_parts.pop(0)
else:
# Duplicity hangs if boto gets a null bucket name.
# HC: Caught a socket error, trying to recover
raise BackendException('Boto requires a bucket name.')
self.scheme = parsed_url.scheme
self.key_class = Key
if self.url_parts:
self.key_prefix = '%s/' % '/'.join(self.url_parts)
else:
self.key_prefix = ''
self.straight_url = duplicity.backend.strip_auth_from_url(parsed_url)
self.parsed_url = parsed_url
self.resetConnection()
def resetConnection(self):
self.bucket = None
self.conn = None
try:
from boto.s3.connection import S3Connection
from boto.s3.key import Key
assert hasattr(S3Connection, 'lookup')
# Newer versions of boto default to using
# virtual hosting for buckets as a result of
# upstream deprecation of the old-style access
# method by Amazon S3. This change is not
# backwards compatible (in particular with
# respect to upper case characters in bucket
# names); so we default to forcing use of the
# old-style method unless the user has
# explicitly asked us to use new-style bucket
# access.
#
# Note that if the user wants to use new-style
# buckets, we use the subdomain calling form
# rather than given the option of both
# subdomain and vhost. The reason being that
# anything addressable as a vhost, is also
# addressable as a subdomain. Seeing as the
# latter is mostly a convenience method of
# allowing browse:able content semi-invisibly
# being hosted on S3, the former format makes
# a lot more sense for us to use - being
# explicit about what is happening (the fact
# that we are talking to S3 servers).
try:
from boto.s3.connection import OrdinaryCallingFormat
from boto.s3.connection import SubdomainCallingFormat
cfs_supported = True
calling_format = OrdinaryCallingFormat()
except ImportError:
cfs_supported = False
calling_format = None
if globals.s3_use_new_style:
if cfs_supported:
calling_format = SubdomainCallingFormat()
else:
log.FatalError("Use of new-style (subdomain) S3 bucket addressing was"
"requested, but does not seem to be supported by the "
"boto library. Either you need to upgrade your boto "
"library or duplicity has failed to correctly detect "
"the appropriate support.",
log.ErrorCode.boto_old_style)
else:
if cfs_supported:
calling_format = OrdinaryCallingFormat()
else:
calling_format = None
except ImportError:
log.FatalError("This backend (s3) requires boto library, version %s or later, "
"(http://code.google.com/p/boto/)." % BOTO_MIN_VERSION,
log.ErrorCode.boto_lib_too_old)
if self.scheme == 's3+http':
# Use the default Amazon S3 host.
self.conn = S3Connection(is_secure=(not globals.s3_unencrypted_connection))
else:
assert self.scheme == 's3'
self.conn = S3Connection(
host=self.parsed_url.hostname,
is_secure=(not globals.s3_unencrypted_connection))
if hasattr(self.conn, 'calling_format'):
if calling_format is None:
log.FatalError("It seems we previously failed to detect support for calling "
"formats in the boto library, yet the support is there. This is "
"almost certainly a duplicity bug.",
log.ErrorCode.boto_calling_format)
else:
self.conn.calling_format = calling_format
else:
# Duplicity hangs if boto gets a null bucket name.
# HC: Caught a socket error, trying to recover
raise BackendException('Boto requires a bucket name.')
self.bucket = self.conn.lookup(self.bucket_name)
def put(self, source_path, remote_filename=None):
from boto.s3.connection import Location
if globals.s3_european_buckets:
if not globals.s3_use_new_style:
log.FatalError("European bucket creation was requested, but not new-style "
"bucket addressing (--s3-use-new-style)",
log.ErrorCode.s3_bucket_not_style)
#Network glitch may prevent first few attempts of creating/looking up a bucket
for n in range(1, globals.num_retries+1):
if self.bucket:
break
if n > 1:
time.sleep(30)
try:
try:
self.bucket = self.conn.get_bucket(self.bucket_name, validate=True)
except Exception, e:
if "NoSuchBucket" in str(e):
if globals.s3_european_buckets:
self.bucket = self.conn.create_bucket(self.bucket_name,
location=Location.EU)
else:
self.bucket = self.conn.create_bucket(self.bucket_name)
else:
raise e
except Exception, e:
log.Warn("Failed to create bucket (attempt #%d) '%s' failed (reason: %s: %s)"
"" % (n, self.bucket_name,
e.__class__.__name__,
str(e)))
self.resetConnection()
if not remote_filename:
remote_filename = source_path.get_filename()
key = self.key_class(self.bucket)
key.key = self.key_prefix + remote_filename
for n in range(1, globals.num_retries+1):
if n > 1:
# sleep before retry (new connection to a **hopeful** new host, so no need to wait so long)
time.sleep(10)
if globals.s3_use_rrs:
storage_class = 'REDUCED_REDUNDANCY'
else:
storage_class = 'STANDARD'
log.Info("Uploading %s/%s to %s Storage" % (self.straight_url, remote_filename, storage_class))
try:
key.set_contents_from_filename(source_path.name, {'Content-Type': 'application/octet-stream',
'x-amz-storage-class': storage_class})
key.close()
self.resetConnection()
return
except Exception, e:
log.Warn("Upload '%s/%s' failed (attempt #%d, reason: %s: %s)"
"" % (self.straight_url,
remote_filename,
n,
e.__class__.__name__,
str(e)))
log.Debug("Backtrace of previous error: %s" % (exception_traceback(),))
self.resetConnection()
def parse_cmdline_options(arglist):
"""Parse argument list"""
global select_opts, select_files, full_backup
global list_current, collection_status, cleanup, remove_time, verify
def use_gio(*args):
try:
import duplicity.backends.giobackend
backend.force_backend(duplicity.backends.giobackend.GIOBackend)
except ImportError:
log.FatalError(_("Unable to load gio module"),
log.ErrorCode.gio_not_available)
def set_log_fd(fd):
if fd < 1:
raise optparse.OptionValueError(
"log-fd must be greater than zero.")
log.add_fd(fd)
def set_time_sep(sep, opt):
if sep == '-':
raise optparse.OptionValueError(
"Dash ('-') not valid for time-separator.")
globals.time_separator = sep
old_fn_deprecation(opt)
def add_selection(o, s, v, p):
select_opts.append((s, v))
def add_filelist(o, s, v, p):
filename = v
select_opts.append((s, filename))
try:
select_files.append(open(filename, "r"))
except IOError:
log.FatalError(
_("Error opening file %s") % filename,
log.ErrorCode.cant_open_filelist)
def print_ver(o, s, v, p):
print "duplicity %s" % (globals.version)
sys.exit(0)
def add_rename(o, s, v, p):
globals.rename[os.path.normcase(os.path.normpath(v[0]))] = v[1]
parser = OPHelpFix(option_class=DupOption, usage=usage())
# If this is true, only warn and don't raise fatal error when backup
# source directory doesn't match previous backup source directory.
parser.add_option("--allow-source-mismatch", action="store_true")
# Set to the path of the archive directory (the directory which
# contains the signatures and manifests of the relevent backup
# collection), and for checkpoint state between volumes.
# TRANSL: Used in usage help to represent a Unix-style path name. Example:
# --archive-dir <path>
parser.add_option("--archive-dir", type="file", metavar=_("path"))
# Asynchronous put/get concurrency limit
# (default of 0 disables asynchronicity).
parser.add_option("--asynchronous-upload",
action="store_const",
const=1,
dest="async_concurrency")
# config dir for future use
parser.add_option("--config-dir",
type="file",
metavar=_("path"),
help=optparse.SUPPRESS_HELP)
# for testing -- set current time
parser.add_option("--current-time",
type="int",
dest="current_time",
help=optparse.SUPPRESS_HELP)
# Don't actually do anything, but still report what would be done
parser.add_option("--dry-run", action="store_true")
# TRANSL: Used in usage help to represent an ID for a GnuPG key. Example:
# --encrypt-key <gpg_key_id>
parser.add_option("--encrypt-key",
type="string",
metavar=_("gpg-key-id"),
dest="",
action="callback",
callback=lambda o, s, v, p: globals.gpg_profile.
recipients.append(v)) #@UndefinedVariable
# secret keyring in which the private encrypt key can be found
parser.add_option("--encrypt-secret-keyring",
type="string",
metavar=_("path"))
parser.add_option(
"--encrypt-sign-key",
type="string",
metavar=_("gpg-key-id"),
dest="",
action="callback",
callback=lambda o, s, v, p:
(globals.gpg_profile.recipients.append(v), set_sign_key(v)))
# TRANSL: Used in usage help to represent a "glob" style pattern for
# matching one or more files, as described in the documentation.
# Example:
# --exclude <shell_pattern>
parser.add_option("--exclude",
action="callback",
metavar=_("shell_pattern"),
dest="",
type="string",
callback=add_selection)
parser.add_option("--exclude-device-files",
action="callback",
dest="",
callback=add_selection)
parser.add_option("--exclude-filelist",
type="file",
metavar=_("filename"),
dest="",
action="callback",
callback=add_filelist)
parser.add_option("--exclude-filelist-stdin",
action="callback",
dest="",
callback=lambda o, s, v, p:
(select_opts.append(
("--exclude-filelist", "standard input")),
select_files.append(sys.stdin)))
parser.add_option("--exclude-globbing-filelist",
type="file",
metavar=_("filename"),
dest="",
action="callback",
callback=add_filelist)
# TRANSL: Used in usage help to represent the name of a file. Example:
# --log-file <filename>
parser.add_option("--exclude-if-present",
metavar=_("filename"),
dest="",
type="file",
action="callback",
callback=add_selection)
parser.add_option("--exclude-other-filesystems",
action="callback",
dest="",
callback=add_selection)
# TRANSL: Used in usage help to represent a regular expression (regexp).
parser.add_option("--exclude-regexp",
metavar=_("regular_expression"),
dest="",
type="string",
action="callback",
callback=add_selection)
# Whether we should be particularly aggressive when cleaning up
parser.add_option("--extra-clean", action="store_true")
# used in testing only - raises exception after volume
parser.add_option("--fail-on-volume",
type="int",
help=optparse.SUPPRESS_HELP)
# used to provide a prefix on top of the defaul tar file name
parser.add_option("--file-prefix",
type="string",
dest="file_prefix",
action="store")
# used in testing only - skips upload for a given volume
parser.add_option("--skip-volume", type="int", help=optparse.SUPPRESS_HELP)
# If set, restore only the subdirectory or file specified, not the
# whole root.
# TRANSL: Used in usage help to represent a Unix-style path name. Example:
# --archive-dir <path>
parser.add_option("--file-to-restore",
"-r",
action="callback",
type="file",
metavar=_("path"),
dest="restore_dir",
callback=lambda o, s, v, p: setattr(
p.values, "restore_dir", v.rstrip('/')))
# Used to confirm certain destructive operations like deleting old files.
parser.add_option("--force", action="store_true")
# FTP data connection type
parser.add_option("--ftp-passive",
action="store_const",
const="passive",
dest="ftp_connection")
parser.add_option("--ftp-regular",
action="store_const",
const="regular",
dest="ftp_connection")
# If set, forces a full backup if the last full backup is older than
# the time specified
parser.add_option("--full-if-older-than",
type="time",
dest="full_force_time",
metavar=_("time"))
parser.add_option("--gio", action="callback", callback=use_gio)
parser.add_option("--gpg-options", action="extend", metavar=_("options"))
# ignore (some) errors during operations; supposed to make it more
# likely that you are able to restore data under problematic
# circumstances. the default should absolutely always be False unless
# you know what you are doing.
parser.add_option(
"--ignore-errors",
action="callback",
dest="ignore_errors",
callback=lambda o, s, v, p: (log.Warn(
_("Running in 'ignore errors' mode due to %s; please "
"re-consider if this was not intended") % s),
setattr(p.values, "ignore errors", True)))
# Whether to use the full email address as the user name when
# logging into an imap server. If false just the user name
# part of the email address is used.
parser.add_option("--imap-full-address",
action="store_true",
help=optparse.SUPPRESS_HELP)
# Name of the imap folder where we want to store backups.
# Can be changed with a command line argument.
# TRANSL: Used in usage help to represent an imap mailbox
parser.add_option("--imap-mailbox", metavar=_("imap_mailbox"))
parser.add_option("--include",
action="callback",
metavar=_("shell_pattern"),
dest="",
type="string",
callback=add_selection)
parser.add_option("--include-filelist",
type="file",
metavar=_("filename"),
dest="",
action="callback",
callback=add_filelist)
parser.add_option("--include-filelist-stdin",
action="callback",
dest="",
callback=lambda o, s, v, p:
(select_opts.append(
("--include-filelist", "standard input")),
select_files.append(sys.stdin)))
parser.add_option("--include-globbing-filelist",
type="file",
metavar=_("filename"),
dest="",
action="callback",
callback=add_filelist)
parser.add_option("--include-regexp",
metavar=_("regular_expression"),
dest="",
type="string",
action="callback",
callback=add_selection)
parser.add_option("--log-fd",
type="int",
metavar=_("file_descriptor"),
dest="",
action="callback",
callback=lambda o, s, v, p: set_log_fd(v))
# TRANSL: Used in usage help to represent the name of a file. Example:
# --log-file <filename>
parser.add_option("--log-file",
type="file",
metavar=_("filename"),
dest="",
action="callback",
callback=lambda o, s, v, p: log.add_file(v))
# TRANSL: Used in usage help (noun)
parser.add_option("--name", dest="backup_name", metavar=_("backup name"))
# If set to false, then do not encrypt files on remote system
parser.add_option("--no-encryption",
action="store_false",
dest="encryption")
# If set to false, then do not compress files on remote system
parser.add_option("--no-compression",
action="store_false",
dest="compression")
# If set, print the statistics after every backup session
parser.add_option("--no-print-statistics",
action="store_false",
dest="print_statistics")
# If true, filelists and directory statistics will be split on
# nulls instead of newlines.
parser.add_option("--null-separator", action="store_true")
# number of retries on network operations
# TRANSL: Used in usage help to represent a desired number of
# something. Example:
# --num-retries <number>
parser.add_option("--num-retries", type="int", metavar=_("number"))
# File owner uid keeps number from tar file. Like same option in GNU tar.
parser.add_option("--numeric-owner", action="store_true")
# Whether the old filename format is in effect.
parser.add_option("--old-filenames",
action="callback",
dest="old_filenames",
callback=lambda o, s, v, p:
(setattr(p.values, o.dest, True), old_fn_deprecation(s)))
# option to trigger Pydev debugger
parser.add_option("--pydevd", action="store_true")
# option to rename files during restore
parser.add_option("--rename",
type="file",
action="callback",
nargs=2,
callback=add_rename)
# Restores will try to bring back the state as of the following time.
# If it is None, default to current time.
# TRANSL: Used in usage help to represent a time spec for a previous
# point in time, as described in the documentation. Example:
# duplicity remove-older-than time [options] target_url
parser.add_option("--restore-time",
"--time",
"-t",
type="time",
metavar=_("time"))
# user added rsync options
parser.add_option("--rsync-options", action="extend", metavar=_("options"))
# Whether to create European buckets (sorry, hard-coded to only
# support european for now).
parser.add_option("--s3-european-buckets", action="store_true")
# Whether to use S3 Reduced Redudancy Storage
parser.add_option("--s3-use-rrs", action="store_true")
# Whether to use "new-style" subdomain addressing for S3 buckets. Such
# use is not backwards-compatible with upper-case buckets, or buckets
# that are otherwise not expressable in a valid hostname.
parser.add_option("--s3-use-new-style", action="store_true")
# Whether to use plain HTTP (without SSL) to send data to S3
# See <https://bugs.launchpad.net/duplicity/+bug/433970>.
parser.add_option("--s3-unencrypted-connection", action="store_true")
# Chunk size used for S3 multipart uploads.The number of parallel uploads to
# S3 be given by chunk size / volume size. Use this to maximize the use of
# your bandwidth. Defaults to 25MB
parser.add_option(
"--s3-multipart-chunk-size",
type="int",
action="callback",
metavar=_("number"),
callback=lambda o, s, v, p: setattr(
p.values, "s3_multipart_chunk_size", v * 1024 * 1024))
# Option to allow the s3/boto backend use the multiprocessing version.
# By default it is off since it does not work for Python 2.4 or 2.5.
if sys.version_info[:2] >= (2, 6):
parser.add_option("--s3-use-multiprocessing", action="store_true")
# scp command to use
# TRANSL: noun
parser.add_option("--scp-command",
nargs=1,
type="string",
action="callback",
callback=scp_deprecation)
# sftp command to use
# TRANSL: noun
parser.add_option("--sftp-command",
nargs=1,
type="string",
action="callback",
callback=scp_deprecation)
# If set, use short (< 30 char) filenames for all the remote files.
parser.add_option("--short-filenames",
action="callback",
dest="short_filenames",
callback=lambda o, s, v, p:
(setattr(p.values, o.dest, True), old_fn_deprecation(s)))
# TRANSL: Used in usage help to represent an ID for a GnuPG key. Example:
# --encrypt-key <gpg_key_id>
parser.add_option("--sign-key",
type="string",
metavar=_("gpg-key-id"),
dest="",
action="callback",
callback=lambda o, s, v, p: set_sign_key(v))
# default to batch mode using public-key encryption
parser.add_option("--ssh-askpass", action="store_true")
# user added ssh options
parser.add_option("--ssh-options", action="extend", metavar=_("options"))
# Working directory for the tempfile module. Defaults to /tmp on most systems.
parser.add_option("--tempdir",
dest="temproot",
type="file",
metavar=_("path"))
# network timeout value
# TRANSL: Used in usage help. Example:
# --timeout <seconds>
parser.add_option("--timeout", type="int", metavar=_("seconds"))
# Character used like the ":" in time strings like
# 2002-08-06T04:22:00-07:00. The colon isn't good for filenames on
# windows machines.
# TRANSL: abbreviation for "character" (noun)
parser.add_option("--time-separator",
type="string",
metavar=_("char"),
action="callback",
callback=lambda o, s, v, p: set_time_sep(v, s))
# Whether to specify --use-agent in GnuPG options
parser.add_option("--use-agent", action="store_true")
parser.add_option("--use-scp", action="store_true")
parser.add_option("--verbosity",
"-v",
type="verbosity",
metavar="[0-9]",
dest="",
action="callback",
callback=lambda o, s, v, p: log.setverbosity(v))
parser.add_option("-V", "--version", action="callback", callback=print_ver)
# volume size
# TRANSL: Used in usage help to represent a desired number of
# something. Example:
# --num-retries <number>
parser.add_option("--volsize",
type="int",
action="callback",
metavar=_("number"),
callback=lambda o, s, v, p: setattr(
p.values, "volsize", v * 1024 * 1024))
# parse the options
(options, args) = parser.parse_args()
# Copy all arguments and their values to the globals module. Don't copy
# attributes that are 'hidden' (start with an underscore) or whose name is
# the empty string (used for arguments that don't directly store a value
# by using dest="")
for f in filter(lambda x: x and not x.startswith("_"), dir(options)):
v = getattr(options, f)
# Only set if v is not None because None is the default for all the
# variables. If user didn't set it, we'll use defaults in globals.py
if v is not None:
setattr(globals, f, v)
socket.setdefaulttimeout(globals.timeout)
# expect no cmd and two positional args
cmd = ""
num_expect = 2
# process first arg as command
if args:
cmd = args.pop(0)
possible = [c for c in commands if c.startswith(cmd)]
# no unique match, that's an error
if len(possible) > 1:
command_line_error("command '%s' not unique, could be %s" %
(cmd, possible))
# only one match, that's a keeper
elif len(possible) == 1:
cmd = possible[0]
# no matches, assume no cmd
elif not possible:
args.insert(0, cmd)
if cmd == "cleanup":
cleanup = True
num_expect = 1
elif cmd == "collection-status":
collection_status = True
num_expect = 1
elif cmd == "full":
full_backup = True
num_expect = 2
elif cmd == "incremental":
globals.incremental = True
num_expect = 2
elif cmd == "list-current-files":
list_current = True
num_expect = 1
elif cmd == "remove-older-than":
try:
arg = args.pop(0)
except Exception:
command_line_error("Missing time string for remove-older-than")
globals.remove_time = dup_time.genstrtotime(arg)
num_expect = 1
elif cmd == "remove-all-but-n-full" or cmd == "remove-all-inc-of-but-n-full":
if cmd == "remove-all-but-n-full":
globals.remove_all_but_n_full_mode = True
if cmd == "remove-all-inc-of-but-n-full":
globals.remove_all_inc_of_but_n_full_mode = True
try:
arg = args.pop(0)
except Exception:
command_line_error("Missing count for " + cmd)
globals.keep_chains = int(arg)
if not globals.keep_chains > 0:
command_line_error(cmd + " count must be > 0")
num_expect = 1
elif cmd == "verify":
verify = True
if len(args) != num_expect:
command_line_error("Expected %d args, got %d" %
(num_expect, len(args)))
# expand pathname args, but not URL
for loc in range(len(args)):
if not '://' in args[loc]:
args[loc] = expand_fn(args[loc])
# Note that ProcessCommandLine depends on us verifying the arg
# count here; do not remove without fixing it. We must make the
# checks here in order to make enough sense of args to identify
# the backend URL/lpath for args_to_path_backend().
if len(args) < 1:
command_line_error("Too few arguments")
elif len(args) == 1:
backend_url = args[0]
elif len(args) == 2:
lpath, backend_url = args_to_path_backend(args[0],
args[1]) #@UnusedVariable
else:
command_line_error("Too many arguments")
if globals.backup_name is None:
globals.backup_name = generate_default_backup_name(backend_url)
# set and expand archive dir
set_archive_dir(
expand_archive_dir(globals.archive_dir, globals.backup_name))
log.Debug(_("Using archive dir: %s") % (globals.archive_dir.name, ))
log.Debug(_("Using backup name: %s") % (globals.backup_name, ))
return args
def get_tarinfo(self):
u"""Generate a tarfile.TarInfo object based on self
Doesn't set size based on stat, because we may want to replace
data wiht other stream. Size should be set separately by
calling function.
"""
ti = tarfile.TarInfo()
if self.index:
ti.name = util.fsdecode(b"/".join(self.index))
else:
ti.name = u"."
if self.isdir():
ti.name += u"/" # tar dir naming convention
ti.size = 0
if self.type:
# Lots of this is specific to tarfile.py, hope it doesn't
# change much...
if self.isreg():
ti.type = tarfile.REGTYPE
ti.size = self.stat.st_size
elif self.isdir():
ti.type = tarfile.DIRTYPE
elif self.isfifo():
ti.type = tarfile.FIFOTYPE
elif self.issym():
ti.type = tarfile.SYMTYPE
ti.linkname = self.symtext
if isinstance(ti.linkname, bytes):
ti.linkname = util.fsdecode(ti.linkname)
elif self.isdev():
if self.type == u"chr":
ti.type = tarfile.CHRTYPE
else:
ti.type = tarfile.BLKTYPE
ti.devmajor, ti.devminor = self.devnums
else:
raise PathException(u"Unrecognized type " + str(self.type))
ti.mode = self.mode
ti.uid, ti.gid = self.stat.st_uid, self.stat.st_gid
if self.stat.st_mtime < 0:
log.Warn(
_(u"Warning: %s has negative mtime, treating as 0.") %
(util.fsdecode(self.get_relative_path())))
ti.mtime = 0
else:
ti.mtime = int(self.stat.st_mtime)
try:
ti.uname = cached_ops.getpwuid(ti.uid)[0]
except KeyError:
ti.uname = u''
try:
ti.gname = cached_ops.getgrgid(ti.gid)[0]
except KeyError:
ti.gname = u''
if ti.type in (tarfile.CHRTYPE, tarfile.BLKTYPE):
if hasattr(os, u"major") and hasattr(os, u"minor"):
ti.devmajor, ti.devminor = self.devnums
else:
# Currently we depend on an uninitiliazed tarinfo file to
# already have appropriate headers. Still, might as well
# make sure mode and size set.
ti.mode, ti.size = 0, 0
return ti
def run_scp_command(self, commandline):
u""" Run an scp command, responding to password prompts """
log.Info(u"Running '%s'" % commandline)
child = pexpect.spawn(commandline, timeout=None)
if globals.ssh_askpass:
state = u"authorizing"
else:
state = u"copying"
while 1:
if state == u"authorizing":
match = child.expect([
pexpect.EOF, u"(?i)timeout, server not responding",
u"(?i)pass(word|phrase .*):", u"(?i)permission denied",
u"authenticity"
])
log.Debug(u"State = %s, Before = '%s'" %
(state, child.before.strip()))
if match == 0:
log.Warn(u"Failed to authenticate")
break
elif match == 1:
log.Warn(u"Timeout waiting to authenticate")
break
elif match == 2:
child.sendline(self.password)
state = u"copying"
elif match == 3:
log.Warn(u"Invalid SSH password")
break
elif match == 4:
log.Warn(
u"Remote host authentication failed (missing known_hosts entry?)"
)
break
elif state == u"copying":
match = child.expect([
pexpect.EOF, u"(?i)timeout, server not responding",
u"stalled", u"authenticity", u"ETA"
])
log.Debug(u"State = %s, Before = '%s'" %
(state, child.before.strip()))
if match == 0:
break
elif match == 1:
log.Warn(u"Timeout waiting for response")
break
elif match == 2:
state = u"stalled"
elif match == 3:
log.Warn(
u"Remote host authentication failed (missing known_hosts entry?)"
)
break
elif state == u"stalled":
match = child.expect([
pexpect.EOF, u"(?i)timeout, server not responding", u"ETA"
])
log.Debug(u"State = %s, Before = '%s'" %
(state, child.before.strip()))
if match == 0:
break
elif match == 1:
log.Warn(u"Stalled for too long, aborted copy")
break
elif match == 2:
state = u"copying"
child.close(force=True)
if child.exitstatus != 0:
raise BackendException(u"Error running '%s'" % commandline)
def run_sftp_command(self, commandline, commands):
""" Run an sftp command, responding to password prompts, passing commands from list """
maxread = 2000 # expected read buffer size
responses = [
pexpect.EOF, "(?i)timeout, server not responding", "sftp>",
"(?i)pass(word|phrase .*):", "(?i)permission denied",
"authenticity", "(?i)no such file or directory",
"Couldn't delete file: No such file or directory",
"Couldn't delete file", "open(.*): Failure"
]
max_response_len = max([len(p) for p in responses[1:]])
for n in range(1, globals.num_retries + 1):
if n > 1:
# sleep before retry
time.sleep(30)
log.Info("Running '%s' (attempt #%d)" % (commandline, n))
child = pexpect.spawn(commandline, timeout=None, maxread=maxread)
cmdloc = 0
while 1:
match = child.expect(responses,
searchwindowsize=maxread +
max_response_len)
log.Debug("State = sftp, Before = '%s'" %
(child.before.strip()))
if match == 0:
break
elif match == 1:
log.Info("Timeout waiting for response")
break
if match == 2:
if cmdloc < len(commands):
command = commands[cmdloc]
log.Info("sftp command: '%s'" % (command, ))
child.sendline(command)
cmdloc += 1
else:
command = 'quit'
child.sendline(command)
res = child.before
elif match == 3:
child.sendline(self.password)
elif match == 4:
log.Warn("Invalid SSH password")
break
elif match == 5:
log.Warn(
"Host key authenticity could not be verified (missing known_hosts entry?)"
)
break
elif match == 6:
if not child.before.strip().startswith("rm"):
log.Warn(
"Remote file or directory does not exist in command='%s'"
% (commandline, ))
break
elif match == 7:
if not child.before.strip().startswith("Removing"):
log.Warn("Could not delete file in command='%s'" %
(commandline, ))
break
elif match == 8:
log.Warn("Could not delete file in command='%s'" %
(commandline, ))
break
elif match == 9:
log.Warn("Could not open file in command='%s'" %
(commandline, ))
break
child.close(force=True)
if child.exitstatus == 0:
return res
log.Warn("Running '%s' failed (attempt #%d)" % (commandline, n))
log.Warn("Giving up trying to execute '%s' after %d attempts" %
(commandline, globals.num_retries))
raise BackendException("Error running '%s'" % commandline)
def Iterate(self, path):
"""Return iterator yielding paths in path
This function looks a bit more complicated than it needs to be
because it avoids extra recursion (and no extra function calls
for non-directory files) while still doing the "directory
scanning" bit.
"""
def error_handler(exc, path, filename):
fullpath = os.path.join(path.name, filename)
try:
mode = os.stat(fullpath)[stat.ST_MODE]
if stat.S_ISSOCK(mode):
log.Info(
_("Skipping socket %s") % fullpath,
log.InfoCode.skipping_socket, util.escape(fullpath))
else:
log.Warn(
_("Error initializing file %s") % fullpath,
log.WarningCode.cannot_iterate, util.escape(fullpath))
except OSError:
log.Warn(
_("Error accessing possibly locked file %s") % fullpath,
log.WarningCode.cannot_stat, util.escape(fullpath))
return None
def diryield(path):
"""Generate relevant files in directory path
Returns (path, num) where num == 0 means path should be
generated normally, num == 1 means the path is a directory
and should be included iff something inside is included.
"""
# todo: get around circular dependency issue by importing here
from duplicity import robust #@Reimport
for filename in robust.listpath(path):
new_path = robust.check_common_error(error_handler,
Path.append,
(path, filename))
# make sure file is read accessible
if (new_path and new_path.type in ["reg", "dir"]
and not os.access(new_path.name, os.R_OK)):
log.Warn(
_("Error accessing possibly locked file %s") %
new_path.name, log.WarningCode.cannot_read,
util.escape(new_path.name))
if diffdir.stats:
diffdir.stats.Errors += 1
new_path = None
elif new_path:
s = self.Select(new_path)
if s == 1:
yield (new_path, 0)
elif s == 2 and new_path.isdir():
yield (new_path, 1)
if not path.type:
# base doesn't exist
log.Warn(
_("Warning: base %s doesn't exist, continuing") % path.name)
return
log.Debug(_("Selecting %s") % path.name)
yield path
if not path.isdir():
return
diryield_stack = [diryield(path)]
delayed_path_stack = []
while diryield_stack:
try:
subpath, val = diryield_stack[-1].next()
except StopIteration:
diryield_stack.pop()
if delayed_path_stack:
delayed_path_stack.pop()
continue
if val == 0:
if delayed_path_stack:
for delayed_path in delayed_path_stack:
log.Log(_("Selecting %s") % delayed_path.name, 6)
yield delayed_path
del delayed_path_stack[:]
log.Debug(_("Selecting %s") % subpath.name)
yield subpath
if subpath.isdir():
diryield_stack.append(diryield(subpath))
elif val == 1:
delayed_path_stack.append(subpath)
diryield_stack.append(diryield(subpath))
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
# we expect an error return, so go low-level and ignore it
try:
p = os.popen(u"ncftpls -v")
fout = p.read()
ret = p.close()
except Exception:
pass
# the expected error is 8 in the high-byte and some output
if ret != 0x0800 or not fout:
log.FatalError(u"NcFTP not found: Please install NcFTP version 3.1.9 or later",
log.ErrorCode.ftp_ncftp_missing)
# version is the second word of the first line
version = fout.split(u'\n')[0].split()[1]
if version < u"3.1.9":
log.FatalError(u"NcFTP too old: Duplicity requires NcFTP version 3.1.9,"
u"3.2.1 or later. Version 3.2.0 will not work properly.",
log.ErrorCode.ftp_ncftp_too_old)
elif version == u"3.2.0":
log.Warn(u"NcFTP (ncftpput) version 3.2.0 may fail with duplicity.\n"
u"see: http://www.ncftpd.com/ncftp/doc/changelog.html\n"
u"If you have trouble, please upgrade to 3.2.1 or later",
log.WarningCode.ftp_ncftp_v320)
log.Notice(u"NcFTP version is %s" % version)
self.parsed_url = parsed_url
self.url_string = duplicity.backend.strip_auth_from_url(self.parsed_url)
# strip ncftp+ prefix
self.url_string = duplicity.backend.strip_prefix(self.url_string, u'ncftp')
# This squelches the "file not found" result from ncftpls when
# the ftp backend looks for a collection that does not exist.
# version 3.2.2 has error code 5, 1280 is some legacy value
self.popen_breaks[u'ncftpls'] = [5, 1280]
# Use an explicit directory name.
if self.url_string[-1] != u'/':
self.url_string += u'/'
self.password = self.get_password()
if config.ftp_connection == u'regular':
self.conn_opt = u'-E'
else:
self.conn_opt = u'-F'
self.tempfd, self.tempname = tempdir.default().mkstemp()
self.tempfile = os.fdopen(self.tempfd, u"w")
self.tempfile.write(u"host %s\n" % self.parsed_url.hostname)
self.tempfile.write(u"user %s\n" % self.parsed_url.username)
self.tempfile.write(u"pass %s\n" % self.password)
self.tempfile.close()
self.flags = u"-f %s %s -t %s -o useCLNT=0,useHELP_SITE=0 " % \
(self.tempname, self.conn_opt, config.timeout)
if parsed_url.port is not None and parsed_url.port != 21:
self.flags += u" -P '%s'" % (parsed_url.port)
def inner_retry(self, *args):
global _last_exception
errors_fatal, errors_default = config.are_errors_fatal.get(
operation, (True, None))
for n in range(1, config.num_retries + 1):
try:
return fn(self, *args)
except FatalBackendException as e:
_last_exception = e
if not errors_fatal:
# backend wants to report and ignore errors
return errors_default
else:
# die on fatal errors
raise e
except Exception as e:
_last_exception = e
if not errors_fatal:
# backend wants to report and ignore errors
return errors_default
else:
# retry on anything else
log.Debug(
_(u"Backtrace of previous error: %s") %
exception_traceback())
at_end = n == config.num_retries
code = _get_code_from_exception(
self.backend, operation, e)
if code == log.ErrorCode.backend_not_found:
# If we tried to do something, but the file just isn't there,
# no need to retry.
at_end = True
if at_end and fatal:
def make_filename(f):
if isinstance(f, path.ROPath):
return util.escape(f.uc_name)
else:
return util.escape(f)
extra = u' '.join([operation] + [
make_filename(x)
for x in args if (x and isinstance(x, str))
])
log.FatalError(
_(u"Giving up after %s attempts. %s: %s") %
(n, e.__class__.__name__, util.uexc(e)),
code=code,
extra=extra)
else:
log.Warn(
_(u"Attempt %s failed. %s: %s") %
(n, e.__class__.__name__, util.uexc(e)))
if not at_end:
if isinstance(e, TemporaryLoadException):
time.sleep(3 * config.backend_retry_delay
) # wait longer before trying again
else:
time.sleep(config.backend_retry_delay
) # wait a bit before trying again
if hasattr(self.backend, u'_retry_cleanup'):
self.backend._retry_cleanup()
class BotoBackend(duplicity.backend.Backend):
"""
Backend for Amazon's Simple Storage System, (aka Amazon S3), though
the use of the boto module, (http://code.google.com/p/boto/).
To make use of this backend you must set aws_access_key_id
and aws_secret_access_key in your ~/.boto or /etc/boto.cfg
with your Amazon Web Services key id and secret respectively.
Alternatively you can export the environment variables
AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.
"""
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
from boto.s3.key import Key
from boto.s3.multipart import MultiPartUpload
# This folds the null prefix and all null parts, which means that:
# //MyBucket/ and //MyBucket are equivalent.
# //MyBucket//My///My/Prefix/ and //MyBucket/My/Prefix are equivalent.
self.url_parts = filter(lambda x: x != '', parsed_url.path.split('/'))
if self.url_parts:
self.bucket_name = self.url_parts.pop(0)
else:
# Duplicity hangs if boto gets a null bucket name.
# HC: Caught a socket error, trying to recover
raise BackendException('Boto requires a bucket name.')
self.scheme = parsed_url.scheme
self.key_class = Key
if self.url_parts:
self.key_prefix = '%s/' % '/'.join(self.url_parts)
else:
self.key_prefix = ''
self.straight_url = duplicity.backend.strip_auth_from_url(parsed_url)
self.parsed_url = parsed_url
self.resetConnection()
def resetConnection(self):
self.bucket = None
self.conn = get_connection(self.scheme, self.parsed_url)
self.bucket = self.conn.lookup(self.bucket_name)
def put(self, source_path, remote_filename=None):
from boto.s3.connection import Location
if globals.s3_european_buckets:
if not globals.s3_use_new_style:
log.FatalError(
"European bucket creation was requested, but not new-style "
"bucket addressing (--s3-use-new-style)",
log.ErrorCode.s3_bucket_not_style)
#Network glitch may prevent first few attempts of creating/looking up a bucket
for n in range(1, globals.num_retries + 1):
if self.bucket:
break
if n > 1:
time.sleep(30)
try:
try:
self.bucket = self.conn.get_bucket(self.bucket_name,
validate=True)
except Exception, e:
if "NoSuchBucket" in str(e):
if globals.s3_european_buckets:
self.bucket = self.conn.create_bucket(
self.bucket_name, location=Location.EU)
else:
self.bucket = self.conn.create_bucket(
self.bucket_name)
else:
raise e
except Exception, e:
log.Warn(
"Failed to create bucket (attempt #%d) '%s' failed (reason: %s: %s)"
"" % (n, self.bucket_name, e.__class__.__name__, str(e)))
self.resetConnection()
if not remote_filename:
remote_filename = source_path.get_filename()
key = self.key_prefix + remote_filename
for n in range(1, globals.num_retries + 1):
if n > 1:
# sleep before retry (new connection to a **hopeful** new host, so no need to wait so long)
time.sleep(10)
if globals.s3_use_rrs:
storage_class = 'REDUCED_REDUNDANCY'
else:
storage_class = 'STANDARD'
log.Info("Uploading %s/%s to %s Storage" %
(self.straight_url, remote_filename, storage_class))
try:
headers = {
'Content-Type': 'application/octet-stream',
'x-amz-storage-class': storage_class
}
self.upload(source_path.name, key, headers)
self.resetConnection()
return
except Exception, e:
log.Warn("Upload '%s/%s' failed (attempt #%d, reason: %s: %s)"
"" % (self.straight_url, remote_filename, n,
e.__class__.__name__, str(e)))
log.Debug("Backtrace of previous error: %s" %
(exception_traceback(), ))
self.resetConnection()
def _delete(self, filename):
file_id = self.id_by_name(filename)
if file_id != '':
self.drive.auth.service.files().delete(fileId=file_id).execute()
else:
log.Warn("File '%s' does not exist while trying to delete it" % (filename,))
def error_handler(exc):
log.Warn(_(u"Error listing directory %s") % path.uc_name)
return []
raise BackendException("scp remote error: %s" % chan.recv(-1))
chan.sendall(f.read()+'\0')
f.close()
response=chan.recv(1)
if (response!="\0"):
raise BackendException("scp remote error: %s" % chan.recv(-1))
chan.close()
return
else:
try:
self.sftp.put(source_path.name,remote_filename)
return
except Exception, e:
raise BackendException("sftp put of %s (as %s) failed: %s" % (source_path.name,remote_filename,e))
except Exception, e:
log.Warn("%s (Try %d of %d) Will retry in %d seconds." % (e,n,globals.num_retries,self.retry_delay))
raise BackendException("Giving up trying to upload '%s' after %d attempts" % (remote_filename,n))
def get(self, remote_filename, local_path):
"""retrieves a single file from the remote side.
In scp mode unavoidable quoting issues will make this fail if the remote directory or file names
contain single quotes."""
for n in range(1, globals.num_retries+1):
if n > 1:
# sleep before retry
time.sleep(self.retry_delay)
try:
if (globals.use_scp):
try:
def run_scp_command(self, commandline):
""" Run an scp command, responding to password prompts """
for n in range(1, globals.num_retries + 1):
if n > 1:
# sleep before retry
time.sleep(30)
log.Info("Running '%s' (attempt #%d)" % (commandline, n))
child = pexpect.spawn(commandline, timeout=None)
if globals.ssh_askpass:
state = "authorizing"
else:
state = "copying"
while 1:
if state == "authorizing":
match = child.expect([
pexpect.EOF, "(?i)timeout, server not responding",
"(?i)pass(word|phrase .*):", "(?i)permission denied",
"authenticity"
])
log.Debug("State = %s, Before = '%s'" %
(state, child.before.strip()))
if match == 0:
log.Warn("Failed to authenticate")
break
elif match == 1:
log.Warn("Timeout waiting to authenticate")
break
elif match == 2:
child.sendline(self.password)
state = "copying"
elif match == 3:
log.Warn("Invalid SSH password")
break
elif match == 4:
log.Warn(
"Remote host authentication failed (missing known_hosts entry?)"
)
break
elif state == "copying":
match = child.expect([
pexpect.EOF, "(?i)timeout, server not responding",
"stalled", "authenticity", "ETA"
])
log.Debug("State = %s, Before = '%s'" %
(state, child.before.strip()))
if match == 0:
break
elif match == 1:
log.Warn("Timeout waiting for response")
break
elif match == 2:
state = "stalled"
elif match == 3:
log.Warn(
"Remote host authentication failed (missing known_hosts entry?)"
)
break
elif state == "stalled":
match = child.expect([
pexpect.EOF, "(?i)timeout, server not responding",
"ETA"
])
log.Debug("State = %s, Before = '%s'" %
(state, child.before.strip()))
if match == 0:
break
elif match == 1:
log.Warn("Stalled for too long, aborted copy")
break
elif match == 2:
state = "copying"
child.close(force=True)
if child.exitstatus == 0:
return
log.Warn("Running '%s' failed (attempt #%d)" % (commandline, n))
log.Warn("Giving up trying to execute '%s' after %d attempts" %
(commandline, globals.num_retries))
raise BackendException("Error running '%s'" % commandline)
something_excluded, tuple_list = None, []
separator = globals.null_separator and "\0" or "\n"
for line in filelist_fp.read().split(separator):
if not line:
continue # skip blanks
try:
tuple = self.filelist_parse_line(line, include)
except FilePrefixError, exc:
incr_warnings(exc)
continue
tuple_list.append(tuple)
if not tuple[1]:
something_excluded = 1
if filelist_fp not in (sys.stdin, ) and filelist_fp.close():
log.Warn(_("Error closing filelist %s") % filelist_name)
return (tuple_list, something_excluded)
def filelist_parse_line(self, line, include):
"""Parse a single line of a filelist, returning a pair
pair will be of form (index, include), where index is another
tuple, and include is 1 if the line specifies that we are
including a file. The default is given as an argument.
prefix is the string that the index is relative to.
"""
line = line.strip()
if line[:2] == "+ ":
# Check for "+ "/"- " syntax
include = 1
def error_handler(exc): # pylint: disable=unused-argument
log.Warn(_(u"Error listing directory %s") % path.uc_name)
return []
log.Info("Uploading '%s/%s' " % (self.container, remote_filename))
try:
sobject = self.container.create_object(remote_filename)
sobject.load_from_filename(source_path.name)
return
except self.resp_exc, error:
log.Warn(
"Upload of '%s' failed (attempt %d): CloudFiles returned: %s %s"
% (remote_filename, n, error.status, error.reason))
except Exception, e:
log.Warn("Upload of '%s' failed (attempt %s): %s: %s" %
(remote_filename, n, e.__class__.__name__, str(e)))
log.Debug("Backtrace of previous error: %s" %
exception_traceback())
time.sleep(30)
log.Warn("Giving up uploading '%s' after %s attempts" %
(remote_filename, globals.num_retries))
raise BackendException("Error uploading '%s'" % remote_filename)
def get(self, remote_filename, local_path):
for n in range(1, globals.num_retries + 1):
log.Info("Downloading '%s/%s'" % (self.container, remote_filename))
try:
sobject = self.container.create_object(remote_filename)
f = open(local_path.name, 'w')
for chunk in sobject.stream():
f.write(chunk)
local_path.setdata()
return
except self.resp_exc, resperr:
log.Warn(
"Download of '%s' failed (attempt %s): CloudFiles returned: %s %s"
try:
key.set_contents_from_filename(source_path.name, {'Content-Type': 'application/octet-stream',
'x-amz-storage-class': storage_class})
key.close()
self.resetConnection()
return
except Exception, e:
log.Warn("Upload '%s/%s' failed (attempt #%d, reason: %s: %s)"
"" % (self.straight_url,
remote_filename,
n,
e.__class__.__name__,
str(e)))
log.Debug("Backtrace of previous error: %s" % (exception_traceback(),))
self.resetConnection()
log.Warn("Giving up trying to upload %s/%s after %d attempts" %
(self.straight_url, remote_filename, globals.num_retries))
raise BackendException("Error uploading %s/%s" % (self.straight_url, remote_filename))
def get(self, remote_filename, local_path):
key = self.key_class(self.bucket)
key.key = self.key_prefix + remote_filename
for n in range(1, globals.num_retries+1):
if n > 1:
# sleep before retry (new connection to a **hopeful** new host, so no need to wait so long)
time.sleep(10)
log.Info("Downloading %s/%s" % (self.straight_url, remote_filename))
try:
key.get_contents_to_filename(local_path.name)
local_path.setdata()
self.resetConnection()
return
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
or see http://www.gnu.org/copyleft/lesser.html
"""
from builtins import object
import os
import sys
import fcntl
from duplicity import log
try:
import threading
except ImportError:
import dummy_threading # @UnusedImport
log.Warn(_(u"Threading not available -- zombie processes may appear"))
__author__ = u"Frank J. Tobin, [email protected]"
__version__ = u"0.3.2"
__revision__ = u"$Id: GnuPGInterface.py,v 1.6 2009/06/06 17:35:19 loafman Exp $"
# "standard" filehandles attached to processes
_stds = [u'stdin', u'stdout', u'stderr']
# the permissions each type of fh needs to be opened with
_fd_modes = {
u'stdin': u'wb',
u'stdout': u'rb',
u'stderr': u'r',
u'passphrase': u'w',
u'command': u'w',
def check_renamed_files(self, file_list):
if not self.user_authenticated():
self.login()
bad_list = [
x for x in file_list
if DPBX_AUTORENAMED_FILE_RE.search(x) is not None
]
if len(bad_list) == 0:
return
log.Warn(u'-' * 72)
log.Warn(
u'Warning! It looks like there are automatically renamed files on backend'
)
log.Warn(
u'They were probably created when using older version of duplicity.'
)
log.Warn(u'')
log.Warn(
u'Please check your backup consistency. Most likely you will need to choose'
)
log.Warn(
u'largest file from duplicity-* (number).gpg and remove brackets from its name.'
)
log.Warn(u'')
log.Warn(
u'These files are not managed by duplicity at all and will not be')
log.Warn(u'removed/rotated automatically.')
log.Warn(u'')
log.Warn(u'Affected files:')
for x in bad_list:
log.Warn(u'\t%s' % x)
log.Warn(u'')
log.Warn(u'In any case it\'s better to create full backup.')
log.Warn(u'-' * 72)
