Esempio n. 1
0
    def test_get_olderthan(self):
        """Test getting list of files older than a certain time"""
        cs = self.get_filelist2_cs()
        oldsets = cs.get_older_than(
            dup_time.genstrtotime("2002-05-01T16:17:01-07:00"))
        oldset_times = [s.get_time() for s in oldsets]
        right_times = [dup_time.genstrtotime('2001-01-01T16:17:01-07:00')]
        assert oldset_times == right_times, \
            [oldset_times, right_times]

        oldsets_required = cs.get_older_than_required(
            dup_time.genstrtotime("2002-08-17T20:00:00-07:00"))
        oldset_times = [s.get_time() for s in oldsets_required]
        right_times_required = [dup_time.genstrtotime('2002-08-17T16:17:01-07:00')]
        assert oldset_times == right_times_required, \
            [oldset_times, right_times_required]
Esempio n. 2
0
    def test_get_olderthan(self):
        """Test getting list of files older than a certain time"""
        cs = self.get_filelist2_cs()
        oldsets = cs.get_older_than(
            dup_time.genstrtotime("2002-05-01T16:17:01-07:00"))
        oldset_times = map(lambda s: s.get_time(), oldsets)
        right_times = map(dup_time.genstrtotime, ['2001-01-01T16:17:01-07:00'])
        assert oldset_times == right_times, \
               [oldset_times, right_times]

        oldsets_required = cs.get_older_than_required(
            dup_time.genstrtotime("2002-08-17T20:00:00-07:00"))
        oldset_times = map(lambda s: s.get_time(), oldsets_required)
        right_times_required = map(dup_time.genstrtotime,
                                   ['2002-08-17T16:17:01-07:00'])
        assert oldset_times == right_times_required, \
               [oldset_times, right_times_required]
Esempio n. 3
0
 def str2time(timestr, short):
     """
     Return time in seconds if string can be converted, None otherwise
     """
     if short:
         t = from_base36(timestr)
     else:
         try:
             t = dup_time.genstrtotime(timestr.upper())
         except dup_time.TimeException:
             return None
     return t
Esempio n. 4
0
def parse_cmdline_options(arglist):
    """Parse argument list"""
    global select_opts, select_files, full_backup
    global list_current, collection_status, cleanup, remove_time, verify

    def set_log_fd(fd):
        if fd < 1:
            raise optparse.OptionValueError("log-fd must be greater than zero.")
        log.add_fd(fd)

    def set_time_sep(sep, opt):
        if sep == '-':
            raise optparse.OptionValueError("Dash ('-') not valid for time-separator.")
        globals.time_separator = sep
        old_fn_deprecation(opt)

    def add_selection(o, s, v, p):
        select_opts.append((s, v))

    def add_filelist(o, s, v, p):
        filename = v
        select_opts.append((s, filename))
        try:
            select_files.append(open(filename, "r"))
        except IOError:
            log.FatalError(_("Error opening file %s") % filename,
                           log.ErrorCode.cant_open_filelist)

    def print_ver(o, s, v, p):
        print "duplicity %s" % (globals.version)
        sys.exit(0)

    def add_rename(o, s, v, p):
        globals.rename[os.path.normcase(os.path.normpath(v[0]))] = v[1]

    parser = OPHelpFix(option_class=DupOption, usage=usage())

    # If this is true, only warn and don't raise fatal error when backup
    # source directory doesn't match previous backup source directory.
    parser.add_option("--allow-source-mismatch", action="store_true")

    # Set to the path of the archive directory (the directory which
    # contains the signatures and manifests of the relevent backup
    # collection), and for checkpoint state between volumes.
    # TRANSL: Used in usage help to represent a Unix-style path name. Example:
    # --archive-dir <path>
    parser.add_option("--archive-dir", type="file", metavar=_("path"))

    # Asynchronous put/get concurrency limit
    # (default of 0 disables asynchronicity).
    parser.add_option("--asynchronous-upload", action="store_const", const=1,
                      dest="async_concurrency")

    parser.add_option("--compare-data", action="store_true")

    # config dir for future use
    parser.add_option("--config-dir", type="file", metavar=_("path"),
                      help=optparse.SUPPRESS_HELP)

    # When symlinks are encountered, the item they point to is copied rather than
    # the symlink.
    parser.add_option("--copy-links", action="store_true")

    # for testing -- set current time
    parser.add_option("--current-time", type="int",
                      dest="current_time", help=optparse.SUPPRESS_HELP)

    # Don't actually do anything, but still report what would be done
    parser.add_option("--dry-run", action="store_true")

    # TRANSL: Used in usage help to represent an ID for a GnuPG key. Example:
    # --encrypt-key <gpg_key_id>
    parser.add_option("--encrypt-key", type="string", metavar=_("gpg-key-id"),
                      dest="", action="callback",
                      callback=lambda o, s, v, p: globals.gpg_profile.recipients.append(v))  # @UndefinedVariable

    # secret keyring in which the private encrypt key can be found
    parser.add_option("--encrypt-secret-keyring", type="string", metavar=_("path"))

    parser.add_option("--encrypt-sign-key", type="string", metavar=_("gpg-key-id"),
                      dest="", action="callback",
                      callback=lambda o, s, v, p: (globals.gpg_profile.recipients.append(v), set_sign_key(v)))

    # TRANSL: Used in usage help to represent a "glob" style pattern for
    # matching one or more files, as described in the documentation.
    # Example:
    # --exclude <shell_pattern>
    parser.add_option("--exclude", action="callback", metavar=_("shell_pattern"),
                      dest="", type="string", callback=add_selection)

    parser.add_option("--exclude-device-files", action="callback",
                      dest="", callback=add_selection)

    parser.add_option("--exclude-filelist", type="file", metavar=_("filename"),
                      dest="", action="callback", callback=add_filelist)

    parser.add_option("--exclude-filelist-stdin", action="callback", dest="",
                      callback=lambda o, s, v, p: (select_opts.append(("--exclude-filelist", "standard input")),
                                                   select_files.append(sys.stdin),
                                                   stdin_deprecation(o)),
                      help=optparse.SUPPRESS_HELP)

    parser.add_option("--exclude-globbing-filelist", type="file", metavar=_("filename"),
                      dest="", action="callback", callback=lambda o, s, v, p: (add_filelist(o, s, v, p),
                                                                               old_globbing_filelist_deprecation(s)),
                      help=optparse.SUPPRESS_HELP)

    # TRANSL: Used in usage help to represent the name of a file. Example:
    # --log-file <filename>
    parser.add_option("--exclude-if-present", metavar=_("filename"), dest="",
                      type="file", action="callback", callback=add_selection)

    parser.add_option("--exclude-other-filesystems", action="callback",
                      dest="", callback=add_selection)

    # TRANSL: Used in usage help to represent a regular expression (regexp).
    parser.add_option("--exclude-regexp", metavar=_("regular_expression"),
                      dest="", type="string", action="callback", callback=add_selection)

    # Exclude any files with modification dates older than this from the backup
    parser.add_option("--exclude-older-than", type="time", metavar=_("time"),
                      dest="", action="callback", callback=add_selection)

    # Whether we should be particularly aggressive when cleaning up
    parser.add_option("--extra-clean", action="store_true")

    # used in testing only - raises exception after volume
    parser.add_option("--fail-on-volume", type="int",
                      help=optparse.SUPPRESS_HELP)

    # used to provide a prefix on top of the defaul tar file name
    parser.add_option("--file-prefix", type="string", dest="file_prefix", action="store")

    # used to provide a suffix for manifest files only
    parser.add_option("--file-prefix-manifest", type="string", dest="file_prefix_manifest", action="store")

    # used to provide a suffix for archive files only
    parser.add_option("--file-prefix-archive", type="string", dest="file_prefix_archive", action="store")

    # used to provide a suffix for sigature files only
    parser.add_option("--file-prefix-signature", type="string", dest="file_prefix_signature", action="store")

    # used in testing only - skips upload for a given volume
    parser.add_option("--skip-volume", type="int",
                      help=optparse.SUPPRESS_HELP)

    # If set, restore only the subdirectory or file specified, not the
    # whole root.
    # TRANSL: Used in usage help to represent a Unix-style path name. Example:
    # --archive-dir <path>
    parser.add_option("--file-to-restore", "-r", action="callback", type="file",
                      metavar=_("path"), dest="restore_dir",
                      callback=lambda o, s, v, p: setattr(p.values, "restore_dir", v.strip('/')))

    # Used to confirm certain destructive operations like deleting old files.
    parser.add_option("--force", action="store_true")

    # FTP data connection type
    parser.add_option("--ftp-passive", action="store_const", const="passive", dest="ftp_connection")
    parser.add_option("--ftp-regular", action="store_const", const="regular", dest="ftp_connection")

    # If set, forces a full backup if the last full backup is older than
    # the time specified
    parser.add_option("--full-if-older-than", type="time", dest="full_force_time", metavar=_("time"))

    parser.add_option("--gio", action="callback", dest="use_gio",
                      callback=lambda o, s, v, p: (setattr(p.values, o.dest, True),
                                                   old_fn_deprecation(s)))

    parser.add_option("--gpg-binary", type="file", metavar=_("path"))

    parser.add_option("--gpg-options", action="extend", metavar=_("options"))

    # TRANSL: Used in usage help to represent an ID for a hidden GnuPG key. Example:
    # --hidden-encrypt-key <gpg_key_id>
    parser.add_option("--hidden-encrypt-key", type="string", metavar=_("gpg-key-id"),
                      dest="", action="callback",
                      callback=lambda o, s, v, p: globals.gpg_profile.hidden_recipients.append(v))  # @UndefinedVariable

    # ignore (some) errors during operations; supposed to make it more
    # likely that you are able to restore data under problematic
    # circumstances. the default should absolutely always be False unless
    # you know what you are doing.
    parser.add_option("--ignore-errors", action="callback",
                      dest="ignore_errors",
                      callback=lambda o, s, v, p: (log.Warn(
                          _("Running in 'ignore errors' mode due to %s; please "
                            "re-consider if this was not intended") % s),
                          setattr(p.values, "ignore_errors", True)))

    # Whether to use the full email address as the user name when
    # logging into an imap server. If false just the user name
    # part of the email address is used.
    parser.add_option("--imap-full-address", action="store_true")

    # Name of the imap folder where we want to store backups.
    # Can be changed with a command line argument.
    # TRANSL: Used in usage help to represent an imap mailbox
    parser.add_option("--imap-mailbox", metavar=_("imap_mailbox"))

    parser.add_option("--include", action="callback", metavar=_("shell_pattern"),
                      dest="", type="string", callback=add_selection)
    parser.add_option("--include-filelist", type="file", metavar=_("filename"),
                      dest="", action="callback", callback=add_filelist)
    parser.add_option("--include-filelist-stdin", action="callback", dest="",
                      callback=lambda o, s, v, p: (select_opts.append(("--include-filelist", "standard input")),
                                                   select_files.append(sys.stdin),
                                                   stdin_deprecation(o)),
                      help=optparse.SUPPRESS_HELP)
    parser.add_option("--include-globbing-filelist", type="file", metavar=_("filename"),
                      dest="", action="callback", callback=lambda o, s, v, p: (add_filelist(o, s, v, p),
                                                                               old_globbing_filelist_deprecation(s)),
                      help=optparse.SUPPRESS_HELP)
    parser.add_option("--include-regexp", metavar=_("regular_expression"), dest="",
                      type="string", action="callback", callback=add_selection)

    parser.add_option("--log-fd", type="int", metavar=_("file_descriptor"),
                      dest="", action="callback",
                      callback=lambda o, s, v, p: set_log_fd(v))

    # TRANSL: Used in usage help to represent the name of a file. Example:
    # --log-file <filename>
    parser.add_option("--log-file", type="file", metavar=_("filename"),
                      dest="", action="callback",
                      callback=lambda o, s, v, p: log.add_file(v))

    # Maximum block size for large files
    parser.add_option("--max-blocksize", type="int", metavar=_("number"))

    # TRANSL: Used in usage help (noun)
    parser.add_option("--name", dest="backup_name", metavar=_("backup name"))

    # If set to false, then do not encrypt files on remote system
    parser.add_option("--no-encryption", action="store_false", dest="encryption")

    # If set to false, then do not compress files on remote system
    parser.add_option("--no-compression", action="store_false", dest="compression")

    # If set, print the statistics after every backup session
    parser.add_option("--no-print-statistics", action="store_false", dest="print_statistics")

    # If true, filelists and directory statistics will be split on
    # nulls instead of newlines.
    parser.add_option("--null-separator", action="store_true")

    # number of retries on network operations
    # TRANSL: Used in usage help to represent a desired number of
    # something. Example:
    # --num-retries <number>
    parser.add_option("--num-retries", type="int", metavar=_("number"))

    # File owner uid keeps number from tar file. Like same option in GNU tar.
    parser.add_option("--numeric-owner", action="store_true")

    # Whether the old filename format is in effect.
    parser.add_option("--old-filenames", action="callback",
                      dest="old_filenames",
                      callback=lambda o, s, v, p: (setattr(p.values, o.dest, True),
                                                   old_fn_deprecation(s)))

    # Level of Redundancy in % for Par2 files
    parser.add_option("--par2-redundancy", type="int", metavar=_("number"))

    # Verbatim par2 options
    parser.add_option("--par2-options", action="extend", metavar=_("options"))

    # Used to display the progress for the full and incremental backup operations
    parser.add_option("--progress", action="store_true")

    # Used to control the progress option update rate in seconds. Default: prompts each 3 seconds
    parser.add_option("--progress-rate", type="int", metavar=_("number"))

    # option to trigger Pydev debugger
    parser.add_option("--pydevd", action="store_true")

    # option to rename files during restore
    parser.add_option("--rename", type="file", action="callback", nargs=2,
                      callback=add_rename)

    # Restores will try to bring back the state as of the following time.
    # If it is None, default to current time.
    # TRANSL: Used in usage help to represent a time spec for a previous
    # point in time, as described in the documentation. Example:
    # duplicity remove-older-than time [options] target_url
    parser.add_option("--restore-time", "--time", "-t", type="time", metavar=_("time"))

    # user added rsync options
    parser.add_option("--rsync-options", action="extend", metavar=_("options"))

    # Whether to create European buckets (sorry, hard-coded to only
    # support european for now).
    parser.add_option("--s3-european-buckets", action="store_true")

    # Whether to use S3 Reduced Redudancy Storage
    parser.add_option("--s3-use-rrs", action="store_true")

    # Whether to use S3 Infrequent Access Storage
    parser.add_option("--s3-use-ia", action="store_true")

    # Whether to use "new-style" subdomain addressing for S3 buckets. Such
    # use is not backwards-compatible with upper-case buckets, or buckets
    # that are otherwise not expressable in a valid hostname.
    parser.add_option("--s3-use-new-style", action="store_true")

    # Whether to use plain HTTP (without SSL) to send data to S3
    # See <https://bugs.launchpad.net/duplicity/+bug/433970>.
    parser.add_option("--s3-unencrypted-connection", action="store_true")

    # Chunk size used for S3 multipart uploads.The number of parallel uploads to
    # S3 be given by chunk size / volume size. Use this to maximize the use of
    # your bandwidth. Defaults to 25MB
    parser.add_option("--s3-multipart-chunk-size", type="int", action="callback", metavar=_("number"),
                      callback=lambda o, s, v, p: setattr(p.values, "s3_multipart_chunk_size", v * 1024 * 1024))

    # Number of processes to set the Processor Pool to when uploading multipart
    # uploads to S3. Use this to control the maximum simultaneous uploads to S3.
    parser.add_option("--s3-multipart-max-procs", type="int", metavar=_("number"))

    # Number of seconds to wait for each part of a multipart upload to S3. Use this
    # to prevent hangups when doing a multipart upload to S3.
    parser.add_option("--s3-multipart-max-timeout", type="int", metavar=_("number"))

    # Option to allow the s3/boto backend use the multiprocessing version.
    parser.add_option("--s3-use-multiprocessing", action="store_true")

    # Option to allow use of server side encryption in s3
    parser.add_option("--s3-use-server-side-encryption", action="store_true", dest="s3_use_sse")

    # scp command to use (ssh pexpect backend)
    parser.add_option("--scp-command", metavar=_("command"))

    # sftp command to use (ssh pexpect backend)
    parser.add_option("--sftp-command", metavar=_("command"))

    # allow the user to switch cloudfiles backend
    parser.add_option("--cf-backend", metavar=_("pyrax|cloudfiles"))

    # If set, use short (< 30 char) filenames for all the remote files.
    parser.add_option("--short-filenames", action="callback",
                      dest="short_filenames",
                      callback=lambda o, s, v, p: (setattr(p.values, o.dest, True),
                                                   old_fn_deprecation(s)))

    # TRANSL: Used in usage help to represent an ID for a GnuPG key. Example:
    # --encrypt-key <gpg_key_id>
    parser.add_option("--sign-key", type="string", metavar=_("gpg-key-id"),
                      dest="", action="callback",
                      callback=lambda o, s, v, p: set_sign_key(v))

    # default to batch mode using public-key encryption
    parser.add_option("--ssh-askpass", action="store_true")

    # user added ssh options
    parser.add_option("--ssh-options", action="extend", metavar=_("options"))

    # user added ssl options (used by webdav, lftp backend)
    parser.add_option("--ssl-cacert-file", metavar=_("pem formatted bundle of certificate authorities"))
    parser.add_option("--ssl-cacert-path", metavar=_("path to a folder with certificate authority files"))
    parser.add_option("--ssl-no-check-certificate", action="store_true")

    # Working directory for the tempfile module. Defaults to /tmp on most systems.
    parser.add_option("--tempdir", dest="temproot", type="file", metavar=_("path"))

    # network timeout value
    # TRANSL: Used in usage help. Example:
    # --timeout <seconds>
    parser.add_option("--timeout", type="int", metavar=_("seconds"))

    # Character used like the ":" in time strings like
    # 2002-08-06T04:22:00-07:00.  The colon isn't good for filenames on
    # windows machines.
    # TRANSL: abbreviation for "character" (noun)
    parser.add_option("--time-separator", type="string", metavar=_("char"),
                      action="callback",
                      callback=lambda o, s, v, p: set_time_sep(v, s))

    # Whether to specify --use-agent in GnuPG options
    parser.add_option("--use-agent", action="store_true")

    parser.add_option("--verbosity", "-v", type="verbosity", metavar="[0-9]",
                      dest="", action="callback",
                      callback=lambda o, s, v, p: log.setverbosity(v))

    parser.add_option("-V", "--version", action="callback", callback=print_ver)

    # volume size
    # TRANSL: Used in usage help to represent a desired number of
    # something. Example:
    # --num-retries <number>
    parser.add_option("--volsize", type="int", action="callback", metavar=_("number"),
                      callback=lambda o, s, v, p: setattr(p.values, "volsize", v * 1024 * 1024))

    # delay time before next try after a failure of a backend operation
    # TRANSL: Used in usage help. Example:
    # --backend-retry-delay <seconds>
    parser.add_option("--backend-retry-delay", type="int", metavar=_("seconds"))

    # parse the options
    (options, args) = parser.parse_args()

    # Copy all arguments and their values to the globals module.  Don't copy
    # attributes that are 'hidden' (start with an underscore) or whose name is
    # the empty string (used for arguments that don't directly store a value
    # by using dest="")
    for f in filter(lambda x: x and not x.startswith("_"), dir(options)):
        v = getattr(options, f)
        # Only set if v is not None because None is the default for all the
        # variables.  If user didn't set it, we'll use defaults in globals.py
        if v is not None:
            setattr(globals, f, v)

    socket.setdefaulttimeout(globals.timeout)

    # expect no cmd and two positional args
    cmd = ""
    num_expect = 2

    # process first arg as command
    if args:
        cmd = args.pop(0)
        possible = [c for c in commands if c.startswith(cmd)]
        # no unique match, that's an error
        if len(possible) > 1:
            command_line_error("command '%s' not unique, could be %s" % (cmd, possible))
        # only one match, that's a keeper
        elif len(possible) == 1:
            cmd = possible[0]
        # no matches, assume no cmd
        elif not possible:
            args.insert(0, cmd)

    if cmd == "cleanup":
        cleanup = True
        num_expect = 1
    elif cmd == "collection-status":
        collection_status = True
        num_expect = 1
    elif cmd == "full":
        full_backup = True
        num_expect = 2
    elif cmd == "incremental":
        globals.incremental = True
        num_expect = 2
    elif cmd == "list-current-files":
        list_current = True
        num_expect = 1
    elif cmd == "remove-older-than":
        try:
            arg = args.pop(0)
        except Exception:
            command_line_error("Missing time string for remove-older-than")
        globals.remove_time = dup_time.genstrtotime(arg)
        num_expect = 1
    elif cmd == "remove-all-but-n-full" or cmd == "remove-all-inc-of-but-n-full":
        if cmd == "remove-all-but-n-full":
            globals.remove_all_but_n_full_mode = True
        if cmd == "remove-all-inc-of-but-n-full":
            globals.remove_all_inc_of_but_n_full_mode = True
        try:
            arg = args.pop(0)
        except Exception:
            command_line_error("Missing count for " + cmd)
        globals.keep_chains = int(arg)
        if not globals.keep_chains > 0:
            command_line_error(cmd + " count must be > 0")
        num_expect = 1
    elif cmd == "verify":
        verify = True

    if len(args) != num_expect:
        command_line_error("Expected %d args, got %d" % (num_expect, len(args)))

    # expand pathname args, but not URL
    for loc in range(len(args)):
        if '://' not in args[loc]:
            args[loc] = expand_fn(args[loc])

    # Note that ProcessCommandLine depends on us verifying the arg
    # count here; do not remove without fixing it. We must make the
    # checks here in order to make enough sense of args to identify
    # the backend URL/lpath for args_to_path_backend().
    if len(args) < 1:
        command_line_error("Too few arguments")
    elif len(args) == 1:
        backend_url = args[0]
    elif len(args) == 2:
        lpath, backend_url = args_to_path_backend(args[0], args[1])  # @UnusedVariable
    else:
        command_line_error("Too many arguments")

    if globals.backup_name is None:
        globals.backup_name = generate_default_backup_name(backend_url)

    # set and expand archive dir
    set_archive_dir(expand_archive_dir(globals.archive_dir,
                                       globals.backup_name))

    log.Info(_("Using archive dir: %s") % (util.ufn(globals.archive_dir.name),))
    log.Info(_("Using backup name: %s") % (globals.backup_name,))

    return args
Esempio n. 5
0
def check_time(option, opt, value):
    try:
        return dup_time.genstrtotime(value)
    except dup_time.TimeException as e:
        raise optparse.OptionValueError(str(e))