Ejemplo n.º 1
0
def _read_hashlist(abs_hashfile, opt):
    # Fetch the signature file.
    hashfile_contents = fetch_contents("file://" + abs_hashfile, opt, short_name=opt.hash_file)

    if opt.compress_signature:
        gziptmp = tempfile.NamedTemporaryFile()
        with gziptmp:
            gziptmp.file.write(hashfile_contents)
            gziptmp.file.close()
            strfile = []
            for l in gzip.open(gziptmp.name):
                strfile.append(l.rstrip())

    else:
        strfile = hashfile_contents.splitlines()

    return hashlist_from_stringlist(strfile, opt, root=opt.source_dir)
Ejemplo n.º 2
0
def _read_hashlist(abs_hashfile, opt):
    # Fetch the signature file.
    hashfile_contents = fetch_contents('file://' + abs_hashfile,
                                       opt,
                                       short_name=opt.hash_file)

    if opt.compress_signature:
        gziptmp = tempfile.NamedTemporaryFile()
        with gziptmp:
            gziptmp.file.write(hashfile_contents)
            gziptmp.file.close()
            strfile = []
            for l in gzip.open(gziptmp.name):
                strfile.append(l.rstrip())

    else:
        strfile = hashfile_contents.splitlines()

    return hashlist_from_stringlist(strfile, opt, root=opt.source_dir)
Ejemplo n.º 3
0
def _dest_impl(abs_hashfile, src_hashlist, shortname, opt):

    existing_hl = None

    # Make a note of whether or not an old hashfile exists.
    old_hashfile_exists = os.path.exists(abs_hashfile)

    if not opt.always_checksum and old_hashfile_exists:
        if not opt.quiet:
            print("Reading existing hashfile")

        # Fetch the signature file.
        hashfile_contents = fetch_contents('file://' + abs_hashfile, opt,
                                           short_name=shortname,
                                           remote_flag=False,
                                           include_in_total=False)
        dst_strfile = hashfile_contents.splitlines()
        existing_hl = hashlist_from_stringlist(dst_strfile, opt,
                                               root=opt.dest_dir)

    # Calculate the differences to the local filesystem.
    #
    # Since we've just done a scan, write the results to the disk - then,
    # if something goes wrong, at least we've saved the scan results.
    (needed, not_needed, dst_hashlist) = \
        hashlist_check(opt.dest_dir,
                       src_hashlist, opt,
                       existing_hashlist=existing_hl,
                       opportunistic_write=True,
                       opwrite_path=abs_hashfile,
                       source_side=False)

    if opt.verify_only:
        return _verify_impl(needed, not_needed, opt)
    else:
        return _fetch_remote_impl(needed, not_needed,
                                  dst_hashlist, abs_hashfile, opt)
Ejemplo n.º 4
0
def dest_side(opt, args):
    '''
    Implement the dest (client) side of hsync.

    opt is the list of command-line options, usually generated by optparse in
    hsync.main().

    args is interpreted as a list of filter options, to allow sync of specific
    pieces of the source.

    '''

    if opt.source_url is None:
        log.error("-u/--source-url must be defined for -D/--dest-dir mode")
        return False

    if args:
        log.debug("Interpreting command line arguments %s as --include items",
                  args)
        if opt.include is None:
            opt.include = []
        opt.include.extend(args)

    if opt.include and not opt.no_delete:
        log.warn("In -I/--include mode, option --no-delete is always on")
        opt.no_delete = True

    # Assemble the chain of urllib2 openers that we need to do potentially
    # more than one type of auth.

    auth_handler = _configure_http_auth(opt)

    proxy_handlers = None
    if opt.proxy_url:
        log.debug("Configuring proxy")
        proxy_handlers = _configure_http_proxy(opt)

    handlers = []
    if proxy_handlers is not None:
        handlers.extend(list(proxy_handlers))
    if auth_handler is not None:
        handlers.append(auth_handler)

    if handlers:
        log.debug("Building opener: Handlers %s", handlers)
        opener = urllib2.build_opener(*handlers)

        log.debug("Installing urllib2 opener: %s", opener)
        urllib2.install_opener(opener)

    (hashurl, shortname, compressed_sig) = _configure_hashurl(opt)

    # Fetch the signature file.
    if not opt.quiet:
        print("Fetching remote hashfile")

    hashfile_contents = fetch_contents(hashurl, opt,
                                       short_name=shortname,
                                       include_in_total=False)

    if hashfile_contents is None:
        # We're not coming back from this.
        log.error("Failed to retrieve signature file from '%s", hashurl)
        return False

    if compressed_sig:
        gziptmp = tempfile.NamedTemporaryFile()
        with gziptmp:
            gziptmp.file.write(hashfile_contents)
            gziptmp.file.close()
            src_strfile = []
            for l in gzip.open(gziptmp.name):
                src_strfile.append(l.rstrip())
    else:
        src_strfile = hashfile_contents.splitlines()

    # Release hashfile contents from memory.
    hashfile_contents = None

    if not src_strfile[-1].startswith("FINAL:"):
        raise TruncatedHashfileError("'FINAL:'' line of hashfile %s appears "
                                     "to be missing!" % hashurl)
    src_hashlist = hashlist_from_stringlist(src_strfile, opt,
                                            root=opt.dest_dir)

    opt.source_url = cano_url(opt.source_url, slash=True)
    log.debug("Source url '%s", opt.source_url)

    abs_hashfile = os.path.join(opt.dest_dir, opt.hash_file)
    abs_lockfile = abs_hashfile + '.lock'
    log.debug("abs_hashfile '%s' abs_lockfile '%s'",
              abs_hashfile, abs_lockfile)

    if not os.path.isdir(opt.dest_dir):
        os.makedirs(opt.dest_dir)

    with LockFileManager(abs_lockfile):

        return _dest_impl(abs_hashfile, src_hashlist, shortname, opt)