def _read_hashlist(abs_hashfile, opt): # Fetch the signature file. hashfile_contents = fetch_contents("file://" + abs_hashfile, opt, short_name=opt.hash_file) if opt.compress_signature: gziptmp = tempfile.NamedTemporaryFile() with gziptmp: gziptmp.file.write(hashfile_contents) gziptmp.file.close() strfile = [] for l in gzip.open(gziptmp.name): strfile.append(l.rstrip()) else: strfile = hashfile_contents.splitlines() return hashlist_from_stringlist(strfile, opt, root=opt.source_dir)
def _read_hashlist(abs_hashfile, opt): # Fetch the signature file. hashfile_contents = fetch_contents('file://' + abs_hashfile, opt, short_name=opt.hash_file) if opt.compress_signature: gziptmp = tempfile.NamedTemporaryFile() with gziptmp: gziptmp.file.write(hashfile_contents) gziptmp.file.close() strfile = [] for l in gzip.open(gziptmp.name): strfile.append(l.rstrip()) else: strfile = hashfile_contents.splitlines() return hashlist_from_stringlist(strfile, opt, root=opt.source_dir)
def _dest_impl(abs_hashfile, src_hashlist, shortname, opt): existing_hl = None # Make a note of whether or not an old hashfile exists. old_hashfile_exists = os.path.exists(abs_hashfile) if not opt.always_checksum and old_hashfile_exists: if not opt.quiet: print("Reading existing hashfile") # Fetch the signature file. hashfile_contents = fetch_contents('file://' + abs_hashfile, opt, short_name=shortname, remote_flag=False, include_in_total=False) dst_strfile = hashfile_contents.splitlines() existing_hl = hashlist_from_stringlist(dst_strfile, opt, root=opt.dest_dir) # Calculate the differences to the local filesystem. # # Since we've just done a scan, write the results to the disk - then, # if something goes wrong, at least we've saved the scan results. (needed, not_needed, dst_hashlist) = \ hashlist_check(opt.dest_dir, src_hashlist, opt, existing_hashlist=existing_hl, opportunistic_write=True, opwrite_path=abs_hashfile, source_side=False) if opt.verify_only: return _verify_impl(needed, not_needed, opt) else: return _fetch_remote_impl(needed, not_needed, dst_hashlist, abs_hashfile, opt)
def dest_side(opt, args): ''' Implement the dest (client) side of hsync. opt is the list of command-line options, usually generated by optparse in hsync.main(). args is interpreted as a list of filter options, to allow sync of specific pieces of the source. ''' if opt.source_url is None: log.error("-u/--source-url must be defined for -D/--dest-dir mode") return False if args: log.debug("Interpreting command line arguments %s as --include items", args) if opt.include is None: opt.include = [] opt.include.extend(args) if opt.include and not opt.no_delete: log.warn("In -I/--include mode, option --no-delete is always on") opt.no_delete = True # Assemble the chain of urllib2 openers that we need to do potentially # more than one type of auth. auth_handler = _configure_http_auth(opt) proxy_handlers = None if opt.proxy_url: log.debug("Configuring proxy") proxy_handlers = _configure_http_proxy(opt) handlers = [] if proxy_handlers is not None: handlers.extend(list(proxy_handlers)) if auth_handler is not None: handlers.append(auth_handler) if handlers: log.debug("Building opener: Handlers %s", handlers) opener = urllib2.build_opener(*handlers) log.debug("Installing urllib2 opener: %s", opener) urllib2.install_opener(opener) (hashurl, shortname, compressed_sig) = _configure_hashurl(opt) # Fetch the signature file. if not opt.quiet: print("Fetching remote hashfile") hashfile_contents = fetch_contents(hashurl, opt, short_name=shortname, include_in_total=False) if hashfile_contents is None: # We're not coming back from this. log.error("Failed to retrieve signature file from '%s", hashurl) return False if compressed_sig: gziptmp = tempfile.NamedTemporaryFile() with gziptmp: gziptmp.file.write(hashfile_contents) gziptmp.file.close() src_strfile = [] for l in gzip.open(gziptmp.name): src_strfile.append(l.rstrip()) else: src_strfile = hashfile_contents.splitlines() # Release hashfile contents from memory. hashfile_contents = None if not src_strfile[-1].startswith("FINAL:"): raise TruncatedHashfileError("'FINAL:'' line of hashfile %s appears " "to be missing!" % hashurl) src_hashlist = hashlist_from_stringlist(src_strfile, opt, root=opt.dest_dir) opt.source_url = cano_url(opt.source_url, slash=True) log.debug("Source url '%s", opt.source_url) abs_hashfile = os.path.join(opt.dest_dir, opt.hash_file) abs_lockfile = abs_hashfile + '.lock' log.debug("abs_hashfile '%s' abs_lockfile '%s'", abs_hashfile, abs_lockfile) if not os.path.isdir(opt.dest_dir): os.makedirs(opt.dest_dir) with LockFileManager(abs_lockfile): return _dest_impl(abs_hashfile, src_hashlist, shortname, opt)