def _generate_hashfile_url(opt): abs_hashfile = None if opt.signature_url: hashurl = cano_url(opt.signature_url) log.debug("Explicit signature URL '%s'", hashurl) up = urlparse.urlparse(hashurl) if up.scheme != 'file': raise URLMustBeOfTypeFileError( "Signature URL '%s' must be a local file", hashurl) abs_hashfile = up.path log.debug("Explicit signature file '%s'", abs_hashfile) else: hashfile = 'HSYNC.SIG' if opt.hash_file: hashfile = opt.hash_file if opt.compress_signature: hashfile += '.gz' abs_hashfile = os.path.join(opt.source_dir, hashfile) log.debug("Synthesised hash file location: '%s'", abs_hashfile) return abs_hashfile
def _configure_hashurl(opt): hashurl = None hashfile = opt.hash_file shortname = hashfile compressed_sig = False if opt.remote_sig_compressed: hashfile += '.gz' compressed_sig = True if opt.signature_url: hashurl = cano_url(opt.signature_url) log.debug("Explicit signature URL '%s'", hashurl) p_url = urlparse.urlparse(hashurl) u_path = p_url.path shortname = os.path.basename(u_path) if opt.signature_url.endswith('.gz'): log.debug("Assuming compression for signature URL '%s'", opt.signature_url) compressed_sig = True if opt.remote_sig_compressed: log.debug("Force remote signature compression mode") compressed_sig = True else: if opt.remote_sig_compressed: compressed_sig = True shortname += '.gz' hashurl = cano_url(opt.source_url, slash=True) + hashfile log.debug("Synthesised signature URL '%s'", hashurl) log.debug("_configure_hashurl(): hashurl=%s shortname=%s " "compressed_sig=%s", hashurl, shortname, compressed_sig) return (hashurl, shortname, compressed_sig)
def _generate_hashfile_url(opt): abs_hashfile = None if opt.signature_url: hashurl = cano_url(opt.signature_url) log.debug("Explicit signature URL '%s'", hashurl) up = urlparse.urlparse(hashurl) if up.scheme != "file": raise URLMustBeOfTypeFileError("Signature URL '%s' must be a local file", hashurl) abs_hashfile = up.path log.debug("Explicit signature file '%s'", abs_hashfile) else: hashfile = "HSYNC.SIG" if opt.hash_file: hashfile = opt.hash_file if opt.compress_signature: hashfile += ".gz" abs_hashfile = os.path.join(opt.source_dir, hashfile) log.debug("Synthesised hash file location: '%s'", abs_hashfile) return abs_hashfile
def dest_side(opt, args): ''' Implement the dest (client) side of hsync. opt is the list of command-line options, usually generated by optparse in hsync.main(). args is interpreted as a list of filter options, to allow sync of specific pieces of the source. ''' if opt.source_url is None: log.error("-u/--source-url must be defined for -D/--dest-dir mode") return False if args: log.debug("Interpreting command line arguments %s as --include items", args) if opt.include is None: opt.include = [] opt.include.extend(args) if opt.include and not opt.no_delete: log.warn("In -I/--include mode, option --no-delete is always on") opt.no_delete = True # Assemble the chain of urllib2 openers that we need to do potentially # more than one type of auth. auth_handler = _configure_http_auth(opt) proxy_handlers = None if opt.proxy_url: log.debug("Configuring proxy") proxy_handlers = _configure_http_proxy(opt) handlers = [] if proxy_handlers is not None: handlers.extend(list(proxy_handlers)) if auth_handler is not None: handlers.append(auth_handler) if handlers: log.debug("Building opener: Handlers %s", handlers) opener = urllib2.build_opener(*handlers) log.debug("Installing urllib2 opener: %s", opener) urllib2.install_opener(opener) (hashurl, shortname, compressed_sig) = _configure_hashurl(opt) # Fetch the signature file. if not opt.quiet: print("Fetching remote hashfile") hashfile_contents = fetch_contents(hashurl, opt, short_name=shortname, include_in_total=False) if hashfile_contents is None: # We're not coming back from this. log.error("Failed to retrieve signature file from '%s", hashurl) return False if compressed_sig: gziptmp = tempfile.NamedTemporaryFile() with gziptmp: gziptmp.file.write(hashfile_contents) gziptmp.file.close() src_strfile = [] for l in gzip.open(gziptmp.name): src_strfile.append(l.rstrip()) else: src_strfile = hashfile_contents.splitlines() # Release hashfile contents from memory. hashfile_contents = None if not src_strfile[-1].startswith("FINAL:"): raise TruncatedHashfileError("'FINAL:'' line of hashfile %s appears " "to be missing!" % hashurl) src_hashlist = hashlist_from_stringlist(src_strfile, opt, root=opt.dest_dir) opt.source_url = cano_url(opt.source_url, slash=True) log.debug("Source url '%s", opt.source_url) abs_hashfile = os.path.join(opt.dest_dir, opt.hash_file) abs_lockfile = abs_hashfile + '.lock' log.debug("abs_hashfile '%s' abs_lockfile '%s'", abs_hashfile, abs_lockfile) if not os.path.isdir(opt.dest_dir): os.makedirs(opt.dest_dir) with LockFileManager(abs_lockfile): return _dest_impl(abs_hashfile, src_hashlist, shortname, opt)