def _generate_hashfile_url(opt):
abs_hashfile = None
if opt.signature_url:
hashurl = cano_url(opt.signature_url)
log.debug("Explicit signature URL '%s'", hashurl)
up = urlparse.urlparse(hashurl)
if up.scheme != 'file':
raise URLMustBeOfTypeFileError(
"Signature URL '%s' must be a local file", hashurl)
abs_hashfile = up.path
log.debug("Explicit signature file '%s'", abs_hashfile)
else:
hashfile = 'HSYNC.SIG'
if opt.hash_file:
hashfile = opt.hash_file
if opt.compress_signature:
hashfile += '.gz'
abs_hashfile = os.path.join(opt.source_dir, hashfile)
log.debug("Synthesised hash file location: '%s'", abs_hashfile)
return abs_hashfile
def _configure_hashurl(opt):
hashurl = None
hashfile = opt.hash_file
shortname = hashfile
compressed_sig = False
if opt.remote_sig_compressed:
hashfile += '.gz'
compressed_sig = True
if opt.signature_url:
hashurl = cano_url(opt.signature_url)
log.debug("Explicit signature URL '%s'", hashurl)
p_url = urlparse.urlparse(hashurl)
u_path = p_url.path
shortname = os.path.basename(u_path)
if opt.signature_url.endswith('.gz'):
log.debug("Assuming compression for signature URL '%s'",
opt.signature_url)
compressed_sig = True
if opt.remote_sig_compressed:
log.debug("Force remote signature compression mode")
compressed_sig = True
else:
if opt.remote_sig_compressed:
compressed_sig = True
shortname += '.gz'
hashurl = cano_url(opt.source_url, slash=True) + hashfile
log.debug("Synthesised signature URL '%s'", hashurl)
log.debug("_configure_hashurl(): hashurl=%s shortname=%s "
"compressed_sig=%s", hashurl, shortname, compressed_sig)
return (hashurl, shortname, compressed_sig)
def _generate_hashfile_url(opt):
abs_hashfile = None
if opt.signature_url:
hashurl = cano_url(opt.signature_url)
log.debug("Explicit signature URL '%s'", hashurl)
up = urlparse.urlparse(hashurl)
if up.scheme != "file":
raise URLMustBeOfTypeFileError("Signature URL '%s' must be a local file", hashurl)
abs_hashfile = up.path
log.debug("Explicit signature file '%s'", abs_hashfile)
else:
hashfile = "HSYNC.SIG"
if opt.hash_file:
hashfile = opt.hash_file
if opt.compress_signature:
hashfile += ".gz"
abs_hashfile = os.path.join(opt.source_dir, hashfile)
log.debug("Synthesised hash file location: '%s'", abs_hashfile)
return abs_hashfile
def dest_side(opt, args):
'''
Implement the dest (client) side of hsync.
opt is the list of command-line options, usually generated by optparse in
hsync.main().
args is interpreted as a list of filter options, to allow sync of specific
pieces of the source.
'''
if opt.source_url is None:
log.error("-u/--source-url must be defined for -D/--dest-dir mode")
return False
if args:
log.debug("Interpreting command line arguments %s as --include items",
args)
if opt.include is None:
opt.include = []
opt.include.extend(args)
if opt.include and not opt.no_delete:
log.warn("In -I/--include mode, option --no-delete is always on")
opt.no_delete = True
# Assemble the chain of urllib2 openers that we need to do potentially
# more than one type of auth.
auth_handler = _configure_http_auth(opt)
proxy_handlers = None
if opt.proxy_url:
log.debug("Configuring proxy")
proxy_handlers = _configure_http_proxy(opt)
handlers = []
if proxy_handlers is not None:
handlers.extend(list(proxy_handlers))
if auth_handler is not None:
handlers.append(auth_handler)
if handlers:
log.debug("Building opener: Handlers %s", handlers)
opener = urllib2.build_opener(*handlers)
log.debug("Installing urllib2 opener: %s", opener)
urllib2.install_opener(opener)
(hashurl, shortname, compressed_sig) = _configure_hashurl(opt)
# Fetch the signature file.
if not opt.quiet:
print("Fetching remote hashfile")
hashfile_contents = fetch_contents(hashurl, opt,
short_name=shortname,
include_in_total=False)
if hashfile_contents is None:
# We're not coming back from this.
log.error("Failed to retrieve signature file from '%s", hashurl)
return False
if compressed_sig:
gziptmp = tempfile.NamedTemporaryFile()
with gziptmp:
gziptmp.file.write(hashfile_contents)
gziptmp.file.close()
src_strfile = []
for l in gzip.open(gziptmp.name):
src_strfile.append(l.rstrip())
else:
src_strfile = hashfile_contents.splitlines()
# Release hashfile contents from memory.
hashfile_contents = None
if not src_strfile[-1].startswith("FINAL:"):
raise TruncatedHashfileError("'FINAL:'' line of hashfile %s appears "
"to be missing!" % hashurl)
src_hashlist = hashlist_from_stringlist(src_strfile, opt,
root=opt.dest_dir)
opt.source_url = cano_url(opt.source_url, slash=True)
log.debug("Source url '%s", opt.source_url)
abs_hashfile = os.path.join(opt.dest_dir, opt.hash_file)
abs_lockfile = abs_hashfile + '.lock'
log.debug("abs_hashfile '%s' abs_lockfile '%s'",
abs_hashfile, abs_lockfile)
if not os.path.isdir(opt.dest_dir):
os.makedirs(opt.dest_dir)
with LockFileManager(abs_lockfile):
return _dest_impl(abs_hashfile, src_hashlist, shortname, opt)
