def _read_hashlist(abs_hashfile, opt):
# Fetch the signature file.
hashfile_contents = fetch_contents("file://" + abs_hashfile, opt, short_name=opt.hash_file)
if opt.compress_signature:
gziptmp = tempfile.NamedTemporaryFile()
with gziptmp:
gziptmp.file.write(hashfile_contents)
gziptmp.file.close()
strfile = []
for l in gzip.open(gziptmp.name):
strfile.append(l.rstrip())
else:
strfile = hashfile_contents.splitlines()
return hashlist_from_stringlist(strfile, opt, root=opt.source_dir)
def _read_hashlist(abs_hashfile, opt):
# Fetch the signature file.
hashfile_contents = fetch_contents('file://' + abs_hashfile,
opt,
short_name=opt.hash_file)
if opt.compress_signature:
gziptmp = tempfile.NamedTemporaryFile()
with gziptmp:
gziptmp.file.write(hashfile_contents)
gziptmp.file.close()
strfile = []
for l in gzip.open(gziptmp.name):
strfile.append(l.rstrip())
else:
strfile = hashfile_contents.splitlines()
return hashlist_from_stringlist(strfile, opt, root=opt.source_dir)
def _dest_impl(abs_hashfile, src_hashlist, shortname, opt):
existing_hl = None
# Make a note of whether or not an old hashfile exists.
old_hashfile_exists = os.path.exists(abs_hashfile)
if not opt.always_checksum and old_hashfile_exists:
if not opt.quiet:
print("Reading existing hashfile")
# Fetch the signature file.
hashfile_contents = fetch_contents('file://' + abs_hashfile, opt,
short_name=shortname,
remote_flag=False,
include_in_total=False)
dst_strfile = hashfile_contents.splitlines()
existing_hl = hashlist_from_stringlist(dst_strfile, opt,
root=opt.dest_dir)
# Calculate the differences to the local filesystem.
#
# Since we've just done a scan, write the results to the disk - then,
# if something goes wrong, at least we've saved the scan results.
(needed, not_needed, dst_hashlist) = \
hashlist_check(opt.dest_dir,
src_hashlist, opt,
existing_hashlist=existing_hl,
opportunistic_write=True,
opwrite_path=abs_hashfile,
source_side=False)
if opt.verify_only:
return _verify_impl(needed, not_needed, opt)
else:
return _fetch_remote_impl(needed, not_needed,
dst_hashlist, abs_hashfile, opt)
def dest_side(opt, args):
'''
Implement the dest (client) side of hsync.
opt is the list of command-line options, usually generated by optparse in
hsync.main().
args is interpreted as a list of filter options, to allow sync of specific
pieces of the source.
'''
if opt.source_url is None:
log.error("-u/--source-url must be defined for -D/--dest-dir mode")
return False
if args:
log.debug("Interpreting command line arguments %s as --include items",
args)
if opt.include is None:
opt.include = []
opt.include.extend(args)
if opt.include and not opt.no_delete:
log.warn("In -I/--include mode, option --no-delete is always on")
opt.no_delete = True
# Assemble the chain of urllib2 openers that we need to do potentially
# more than one type of auth.
auth_handler = _configure_http_auth(opt)
proxy_handlers = None
if opt.proxy_url:
log.debug("Configuring proxy")
proxy_handlers = _configure_http_proxy(opt)
handlers = []
if proxy_handlers is not None:
handlers.extend(list(proxy_handlers))
if auth_handler is not None:
handlers.append(auth_handler)
if handlers:
log.debug("Building opener: Handlers %s", handlers)
opener = urllib2.build_opener(*handlers)
log.debug("Installing urllib2 opener: %s", opener)
urllib2.install_opener(opener)
(hashurl, shortname, compressed_sig) = _configure_hashurl(opt)
# Fetch the signature file.
if not opt.quiet:
print("Fetching remote hashfile")
hashfile_contents = fetch_contents(hashurl, opt,
short_name=shortname,
include_in_total=False)
if hashfile_contents is None:
# We're not coming back from this.
log.error("Failed to retrieve signature file from '%s", hashurl)
return False
if compressed_sig:
gziptmp = tempfile.NamedTemporaryFile()
with gziptmp:
gziptmp.file.write(hashfile_contents)
gziptmp.file.close()
src_strfile = []
for l in gzip.open(gziptmp.name):
src_strfile.append(l.rstrip())
else:
src_strfile = hashfile_contents.splitlines()
# Release hashfile contents from memory.
hashfile_contents = None
if not src_strfile[-1].startswith("FINAL:"):
raise TruncatedHashfileError("'FINAL:'' line of hashfile %s appears "
"to be missing!" % hashurl)
src_hashlist = hashlist_from_stringlist(src_strfile, opt,
root=opt.dest_dir)
opt.source_url = cano_url(opt.source_url, slash=True)
log.debug("Source url '%s", opt.source_url)
abs_hashfile = os.path.join(opt.dest_dir, opt.hash_file)
abs_lockfile = abs_hashfile + '.lock'
log.debug("abs_hashfile '%s' abs_lockfile '%s'",
abs_hashfile, abs_lockfile)
if not os.path.isdir(opt.dest_dir):
os.makedirs(opt.dest_dir)
with LockFileManager(abs_lockfile):
return _dest_impl(abs_hashfile, src_hashlist, shortname, opt)