def fcp_start():
global circle, fcp, treewalk
workq = None # if fresh start, workq is None
if not args.rid: # if not in recovery
treewalk = FWalk(circle, G.src, G.dest, force=args.force)
circle.begin(treewalk)
circle.finalize()
G.totalsize = treewalk.epilogue()
else: # okay, let's do checkpoint recovery
workq = prep_recovery()
circle = Circle()
fcp = FCP(circle, G.src, G.dest,
treewalk=treewalk,
totalsize=G.totalsize,
verify=args.verify,
workq=workq,
hostcnt=num_of_hosts)
if comm.rank == 0 and G.verbosity > 0:
rcl, wcl = fcp.rw_cache_limit()
print("")
print("\t{:<25}{:<10}{:5}{:<25}{:<10}".format("Read Cache:", "%s" % rcl, "|",
"Write Cache:", "%s" % wcl))
print("")
set_chunksize(fcp, G.totalsize)
fcp.checkpoint_interval = args.cptime
fcp.checkpoint_file = ".pcp_workq.%s.%s" % (args.cpid, circle.rank)
circle.begin(fcp)
circle.finalize()
fcp.cleanup()
def fcp_start():
global circle, fcp, treewalk
workq = None # if fresh start, workq is None
if not args.rid: # if not in recovery
treewalk = FWalk(circle, G.src, G.dest, force=args.force)
circle.begin(treewalk)
circle.finalize()
treewalk.epilogue()
else: # okay, let's do checkpoint recovery
workq = prep_recovery()
circle = Circle(dbname="fcp")
fcp = FCP(circle, G.src, G.dest,
treewalk=treewalk,
totalsize=T.total_filesize,
verify=args.verify,
workq=workq,
hostcnt=num_of_hosts)
set_chunksize(fcp, T.total_filesize)
fcp.checkpoint_interval = args.cptime
fcp.checkpoint_file = ".pcp_workq.%s.%s" % (args.cpid, circle.rank)
circle.begin(fcp)
circle.finalize()
fcp.epilogue()
def fcp_start():
global circle, fcp, treewalk
workq = None # if fresh start, workq is None
if not args.rid: # if not in recovery
treewalk = FWalk(circle, G.src, G.dest, force=args.force)
circle.begin(treewalk)
circle.finalize()
treewalk.epilogue()
else: # okay, let's do checkpoint recovery
workq = prep_recovery()
circle = Circle(dbname="fcp")
fcp = FCP(circle, G.src, G.dest,
treewalk=treewalk,
totalsize=T.total_filesize,
verify=args.verify,
workq=workq,
hostcnt=num_of_hosts)
if comm.rank == 0 and G.verbosity > 0:
rcl, wcl = fcp.rw_cache_limit()
print("")
print("\t{:<25}{:<10}{:5}{:<25}{:<10}".format("Read Cache:", "%s" % rcl, "|",
"Write Cache:", "%s" % wcl))
print("")
set_chunksize(fcp, T.total_filesize)
fcp.checkpoint_interval = args.cptime
fcp.checkpoint_file = ".pcp_workq.%s.%s" % (args.cpid, circle.rank)
circle.begin(fcp)
circle.finalize()
fcp.epilogue()
def main():
global args, comm
signal.signal(signal.SIGINT, sig_handler)
args = parse_and_bcast(comm, gen_parser)
try:
G.src = utils.check_src(args.path)
except ValueError as e:
err_and_exit("Error: %s not accessible" % e)
G.loglevel = args.loglevel
#G.use_store = args.use_store
G.reduce_interval = args.interval
G.memitem_threshold = args.item
hosts_cnt = tally_hosts()
circle = Circle()
if circle.rank == 0:
print("Running Parameters:\n")
print("\t{:<20}{:<20}".format("FSUM version:", __version__))
print("\t{:<20}{:<20}".format("Num of hosts:", hosts_cnt))
print("\t{:<20}{:<20}".format("Num of processes:", MPI.COMM_WORLD.Get_size()))
print("\t{:<20}{:<20}".format("Root path:", utils.choplist(G.src)))
print("\t{:<20}{:<20}".format("Items in memory:", G.memitem_threshold))
fwalk = FWalk(circle, G.src)
circle.begin(fwalk)
if G.use_store:
fwalk.flushdb()
fwalk.epilogue()
circle.finalize()
# by default, we use adaptive chunksize
chunksize = utils.calc_chunksize(T.total_filesize)
if args.chunksize:
chunksize = conv_unit(args.chunksize)
if circle.rank == 0:
print("Chunksize = ", chunksize)
circle = Circle()
fcheck = Checksum(circle, fwalk, chunksize, T.total_filesize, T.total_files)
circle.begin(fcheck)
circle.finalize()
if circle.rank == 0:
sys.stdout.write("\nAggregating ... ")
"""
chunkl = circle.comm.gather(fcheck.chunkq)
if circle.rank == 0:
chunks = [item for sublist in chunkl for item in sublist]
chunks.sort()
sys.stdout.write("%s chunks\n" % len(chunks))
sha1val = do_checksum(chunks)
with open(args.output, "w") as f:
f.write("sha1: %s\n" % sha1val)
f.write("chunksize: %s\n" % chunksize)
f.write("fwalk version: %s\n" % __version__)
f.write("src: %s\n" % utils.choplist(G.src))
f.write("date: %s\n" % utils.current_time())
f.write("totalsize: %s\n" % T.total_filesize)
print("\nSHA1: %s" % sha1val)
print("Signature file: [%s]" % args.output)
if args.export_block_signatures:
export_checksum2(chunks, args.output)
print("Exporting block signatures ... \n")
"""
if circle.rank > 0:
circle.comm.send(fcheck.bfsign.bitarray, dest=0)
else:
for p in xrange(1, circle.comm.size):
other_bitarray = circle.comm.recv(source=p)
fcheck.bfsign.or_bf(other_bitarray)
circle.comm.Barrier()
if circle.comm.rank == 0:
sha1val = fcheck.bfsign.gen_signature()
with open(args.output, "w") as f:
f.write("sha1: %s\n" % sha1val)
f.write("chunksize: %s\n" % chunksize)
f.write("fwalk version: %s\n" % __version__)
f.write("src: %s\n" % utils.choplist(G.src))
f.write("date: %s\n" % utils.current_time())
f.write("totalsize: %s\n" % T.total_filesize)
print("\nSHA1: %s" % sha1val)
print("Signature file: [%s]" % args.output)
fcheck.epilogue()
if circle.comm.rank == 0:
if os.path.exists(G.tempdir):
shutil.rmtree(G.tempdir, ignore_errors=True)
def main():
global args, comm
signal.signal(signal.SIGINT, sig_handler)
args = parse_and_bcast(comm, gen_parser)
try:
G.src = utils.check_src(args.path)
except ValueError as e:
err_and_exit("Error: %s not accessible" % e)
G.loglevel = args.loglevel
#G.use_store = args.use_store
G.reduce_interval = args.interval
G.memitem_threshold = args.item
hosts_cnt = tally_hosts()
circle = Circle()
if circle.rank == 0:
print("Running Parameters:\n")
print("\t{:<20}{:<20}".format("FSUM version:", __version__))
print("\t{:<20}{:<20}".format("Num of hosts:", hosts_cnt))
print("\t{:<20}{:<20}".format("Num of processes:",
MPI.COMM_WORLD.Get_size()))
print("\t{:<20}{:<20}".format("Root path:", utils.choplist(G.src)))
print("\t{:<20}{:<20}".format("Items in memory:", G.memitem_threshold))
fwalk = FWalk(circle, G.src)
circle.begin(fwalk)
if G.use_store:
fwalk.flushdb()
fwalk.epilogue()
circle.finalize()
# by default, we use adaptive chunksize
chunksize = utils.calc_chunksize(T.total_filesize)
if args.chunksize:
chunksize = conv_unit(args.chunksize)
if circle.rank == 0:
print("Chunksize = ", chunksize)
circle = Circle()
fcheck = Checksum(circle, fwalk, chunksize, T.total_filesize,
T.total_files)
circle.begin(fcheck)
circle.finalize()
if circle.rank == 0:
sys.stdout.write("\nAggregating ... ")
"""
chunkl = circle.comm.gather(fcheck.chunkq)
if circle.rank == 0:
chunks = [item for sublist in chunkl for item in sublist]
chunks.sort()
sys.stdout.write("%s chunks\n" % len(chunks))
sha1val = do_checksum(chunks)
with open(args.output, "w") as f:
f.write("sha1: %s\n" % sha1val)
f.write("chunksize: %s\n" % chunksize)
f.write("fwalk version: %s\n" % __version__)
f.write("src: %s\n" % utils.choplist(G.src))
f.write("date: %s\n" % utils.current_time())
f.write("totalsize: %s\n" % T.total_filesize)
print("\nSHA1: %s" % sha1val)
print("Signature file: [%s]" % args.output)
if args.export_block_signatures:
export_checksum2(chunks, args.output)
print("Exporting block signatures ... \n")
"""
if circle.rank > 0:
circle.comm.send(fcheck.bfsign.bitarray, dest=0)
else:
for p in xrange(1, circle.comm.size):
other_bitarray = circle.comm.recv(source=p)
fcheck.bfsign.or_bf(other_bitarray)
circle.comm.Barrier()
if circle.comm.rank == 0:
sha1val = fcheck.bfsign.gen_signature()
with open(args.output, "w") as f:
f.write("sha1: %s\n" % sha1val)
f.write("chunksize: %s\n" % chunksize)
f.write("fwalk version: %s\n" % __version__)
f.write("src: %s\n" % utils.choplist(G.src))
f.write("date: %s\n" % utils.current_time())
f.write("totalsize: %s\n" % T.total_filesize)
print("\nSHA1: %s" % sha1val)
print("Signature file: [%s]" % args.output)
fcheck.epilogue()
if circle.comm.rank == 0:
if os.path.exists(G.tempdir):
shutil.rmtree(G.tempdir, ignore_errors=True)
def main():
global args, comm
signal.signal(signal.SIGINT, sig_handler)
args = parse_and_bcast(comm, gen_parser)
try:
G.src = utils.check_src(args.path)
except ValueError as e:
err_and_exit("Error: %s not accessible" % e)
G.loglevel = args.loglevel
G.use_store = args.use_store
G.reduce_interval = args.interval
hosts_cnt = tally_hosts()
circle = Circle()
if circle.rank == 0:
print("Running Parameters:\n")
print("\t{:<20}{:<20}".format("FSUM version:", __version__))
print("\t{:<20}{:<20}".format("Num of hosts:", hosts_cnt))
print("\t{:<20}{:<20}".format("Num of processes:", MPI.COMM_WORLD.Get_size()))
print("\t{:<20}{:<20}".format("Root path:", utils.choplist(G.src)))
fwalk = FWalk(circle, G.src)
circle.begin(fwalk)
if G.use_store:
fwalk.flushdb()
totalsize = fwalk.epilogue()
circle.finalize()
# by default, we use adaptive chunksize
chunksize = utils.calc_chunksize(totalsize)
if args.chunksize:
chunksize = conv_unit(args.chunksize)
if circle.rank == 0:
print("Chunksize = ", chunksize)
circle = Circle()
fcheck = Checksum(circle, fwalk, chunksize, totalsize)
circle.begin(fcheck)
circle.finalize()
if circle.rank == 0:
sys.stdout.write("\nAggregating ... ")
chunkl = circle.comm.gather(fcheck.chunkq)
if circle.rank == 0:
chunks = [item for sublist in chunkl for item in sublist]
chunks.sort()
sys.stdout.write("%s chunks\n" % len(chunks))
sha1val = do_checksum(chunks)
with open(args.output, "w") as f:
f.write("sha1: %s\n" % sha1val)
f.write("chunksize: %s\n" % chunksize)
f.write("fwalk version: %s\n" % __version__)
f.write("src: %s\n" % utils.choplist(G.src))
f.write("date: %s\n" % utils.current_time())
f.write("totalsize: %s\n" % totalsize)
print("\nSHA1: %s" % sha1val)
print("Signature file: [%s]" % args.output)
if args.export_block_signatures:
export_checksum2(chunks, args.output)
print("Exporting block signatures ... \n")
fcheck.epilogue()