def main():
global args, comm
signal.signal(signal.SIGINT, sig_handler)
args = parse_and_bcast(comm, gen_parser)
try:
G.src = utils.check_src(args.path)
except ValueError as e:
err_and_exit("Error: %s not accessible" % e)
G.loglevel = args.loglevel
#G.use_store = args.use_store
G.reduce_interval = args.interval
G.memitem_threshold = args.item
hosts_cnt = tally_hosts()
circle = Circle()
if circle.rank == 0:
print("Running Parameters:\n")
print("\t{:<20}{:<20}".format("FSUM version:", __version__))
print("\t{:<20}{:<20}".format("Num of hosts:", hosts_cnt))
print("\t{:<20}{:<20}".format("Num of processes:", MPI.COMM_WORLD.Get_size()))
print("\t{:<20}{:<20}".format("Root path:", utils.choplist(G.src)))
print("\t{:<20}{:<20}".format("Items in memory:", G.memitem_threshold))
fwalk = FWalk(circle, G.src)
circle.begin(fwalk)
if G.use_store:
fwalk.flushdb()
fwalk.epilogue()
circle.finalize()
# by default, we use adaptive chunksize
chunksize = utils.calc_chunksize(T.total_filesize)
if args.chunksize:
chunksize = conv_unit(args.chunksize)
if circle.rank == 0:
print("Chunksize = ", chunksize)
circle = Circle()
fcheck = Checksum(circle, fwalk, chunksize, T.total_filesize, T.total_files)
circle.begin(fcheck)
circle.finalize()
if circle.rank == 0:
sys.stdout.write("\nAggregating ... ")
"""
chunkl = circle.comm.gather(fcheck.chunkq)
if circle.rank == 0:
chunks = [item for sublist in chunkl for item in sublist]
chunks.sort()
sys.stdout.write("%s chunks\n" % len(chunks))
sha1val = do_checksum(chunks)
with open(args.output, "w") as f:
f.write("sha1: %s\n" % sha1val)
f.write("chunksize: %s\n" % chunksize)
f.write("fwalk version: %s\n" % __version__)
f.write("src: %s\n" % utils.choplist(G.src))
f.write("date: %s\n" % utils.current_time())
f.write("totalsize: %s\n" % T.total_filesize)
print("\nSHA1: %s" % sha1val)
print("Signature file: [%s]" % args.output)
if args.export_block_signatures:
export_checksum2(chunks, args.output)
print("Exporting block signatures ... \n")
"""
if circle.rank > 0:
circle.comm.send(fcheck.bfsign.bitarray, dest=0)
else:
for p in xrange(1, circle.comm.size):
other_bitarray = circle.comm.recv(source=p)
fcheck.bfsign.or_bf(other_bitarray)
circle.comm.Barrier()
if circle.comm.rank == 0:
sha1val = fcheck.bfsign.gen_signature()
with open(args.output, "w") as f:
f.write("sha1: %s\n" % sha1val)
f.write("chunksize: %s\n" % chunksize)
f.write("fwalk version: %s\n" % __version__)
f.write("src: %s\n" % utils.choplist(G.src))
f.write("date: %s\n" % utils.current_time())
f.write("totalsize: %s\n" % T.total_filesize)
print("\nSHA1: %s" % sha1val)
print("Signature file: [%s]" % args.output)
fcheck.epilogue()
if circle.comm.rank == 0:
if os.path.exists(G.tempdir):
shutil.rmtree(G.tempdir, ignore_errors=True)
def main():
global args, comm
signal.signal(signal.SIGINT, sig_handler)
args = parse_and_bcast(comm, gen_parser)
try:
G.src = utils.check_src(args.path)
except ValueError as e:
err_and_exit("Error: %s not accessible" % e)
G.loglevel = args.loglevel
#G.use_store = args.use_store
G.reduce_interval = args.interval
G.memitem_threshold = args.item
hosts_cnt = tally_hosts()
circle = Circle()
if circle.rank == 0:
print("Running Parameters:\n")
print("\t{:<20}{:<20}".format("FSUM version:", __version__))
print("\t{:<20}{:<20}".format("Num of hosts:", hosts_cnt))
print("\t{:<20}{:<20}".format("Num of processes:",
MPI.COMM_WORLD.Get_size()))
print("\t{:<20}{:<20}".format("Root path:", utils.choplist(G.src)))
print("\t{:<20}{:<20}".format("Items in memory:", G.memitem_threshold))
fwalk = FWalk(circle, G.src)
circle.begin(fwalk)
if G.use_store:
fwalk.flushdb()
fwalk.epilogue()
circle.finalize()
# by default, we use adaptive chunksize
chunksize = utils.calc_chunksize(T.total_filesize)
if args.chunksize:
chunksize = conv_unit(args.chunksize)
if circle.rank == 0:
print("Chunksize = ", chunksize)
circle = Circle()
fcheck = Checksum(circle, fwalk, chunksize, T.total_filesize,
T.total_files)
circle.begin(fcheck)
circle.finalize()
if circle.rank == 0:
sys.stdout.write("\nAggregating ... ")
"""
chunkl = circle.comm.gather(fcheck.chunkq)
if circle.rank == 0:
chunks = [item for sublist in chunkl for item in sublist]
chunks.sort()
sys.stdout.write("%s chunks\n" % len(chunks))
sha1val = do_checksum(chunks)
with open(args.output, "w") as f:
f.write("sha1: %s\n" % sha1val)
f.write("chunksize: %s\n" % chunksize)
f.write("fwalk version: %s\n" % __version__)
f.write("src: %s\n" % utils.choplist(G.src))
f.write("date: %s\n" % utils.current_time())
f.write("totalsize: %s\n" % T.total_filesize)
print("\nSHA1: %s" % sha1val)
print("Signature file: [%s]" % args.output)
if args.export_block_signatures:
export_checksum2(chunks, args.output)
print("Exporting block signatures ... \n")
"""
if circle.rank > 0:
circle.comm.send(fcheck.bfsign.bitarray, dest=0)
else:
for p in xrange(1, circle.comm.size):
other_bitarray = circle.comm.recv(source=p)
fcheck.bfsign.or_bf(other_bitarray)
circle.comm.Barrier()
if circle.comm.rank == 0:
sha1val = fcheck.bfsign.gen_signature()
with open(args.output, "w") as f:
f.write("sha1: %s\n" % sha1val)
f.write("chunksize: %s\n" % chunksize)
f.write("fwalk version: %s\n" % __version__)
f.write("src: %s\n" % utils.choplist(G.src))
f.write("date: %s\n" % utils.current_time())
f.write("totalsize: %s\n" % T.total_filesize)
print("\nSHA1: %s" % sha1val)
print("Signature file: [%s]" % args.output)
fcheck.epilogue()
if circle.comm.rank == 0:
if os.path.exists(G.tempdir):
shutil.rmtree(G.tempdir, ignore_errors=True)
def set_adaptive_chunksize(self, totalsz):
self.chunksize = utils.calc_chunksize(totalsz)
if self.circle.rank == 0:
print("Adaptive chunksize: %s" % bytes_fmt(self.chunksize))
def main():
global args, comm
signal.signal(signal.SIGINT, sig_handler)
args = parse_and_bcast(comm, gen_parser)
try:
G.src = utils.check_src(args.path)
except ValueError as e:
err_and_exit("Error: %s not accessible" % e)
G.loglevel = args.loglevel
G.use_store = args.use_store
G.reduce_interval = args.interval
hosts_cnt = tally_hosts()
circle = Circle()
if circle.rank == 0:
print("Running Parameters:\n")
print("\t{:<20}{:<20}".format("FSUM version:", __version__))
print("\t{:<20}{:<20}".format("Num of hosts:", hosts_cnt))
print("\t{:<20}{:<20}".format("Num of processes:", MPI.COMM_WORLD.Get_size()))
print("\t{:<20}{:<20}".format("Root path:", utils.choplist(G.src)))
fwalk = FWalk(circle, G.src)
circle.begin(fwalk)
if G.use_store:
fwalk.flushdb()
totalsize = fwalk.epilogue()
circle.finalize()
# by default, we use adaptive chunksize
chunksize = utils.calc_chunksize(totalsize)
if args.chunksize:
chunksize = conv_unit(args.chunksize)
if circle.rank == 0:
print("Chunksize = ", chunksize)
circle = Circle()
fcheck = Checksum(circle, fwalk, chunksize, totalsize)
circle.begin(fcheck)
circle.finalize()
if circle.rank == 0:
sys.stdout.write("\nAggregating ... ")
chunkl = circle.comm.gather(fcheck.chunkq)
if circle.rank == 0:
chunks = [item for sublist in chunkl for item in sublist]
chunks.sort()
sys.stdout.write("%s chunks\n" % len(chunks))
sha1val = do_checksum(chunks)
with open(args.output, "w") as f:
f.write("sha1: %s\n" % sha1val)
f.write("chunksize: %s\n" % chunksize)
f.write("fwalk version: %s\n" % __version__)
f.write("src: %s\n" % utils.choplist(G.src))
f.write("date: %s\n" % utils.current_time())
f.write("totalsize: %s\n" % totalsize)
print("\nSHA1: %s" % sha1val)
print("Signature file: [%s]" % args.output)
if args.export_block_signatures:
export_checksum2(chunks, args.output)
print("Exporting block signatures ... \n")
fcheck.epilogue()
def set_adaptive_chunksize(self, totalsz):
self.chunksize = utils.calc_chunksize(totalsz)
if self.circle.rank == 0:
print("Adaptive chunksize: %s" % bytes_fmt(self.chunksize))