def main(): global comm, args args = parse_and_bcast(comm, gen_parser) try: G.src = utils.check_src(args.path) except ValueError as e: err_and_exit("Error: %s not accessible" % e) G.use_store = args.use_store G.loglevel = args.loglevel hosts_cnt = tally_hosts() if comm.rank == 0: print("Running Parameters:\n") print("\t{:<20}{:<20}".format("FWALK version:", __version__)) print("\t{:<20}{:<20}".format("Num of hosts:", hosts_cnt)) print("\t{:<20}{:<20}".format("Num of processes:", MPI.COMM_WORLD.Get_size())) print("\t{:<20}{:<20}".format("Root path:", utils.choplist(G.src))) circle = Circle() treewalk = FWalk(circle, G.src) circle.begin(treewalk) if G.use_store: treewalk.flushdb() if args.stats: hist = global_histogram(treewalk) total = hist.sum() bucket_scale = 0.5 if comm.rank == 0: print("\nFileset histograms:\n") for idx, rightbound in enumerate(bins[1:]): percent = 100 * hist[idx] / float(total) star_count = int(bucket_scale * percent) print("\t{:<3}{:<15}{:<8}{:<8}{:<50}".format("< ", utils.bytes_fmt(rightbound), hist[idx], "%0.2f%%" % percent, '∎' * star_count)) if args.stats: treewalk.flist.sort(lambda f1, f2: cmp(f1.st_size, f2.st_size), reverse=True) globaltops = comm.gather(treewalk.flist[:args.top]) if comm.rank == 0: globaltops = [item for sublist in globaltops for item in sublist] globaltops.sort(lambda f1, f2: cmp(f1.st_size, f2.st_size), reverse=True) if len(globaltops) < args.top: args.top = len(globaltops) print("\nStats, top %s files\n" % args.top) for i in xrange(args.top): print("\t{:15}{:<30}".format(utils.bytes_fmt(globaltops[i].st_size), globaltops[i].path)) treewalk.epilogue() treewalk.cleanup() circle.finalize()
def main(): global comm, args args = parse_and_bcast(comm, gen_parser) try: G.src = utils.check_src2(args.path) except ValueError as e: err_and_exit("Error: %s not accessible" % e) G.loglevel = args.loglevel hosts_cnt = tally_hosts() if comm.rank == 0: print("Running Parameters:\n") print("\t{:<20}{:<20}".format("fprof version:", __version__)) print("\t{:<20}{:<20}".format("Num of hosts:", hosts_cnt)) print("\t{:<20}{:<20}".format("Num of processes:", MPI.COMM_WORLD.Get_size())) print("\t{:<20}{:<20}".format("Root path:", G.src)) circle = Circle() treewalk = ProfileWalk(circle, G.src, perfile=args.perfile) circle.begin(treewalk) gen_histogram() # we need the total file size to calculate GPFS efficiency total_file_size = treewalk.epilogue() if args.gpfs_block_alloc: gpfs_blocks = gather_gpfs_blocks() if comm.rank == 0: print("\nGPFS Block Alloc Report:\n") print("\tSubblocks: %s\n" % gpfs_blocks) for idx, bsz in enumerate(G.gpfs_block_size): gpfs_file_size = gpfs_blocks[idx] * G.gpfs_subs[idx] fmt_msg = "\tBlocksize: {:<6} Estimated Space: {:<20s} Efficiency: {:>6.0%}" if gpfs_file_size != 0: print(fmt_msg.format(bsz, bytes_fmt(gpfs_file_size), total_file_size/float(gpfs_file_size))) else: print(fmt_msg.format(bsz, bytes_fmt(gpfs_file_size), 0)) treewalk.cleanup() circle.finalize()
def main(): global comm, args, stripe_out, DIR_BINS, DIR_HIST fpipe.listen() args = parse_and_bcast(comm, gen_parser) try: G.src = utils.check_src2(args.path) except ValueError as e: err_and_exit("Error: %s not accessible" % e) G.memitem_threshold = args.item G.loglevel = args.loglevel hosts_cnt = tally_hosts() # doing directory profiling? if args.dirprof: # check the input if args.dirbins is None: # err_and_exit("Error: missing directory bin parameters: a sorted integer list\n") args.dirbins = [ 0, 10, 100, 1000, 10**4, 10**5, 10**6, 10**7, 10**8 ] else: myList = sorted(set(args.dirbins)) if myList != args.dirbins: err_and_exit("Error: duplicated, or unsorted bins: %s\n" % args.dirbins) DIR_BINS = args.dirbins DIR_HIST = [0] * (len(DIR_BINS) + 1) # Doing stripe analysis? lfs is not really bullet-proof way # we might need a better way of doing fstype check. if args.lustre_stripe: G.lfs_bin = lfs.check_lfs() G.stripe_threshold = utils.conv_unit(args.stripe_threshold) try: stripe_out = os.open(args.stripe_output, os.O_CREAT | os.O_WRONLY | os.O_APPEND) except: err_and_exit("Error: can't create stripe output: %s" % args.stripe_output) if args.exclude: process_exclude_file() if comm.rank == 0: print("Running Parameters:\n") print("\t{0:<20}{1:<20}".format("fprof version:", __version__)) print("\t{0:<20}{1:<20}".format("Full rev id:", __revid__)) print("\t{0:<20}{1:<20}".format("Num of hosts:", hosts_cnt)) print("\t{0:<20}{1:<20}".format("Num of processes:", MPI.COMM_WORLD.Get_size())) if args.syslog: print("\t{0:<20}{1:<20}".format("Syslog report: ", "yes")) else: print("\t{0:<20}{1:<20}".format("Syslog report: ", "no")) if args.dirprof: print("\t{0:<20}{1:<20}".format("Dir bins: ", args.dirbins)) if args.lustre_stripe: print("\t{0:<20}{1:<20}".format("Stripe analysis: ", "yes")) print("\t{0:<20}{1:<20}".format("Stripe threshold: ", args.stripe_threshold)) else: print("\t{0:<20}{1:<20}".format("Stripe analysis: ", "no")) print("\t{0:<20}{1:<20}".format("Root path:", G.src)) if args.exclude: print("\nExclusions:\n") for ele in EXCLUDE: print("\t %s" % ele) circle = Circle() if args.perprocess: circle.report_enabled = True else: circle.report_enabled = False if args.progress: circle.report_enabled = False circle.reduce_enabled = True treewalk = ProfileWalk(circle, G.src, perfile=args.perfile) circle.begin(treewalk) # we need the total file size to calculate GPFS efficiency total_file_size = treewalk.epilogue() msg1, msg2 = gen_histogram(total_file_size) if args.dirprof: gen_directory_histogram() if comm.rank == 0 and args.syslog: sendto_syslog("fprof.filecount.hist", msg1) sendto_syslog("fprof.fsize_perc.hist", msg2) if args.topn_files: topfiles = gather_topfiles() if comm.rank == 0: print("\nTop N File Report:\n") # edge case: not enough files (< args.top) totaln = args.topn_files if len( topfiles) > args.topn_files else len(topfiles) for index, _ in enumerate(xrange(totaln)): size, path = topfiles[index] print("\t%s: %s (%s)" % (index + 1, path, utils.bytes_fmt(size))) print("") if args.topn_dirs: topdirs = gather_topdirs() if comm.rank == 0: print("\nTop N Directory Report:\n") totaln = args.topn_dirs if len(topdirs) > args.topn_dirs else len( topdirs) for index, _ in enumerate(xrange(totaln)): size, path = topdirs[index] print("\t{0:}: {1:} ({2:,} items)".format( index + 1, path, size)) print("") if args.gpfs_block_alloc: gpfs_blocks = gather_gpfs_blocks() gather_gpfs_dii() if comm.rank == 0: print("\nGPFS Block Alloc Report:\n") print("\t{0:<15}{1:<4}".format("inode size:", args.inodesz)) print("\t{0:<25}{1:>15,}".format("DII (data-in-inode) count:", DII_COUNT)) print("\tSubblocks: %s\n" % gpfs_blocks) fmt_msg = "\tBlocksize: {0:<6} Estimated Space: {1:<20s} Efficiency: {2:>6.2%}" for idx, bsz in enumerate(G.gpfs_block_size): gpfs_file_size = gpfs_blocks[idx] * G.gpfs_subs[idx] if gpfs_file_size != 0: print( fmt_msg.format(bsz, bytes_fmt(gpfs_file_size), total_file_size / float(gpfs_file_size))) else: print(fmt_msg.format(bsz, bytes_fmt(gpfs_file_size), 0)) treewalk.cleanup() circle.finalize() if args.lustre_stripe and stripe_out: os.close(stripe_out) sp_workload = comm.gather(Tally.spcnt) if comm.rank == 0: print("Stripe workload total: %s, distribution: %s" % (sum(sp_workload), sp_workload))
def main(): global comm, args args = parse_and_bcast(comm, gen_parser) try: G.src = utils.check_src(args.path) except ValueError as e: err_and_exit("Error: %s not accessible" % e) G.use_store = args.use_store G.loglevel = args.loglevel hosts_cnt = tally_hosts() if comm.rank == 0: print("Running Parameters:\n") print("\t{:<20}{:<20}".format("FWALK version:", __version__)) print("\t{:<20}{:<20}".format("Num of hosts:", hosts_cnt)) print("\t{:<20}{:<20}".format("Num of processes:", MPI.COMM_WORLD.Get_size())) print("\t{:<20}{:<20}".format("Root path:", utils.choplist(G.src))) circle = Circle() treewalk = FWalk(circle, G.src) circle.begin(treewalk) if G.use_store: treewalk.flushdb() if args.stats: hist = global_histogram(treewalk) total = hist.sum() bucket_scale = 0.5 if comm.rank == 0: print("\nFileset histograms:\n") for idx, rightbound in enumerate(bins[1:]): percent = 100 * hist[idx] / float(total) star_count = int(bucket_scale * percent) print("\t{:<3}{:<15}{:<8}{:<8}{:<50}".format( "< ", utils.bytes_fmt(rightbound), hist[idx], "%0.2f%%" % percent, '∎' * star_count)) if args.stats: treewalk.flist.sort(lambda f1, f2: cmp(f1.st_size, f2.st_size), reverse=True) globaltops = comm.gather(treewalk.flist[:args.top]) if comm.rank == 0: globaltops = [item for sublist in globaltops for item in sublist] globaltops.sort(lambda f1, f2: cmp(f1.st_size, f2.st_size), reverse=True) if len(globaltops) < args.top: args.top = len(globaltops) print("\nStats, top %s files\n" % args.top) for i in xrange(args.top): print("\t{:15}{:<30}".format( utils.bytes_fmt(globaltops[i].st_size), globaltops[i].path)) treewalk.epilogue() treewalk.cleanup() circle.finalize()
def main(): global comm, args fpipe.listen() args = parse_and_bcast(comm, gen_parser) try: G.src = utils.check_src2(args.path) except ValueError as e: err_and_exit("Error: %s not accessible" % e) G.loglevel = args.loglevel hosts_cnt = tally_hosts() if args.exclude: process_exclude_file() if comm.rank == 0: print("Running Parameters:\n") print("\t{:<20}{:<20}".format("fprof version:", __version__)) print("\t{:<20}{:<20}".format("Num of hosts:", hosts_cnt)) print("\t{:<20}{:<20}".format("Num of processes:", MPI.COMM_WORLD.Get_size())) print("\t{:<20}{:<20}".format("Root path:", G.src)) if args.exclude: print("\nExclusions:\n") for ele in EXCLUDE: print("\t %s" % ele) circle = Circle() if args.perprocess: circle.report_enabled = True else: circle.reduce_enabled = True treewalk = ProfileWalk(circle, G.src, perfile=args.perfile) circle.begin(treewalk) # we need the total file size to calculate GPFS efficiency total_file_size = treewalk.epilogue() msg1, msg2 = gen_histogram(total_file_size) if comm.rank == 0: sendto_syslog("fprof.filecount.hist", msg1) sendto_syslog("fprof.fsize_perc.hist", msg2) if args.top: topfiles = gather_topfiles() if comm.rank == 0: print("\nTop File Report:\n") # edge case: not enough files (< args.top) totaln = args.top if len(topfiles) > args.top else len(topfiles) for index, _ in enumerate(xrange(totaln)): size, path = topfiles[index] print("\t%s: %s (%s)" % (index + 1, path, utils.bytes_fmt(size))) print("") if args.gpfs_block_alloc: gpfs_blocks = gather_gpfs_blocks() if comm.rank == 0: print("\nGPFS Block Alloc Report:\n") print("\tinode size: %s" % args.inodesz) print("\tDII (data-in-inode) count: %s" % DII_COUNT) print("\tSubblocks: %s\n" % gpfs_blocks) for idx, bsz in enumerate(G.gpfs_block_size): gpfs_file_size = gpfs_blocks[idx] * G.gpfs_subs[idx] fmt_msg = "\tBlocksize: {:<6} Estimated Space: {:<20s} Efficiency: {:>6.2%}" if gpfs_file_size != 0: print(fmt_msg.format(bsz, bytes_fmt(gpfs_file_size), total_file_size/float(gpfs_file_size))) else: print(fmt_msg.format(bsz, bytes_fmt(gpfs_file_size), 0)) treewalk.cleanup() circle.finalize()