def main():
global comm, args
args = parse_and_bcast(comm, gen_parser)
try:
G.src = utils.check_src(args.path)
except ValueError as e:
err_and_exit("Error: %s not accessible" % e)
G.use_store = args.use_store
G.loglevel = args.loglevel
hosts_cnt = tally_hosts()
if comm.rank == 0:
print("Running Parameters:\n")
print("\t{:<20}{:<20}".format("FWALK version:", __version__))
print("\t{:<20}{:<20}".format("Num of hosts:", hosts_cnt))
print("\t{:<20}{:<20}".format("Num of processes:", MPI.COMM_WORLD.Get_size()))
print("\t{:<20}{:<20}".format("Root path:", utils.choplist(G.src)))
circle = Circle()
treewalk = FWalk(circle, G.src)
circle.begin(treewalk)
if G.use_store:
treewalk.flushdb()
if args.stats:
hist = global_histogram(treewalk)
total = hist.sum()
bucket_scale = 0.5
if comm.rank == 0:
print("\nFileset histograms:\n")
for idx, rightbound in enumerate(bins[1:]):
percent = 100 * hist[idx] / float(total)
star_count = int(bucket_scale * percent)
print("\t{:<3}{:<15}{:<8}{:<8}{:<50}".format("< ",
utils.bytes_fmt(rightbound), hist[idx],
"%0.2f%%" % percent, '∎' * star_count))
if args.stats:
treewalk.flist.sort(lambda f1, f2: cmp(f1.st_size, f2.st_size), reverse=True)
globaltops = comm.gather(treewalk.flist[:args.top])
if comm.rank == 0:
globaltops = [item for sublist in globaltops for item in sublist]
globaltops.sort(lambda f1, f2: cmp(f1.st_size, f2.st_size), reverse=True)
if len(globaltops) < args.top:
args.top = len(globaltops)
print("\nStats, top %s files\n" % args.top)
for i in xrange(args.top):
print("\t{:15}{:<30}".format(utils.bytes_fmt(globaltops[i].st_size),
globaltops[i].path))
treewalk.epilogue()
treewalk.cleanup()
circle.finalize()
def main():
global comm, args
args = parse_and_bcast(comm, gen_parser)
try:
G.src = utils.check_src2(args.path)
except ValueError as e:
err_and_exit("Error: %s not accessible" % e)
G.loglevel = args.loglevel
hosts_cnt = tally_hosts()
if comm.rank == 0:
print("Running Parameters:\n")
print("\t{:<20}{:<20}".format("fprof version:", __version__))
print("\t{:<20}{:<20}".format("Num of hosts:", hosts_cnt))
print("\t{:<20}{:<20}".format("Num of processes:", MPI.COMM_WORLD.Get_size()))
print("\t{:<20}{:<20}".format("Root path:", G.src))
circle = Circle()
treewalk = ProfileWalk(circle, G.src, perfile=args.perfile)
circle.begin(treewalk)
gen_histogram()
# we need the total file size to calculate GPFS efficiency
total_file_size = treewalk.epilogue()
if args.gpfs_block_alloc:
gpfs_blocks = gather_gpfs_blocks()
if comm.rank == 0:
print("\nGPFS Block Alloc Report:\n")
print("\tSubblocks: %s\n" % gpfs_blocks)
for idx, bsz in enumerate(G.gpfs_block_size):
gpfs_file_size = gpfs_blocks[idx] * G.gpfs_subs[idx]
fmt_msg = "\tBlocksize: {:<6} Estimated Space: {:<20s} Efficiency: {:>6.0%}"
if gpfs_file_size != 0:
print(fmt_msg.format(bsz, bytes_fmt(gpfs_file_size), total_file_size/float(gpfs_file_size)))
else:
print(fmt_msg.format(bsz, bytes_fmt(gpfs_file_size), 0))
treewalk.cleanup()
circle.finalize()
def main():
global comm, args, stripe_out, DIR_BINS, DIR_HIST
fpipe.listen()
args = parse_and_bcast(comm, gen_parser)
try:
G.src = utils.check_src2(args.path)
except ValueError as e:
err_and_exit("Error: %s not accessible" % e)
G.memitem_threshold = args.item
G.loglevel = args.loglevel
hosts_cnt = tally_hosts()
# doing directory profiling?
if args.dirprof:
# check the input
if args.dirbins is None:
# err_and_exit("Error: missing directory bin parameters: a sorted integer list\n")
args.dirbins = [
0, 10, 100, 1000, 10**4, 10**5, 10**6, 10**7, 10**8
]
else:
myList = sorted(set(args.dirbins))
if myList != args.dirbins:
err_and_exit("Error: duplicated, or unsorted bins: %s\n" %
args.dirbins)
DIR_BINS = args.dirbins
DIR_HIST = [0] * (len(DIR_BINS) + 1)
# Doing stripe analysis? lfs is not really bullet-proof way
# we might need a better way of doing fstype check.
if args.lustre_stripe:
G.lfs_bin = lfs.check_lfs()
G.stripe_threshold = utils.conv_unit(args.stripe_threshold)
try:
stripe_out = os.open(args.stripe_output,
os.O_CREAT | os.O_WRONLY | os.O_APPEND)
except:
err_and_exit("Error: can't create stripe output: %s" %
args.stripe_output)
if args.exclude:
process_exclude_file()
if comm.rank == 0:
print("Running Parameters:\n")
print("\t{0:<20}{1:<20}".format("fprof version:", __version__))
print("\t{0:<20}{1:<20}".format("Full rev id:", __revid__))
print("\t{0:<20}{1:<20}".format("Num of hosts:", hosts_cnt))
print("\t{0:<20}{1:<20}".format("Num of processes:",
MPI.COMM_WORLD.Get_size()))
if args.syslog:
print("\t{0:<20}{1:<20}".format("Syslog report: ", "yes"))
else:
print("\t{0:<20}{1:<20}".format("Syslog report: ", "no"))
if args.dirprof:
print("\t{0:<20}{1:<20}".format("Dir bins: ", args.dirbins))
if args.lustre_stripe:
print("\t{0:<20}{1:<20}".format("Stripe analysis: ", "yes"))
print("\t{0:<20}{1:<20}".format("Stripe threshold: ",
args.stripe_threshold))
else:
print("\t{0:<20}{1:<20}".format("Stripe analysis: ", "no"))
print("\t{0:<20}{1:<20}".format("Root path:", G.src))
if args.exclude:
print("\nExclusions:\n")
for ele in EXCLUDE:
print("\t %s" % ele)
circle = Circle()
if args.perprocess:
circle.report_enabled = True
else:
circle.report_enabled = False
if args.progress:
circle.report_enabled = False
circle.reduce_enabled = True
treewalk = ProfileWalk(circle, G.src, perfile=args.perfile)
circle.begin(treewalk)
# we need the total file size to calculate GPFS efficiency
total_file_size = treewalk.epilogue()
msg1, msg2 = gen_histogram(total_file_size)
if args.dirprof:
gen_directory_histogram()
if comm.rank == 0 and args.syslog:
sendto_syslog("fprof.filecount.hist", msg1)
sendto_syslog("fprof.fsize_perc.hist", msg2)
if args.topn_files:
topfiles = gather_topfiles()
if comm.rank == 0:
print("\nTop N File Report:\n")
# edge case: not enough files (< args.top)
totaln = args.topn_files if len(
topfiles) > args.topn_files else len(topfiles)
for index, _ in enumerate(xrange(totaln)):
size, path = topfiles[index]
print("\t%s: %s (%s)" %
(index + 1, path, utils.bytes_fmt(size)))
print("")
if args.topn_dirs:
topdirs = gather_topdirs()
if comm.rank == 0:
print("\nTop N Directory Report:\n")
totaln = args.topn_dirs if len(topdirs) > args.topn_dirs else len(
topdirs)
for index, _ in enumerate(xrange(totaln)):
size, path = topdirs[index]
print("\t{0:}: {1:} ({2:,} items)".format(
index + 1, path, size))
print("")
if args.gpfs_block_alloc:
gpfs_blocks = gather_gpfs_blocks()
gather_gpfs_dii()
if comm.rank == 0:
print("\nGPFS Block Alloc Report:\n")
print("\t{0:<15}{1:<4}".format("inode size:", args.inodesz))
print("\t{0:<25}{1:>15,}".format("DII (data-in-inode) count:",
DII_COUNT))
print("\tSubblocks: %s\n" % gpfs_blocks)
fmt_msg = "\tBlocksize: {0:<6} Estimated Space: {1:<20s} Efficiency: {2:>6.2%}"
for idx, bsz in enumerate(G.gpfs_block_size):
gpfs_file_size = gpfs_blocks[idx] * G.gpfs_subs[idx]
if gpfs_file_size != 0:
print(
fmt_msg.format(bsz, bytes_fmt(gpfs_file_size),
total_file_size /
float(gpfs_file_size)))
else:
print(fmt_msg.format(bsz, bytes_fmt(gpfs_file_size), 0))
treewalk.cleanup()
circle.finalize()
if args.lustre_stripe and stripe_out:
os.close(stripe_out)
sp_workload = comm.gather(Tally.spcnt)
if comm.rank == 0:
print("Stripe workload total: %s, distribution: %s" %
(sum(sp_workload), sp_workload))
def main():
global comm, args
args = parse_and_bcast(comm, gen_parser)
try:
G.src = utils.check_src(args.path)
except ValueError as e:
err_and_exit("Error: %s not accessible" % e)
G.use_store = args.use_store
G.loglevel = args.loglevel
hosts_cnt = tally_hosts()
if comm.rank == 0:
print("Running Parameters:\n")
print("\t{:<20}{:<20}".format("FWALK version:", __version__))
print("\t{:<20}{:<20}".format("Num of hosts:", hosts_cnt))
print("\t{:<20}{:<20}".format("Num of processes:",
MPI.COMM_WORLD.Get_size()))
print("\t{:<20}{:<20}".format("Root path:", utils.choplist(G.src)))
circle = Circle()
treewalk = FWalk(circle, G.src)
circle.begin(treewalk)
if G.use_store:
treewalk.flushdb()
if args.stats:
hist = global_histogram(treewalk)
total = hist.sum()
bucket_scale = 0.5
if comm.rank == 0:
print("\nFileset histograms:\n")
for idx, rightbound in enumerate(bins[1:]):
percent = 100 * hist[idx] / float(total)
star_count = int(bucket_scale * percent)
print("\t{:<3}{:<15}{:<8}{:<8}{:<50}".format(
"< ", utils.bytes_fmt(rightbound), hist[idx],
"%0.2f%%" % percent, '∎' * star_count))
if args.stats:
treewalk.flist.sort(lambda f1, f2: cmp(f1.st_size, f2.st_size),
reverse=True)
globaltops = comm.gather(treewalk.flist[:args.top])
if comm.rank == 0:
globaltops = [item for sublist in globaltops for item in sublist]
globaltops.sort(lambda f1, f2: cmp(f1.st_size, f2.st_size),
reverse=True)
if len(globaltops) < args.top:
args.top = len(globaltops)
print("\nStats, top %s files\n" % args.top)
for i in xrange(args.top):
print("\t{:15}{:<30}".format(
utils.bytes_fmt(globaltops[i].st_size),
globaltops[i].path))
treewalk.epilogue()
treewalk.cleanup()
circle.finalize()
def main():
global comm, args
fpipe.listen()
args = parse_and_bcast(comm, gen_parser)
try:
G.src = utils.check_src2(args.path)
except ValueError as e:
err_and_exit("Error: %s not accessible" % e)
G.loglevel = args.loglevel
hosts_cnt = tally_hosts()
if args.exclude:
process_exclude_file()
if comm.rank == 0:
print("Running Parameters:\n")
print("\t{:<20}{:<20}".format("fprof version:", __version__))
print("\t{:<20}{:<20}".format("Num of hosts:", hosts_cnt))
print("\t{:<20}{:<20}".format("Num of processes:", MPI.COMM_WORLD.Get_size()))
print("\t{:<20}{:<20}".format("Root path:", G.src))
if args.exclude:
print("\nExclusions:\n")
for ele in EXCLUDE:
print("\t %s" % ele)
circle = Circle()
if args.perprocess:
circle.report_enabled = True
else:
circle.reduce_enabled = True
treewalk = ProfileWalk(circle, G.src, perfile=args.perfile)
circle.begin(treewalk)
# we need the total file size to calculate GPFS efficiency
total_file_size = treewalk.epilogue()
msg1, msg2 = gen_histogram(total_file_size)
if comm.rank == 0:
sendto_syslog("fprof.filecount.hist", msg1)
sendto_syslog("fprof.fsize_perc.hist", msg2)
if args.top:
topfiles = gather_topfiles()
if comm.rank == 0:
print("\nTop File Report:\n")
# edge case: not enough files (< args.top)
totaln = args.top if len(topfiles) > args.top else len(topfiles)
for index, _ in enumerate(xrange(totaln)):
size, path = topfiles[index]
print("\t%s: %s (%s)" % (index + 1,
path,
utils.bytes_fmt(size)))
print("")
if args.gpfs_block_alloc:
gpfs_blocks = gather_gpfs_blocks()
if comm.rank == 0:
print("\nGPFS Block Alloc Report:\n")
print("\tinode size: %s" % args.inodesz)
print("\tDII (data-in-inode) count: %s" % DII_COUNT)
print("\tSubblocks: %s\n" % gpfs_blocks)
for idx, bsz in enumerate(G.gpfs_block_size):
gpfs_file_size = gpfs_blocks[idx] * G.gpfs_subs[idx]
fmt_msg = "\tBlocksize: {:<6} Estimated Space: {:<20s} Efficiency: {:>6.2%}"
if gpfs_file_size != 0:
print(fmt_msg.format(bsz, bytes_fmt(gpfs_file_size), total_file_size/float(gpfs_file_size)))
else:
print(fmt_msg.format(bsz, bytes_fmt(gpfs_file_size), 0))
treewalk.cleanup()
circle.finalize()
