def get_metric_by_function(inpfilename): ''' get depth level for each function in the call tree ''' print(f"Processing {inpfilename}...") call_tree = ct.get_call_tree(inpfilename) func_node = next(node for node in ct.iterate_on_call_tree(call_tree) if node.fname == funcname) children_info = [ node.fname + "," + str(node.cnode_id) for node in ct.iterate_on_call_tree(func_node, 1) ] ##### # output_i = mg.process_cubex(inpfilename, exclusive=exclincl) # We convert the Cnode IDs to short callpaths in the dataframe. df_i = ic.convert_index(output_i.df, output_i.ctree_df, target='Short Callpath') res_df = df_i.loc[children_info] res = res_df.reset_index()[[ 'Short Callpath', 'Thread ID', metric ]].groupby('Short Callpath').sum().sort_values([metric], ascending=False)[metric] res = res.head(11 if len(res) > 11 else len(res)) return res
def test_convert_index(): input_files = glob(f"{tu.SCALASCA_OUTPUT}/*/profile.cubex") output = mg.process_multi(input_files) common = output.common assert common.index.names == ['Cnode ID', 'Thread ID'] assert common.columns.names == ['run', 'metric'] noncommon = output.noncommon assert noncommon.index.names == ['Cnode ID', 'Thread ID'] assert noncommon.columns.names == ['metric'] tree_df = output.ctree_df # Checkin all conversions in cicrle for target in ['Full Callpath', # one way... 'Short Callpath', 'Cnode ID', 'Short Callpath', # .. and back. 'Full Callpath', 'Cnode ID' ]: print(f"Target : {target}") # Common metrics common = ic.convert_index(common,tree_df,target) print(common.index.names ) print(common.columns.names ) assert common.index.names == [target, 'Thread ID'] assert common.columns.names == ['run', 'metric'] print("Column names are as expected for the 'common' dataframe.") # Non common metrics noncommon = ic.convert_index(noncommon,tree_df,target) print(noncommon.index.names ) print(noncommon.columns.names) assert noncommon.index.names == [target, 'Thread ID'] assert noncommon.columns.names == ['metric'] print("Column names are as expected for the 'non-common' dataframe.")
def get_metric_by_level(inpfilename): """ get depth level for each function in the call tree """ print(f"Processing {inpfilename}") # This gives us a number of outputs # (see https://cupybelib.readthedocs.io/en/latest/merger.html) output_i = mg.process_cubex(inpfilename, exclusive=exclincl) parent_series = output_i.ctree_df.set_index( "Cnode ID").loc[:, "Parent Cnode ID"] # get depth level for each function in the call tree levels = ct.get_level(parent_series) # We convert the Cnode IDs to short callpaths in the dataframe. df_i = ic.convert_index(output_i.df, output_i.ctree_df, target="Short Callpath") # extract the data res = (df_i.reset_index()[[ "Short Callpath", "Thread ID", metric ]].groupby("Short Callpath").sum().sort_values([metric])[metric]) res_df = pd.DataFrame(res) time = res_df.reset_index()["time"] fname = res_df.reset_index()["Short Callpath"].str.extract( r"(\w+),([0-9]+)")[0] cnode_id = (res_df.reset_index()["Short Callpath"].str.extract( r"(\w+),([0-9]+)")[1].astype(int)) combined = pd.merge( left=(pd.concat([time, fname, cnode_id], axis="columns").rename( { "time": "time", 0: "fname", 1: "Cnode ID" }, axis="columns")), right=levels.reset_index().rename({0: "level"}, axis="columns"), on="Cnode ID", ) # to extract functions called only at the 3rd level time_data = combined[combined["level"] == 2].sort_values( by=["level", "time"], ascending=False) return time_data
def convert_df_to_inclusive(df_convertible, call_tree, tree_df=None): """ Converts a DataFrame from exclusive to inclusive. A level named ``Cnode ID``, ``Full Callpath`` or ``Short Callpath`` must be in the index. Parameters ---------- df_convertible : pandas.DataFrame A DataFrame containing only metrics that can be converted safely from exclusive to inclusive. call_tree: CallTreeNode A recursive representation of the call tree. tree_df : pandas.DataFrame or None In case ``df_convertible`` is not indexed by ``Cnode ID``, a dataframe that can be used to retrieve the ``Cnode ID`` from the index of ``df_convertible``. Returns ------- res : DataFrame A DataFrame """ old_index_name = ic.find_index_col(df_convertible) # dfcr = df_convertible_reindexed dfcr = ic.convert_index(df_convertible, tree_df, target="Cnode ID") levels_to_unstack = [ name for name in df_convertible.index.names if name != "Cnode ID" ] df_transposed = df_convertible.unstack(levels_to_unstack) def aggregate(root): value = df_transposed.loc[root.cnode_id, :] for child in root.children: value += aggregate(child) return value names = df_transposed.columns.names return (pd.concat( objs=[aggregate(n) for n in ct.iterate_on_call_tree(call_tree)], keys=[n.cnode_id for n in ct.iterate_on_call_tree(call_tree)], ).rename_axis(mapper=["Cnode ID"] + names, axis="index").unstack(names).pipe( ic.convert_index, tree_df, old_index_name).stack(levels_to_unstack))
# Reading, parsing and loading data in the cubex file output_i = mg.process_cubex(inpfilename, exclusive=exclincl) call_tree = output_i.ctree func_node = next(node for node in ct.iterate_on_call_tree(call_tree) if node.fname == rootfuncname) children_info = [ node.fname + "," + str(node.cnode_id) for node in ct.iterate_on_call_tree(func_node, 1) ] # We convert the Cnode IDs to short callpaths in the dataframe. df_i = ic.convert_index(output_i.df, output_i.ctree_df, target="Short Callpath") res_df = df_i.loc[children_info] res = (res_df.reset_index()[[ "Short Callpath", "Thread ID", metric ]].groupby("Short Callpath").sum().sort_values([metric], ascending=False)[metric]) res = res.head(11 if len(res) > 11 else len(res)).tail( 10 if len(res) > 10 else len(res) - 1) res.plot(kind="bar") plt.xlabel("Function name: " + rootfuncname,
# (see https://cupybe.readthedocs.io/en/latest/merger.html) if len(argv) == 1: input_file = "../test_data/profile.cubex" else: input_file = argv[1] print("Opening file", input_file) output_i = mg.process_cubex(input_file, exclusive=False) df_i = output_i.df # Dataframes with the metrics tree = output_i.ctree_df # Dataframe containing info on the calltree # We convert the Cnode IDs to short callpaths in the dataframe. df_i = ic.convert_index(df_i, tree, target="Short Callpath") # We calculate the mean of the time times_mean = df_i.time.groupby("Short Callpath").mean() # We do a merge (=join) on the tree dataframe to find the parent-child relation parent_child = ( pd.merge( left=tree, right=tree, left_on="Cnode ID", right_on="Parent Cnode ID", suffixes=("-Parent", ""), ) # we select the two columns we're interested in .loc[:, ["Short Callpath", "Short Callpath-Parent"]].set_index(
inoutput, exoutput, files_check = data if files_check != files: raise Exception("Data does not contain the expected files.") except: inoutput = mg.process_multi(files, exclusive=False) exoutput = mg.process_multi(files, exclusive=True) with open(pickle_archive, "wb") as f: data = pickle.dump((inoutput, exoutput, files), f) # renaming variables for convenience inctree_df = inoutput.ctree_df inctree = inoutput.ctree inncmetrics = inoutput.ncmetrics # index from Cnode ID to short callpath for readability incommon = ic.convert_index(inoutput.common, inctree_df, target="Short Callpath") innoncommon = ic.convert_index( inoutput.noncommon, inctree_df, target="Short Callpath" ) # renaming variables for convenience exctree_df = exoutput.ctree_df exctree = exoutput.ctree exncmetrics = exoutput.ncmetrics # index from Cnode ID to short callpath for readability excommon = ic.convert_index(exoutput.common, exctree_df, target="Short Callpath") exnoncommon = ic.convert_index( exoutput.noncommon, exctree_df, target="Short Callpath" )
assert noncommon.columns.names == ['metric'] tree_df = output.ctree_df # Checkin all conversions in cicrle for target in ['Full Callpath', # one way... 'Short Callpath', 'Cnode ID', 'Short Callpath', # .. and back. 'Full Callpath', 'Cnode ID' ]: print(f"Target : {target}") # Common metrics common = ic.convert_index(common,tree_df,target) print(common.index.names ) print(common.columns.names ) assert common.index.names == [target, 'Thread ID'] assert common.columns.names == ['run', 'metric'] print("Column names are as expected for the 'common' dataframe.") # Non common metrics noncommon = ic.convert_index(noncommon,tree_df,target) print(noncommon.index.names ) print(noncommon.columns.names) assert noncommon.index.names == [target, 'Thread ID']