def _get_metrics_str_for_product(self, metric_keys: Set[DagKey]) -> str: """Builds the Metrics string for the product markdown file. Creates a table of necessary metric types and whether a state calculates those metrics""" metrics_header = ( "##METRICS\n_All metrics required to support this product and" " whether or not each state regularly calculates the metric._" "\n\n** DISCLAIMER **\nThe presence of all required metrics" " for a state does not guarantee that this product is ready to" " launch in that state.\n\n") if not metric_keys: return (metrics_header + "*This product does not rely on Dataflow metrics.*\n") state_codes = sorted(self._get_dataflow_pipeline_enabled_states(), key=lambda code: code.value) headers = ["**Metric**"] + [ f"**{state_code.value}**" for state_code in state_codes ] table_matrix = [[ f"[{DATAFLOW_TABLES_TO_METRIC_TYPES[metric_key.table_id].value}](../../metrics/{self.generic_types_by_metric_name[metric_key.table_id].lower()}/{metric_key.table_id}.md)" ] + [ "X" if DATAFLOW_TABLES_TO_METRIC_TYPES[metric_key.table_id].value in [ metric.name for metric in self.metric_calculations_by_state[ str(state_code.get_state())] ] else "" for state_code in state_codes ] for metric_key in sorted(metric_keys, key=lambda dag_key: dag_key.table_id)] writer = MarkdownTableWriter(headers=headers, value_matrix=table_matrix, margin=0) return metrics_header + writer.dumps()
def event_report(self): # TODO: Use pandas dataframe's feature _all_reports = '' all_events = collections.OrderedDict( sorted(self._container.events.items(), key=lambda t: len(t[0]))) for attr, event_list in all_events.items(): writer = MarkdownTableWriter() writer.title = attr if len(event_list) == 0: continue else: keys = list(event_list[0].keys()) values = [] writer.headers = keys for event in event_list: line = [] for k in keys: event_val = event[k] if isinstance(event_val, (np.ndarray, np.generic)): event_val = event_val.tolist() if isinstance(event_val, int): line.append(event_val) elif isinstance(event_val, float): line.append(event_val) elif isinstance(event_val, list): if all(isinstance(n, int) for n in event_val): line.append(', '.join(str(v) for v in event_val)) elif all(isinstance(n, float) for n in event_val): line.append(', '.join('{0:.5f}'.format(v) for v in event_val)) values.append(line) writer.value_matrix = values _all_reports += writer.dumps() return _all_reports
def _get_fields(fields: List[sqlalchemy.Column]) -> str: """Returns a table of the entity's fields and their corresponding descriptions.""" if fields is None: return "No Fields" if not fields: return "<No columns>" table_matrix = [] for field in fields: if field.comment is None: raise ValueError( f"Every entity field must have an associated comment. " f"Field {field.name} has no comment.") field_values = [ field.name, field.comment, f"ENUM: <br />{'<br />'.join([f'{e}' for e in field.type.enums])}" if hasattr(field.type, "enums") else field.type.python_type.__name__.upper(), ] table_matrix.append(field_values) writer = MarkdownTableWriter( headers=[ "Entity Field", "Entity Description", "Entity Type", ], value_matrix=table_matrix, margin=0, ) return writer.dumps()
def game_after_survival_training_no_interrupt(repeat_times): move_counts = 12000 data = [['Lives Consumed', 'Moves', 'Ratio']] for count in range(repeat_times): mp, sc, lil = curses.wrapper(draw_menu_after_survival_no_interrupt, move_counts) ratioo = float(lil) / float(mp) data.append([lil, mp, ratioo]) data_np = pd.DataFrame(data) data_np_lack = pd.DataFrame(data[1:]) data_np = data_np.append(data_np_lack.mean(axis=0), ignore_index=True) data_np = np.transpose(data_np) data_np_list = np.array(data_np).tolist() writer = MarkdownTableWriter() writer.table_name = "collecting data after survival training" writer.headers = [" "] + \ [str(i + 1) for i in range(repeat_times)] + ["average"] writer.value_matrix = data_np_list table_output = writer.dumps() logger = logging.getLogger('no_interrupt') no_fire_stat = 'no_fires status: ' + \ str(nf_global_survival_training) + '\n' enemy_freq_stat = 'enemy freq status: ' + \ str(enemy_freq_sur_train) + '\n' ot_str = no_fire_stat + enemy_freq_stat + \ 'After survival training: \n' + table_output logger.info(ot_str)
def print_starred_info(starred_info_set, repo_depth_map, verbosity): records = [] for info in sorted(starred_info_set): record = [ info.pypi_pkg_name, info.github_repo_id, _star_status_map[info.star_status], info.is_owned if info.star_status in [StarStatus.STARRED, StarStatus.NOT_STARRED] else _NA, repo_depth_map[info.pypi_pkg_name.lower()], info.url, ] records.append(record) writer = MarkdownTableWriter() writer.headers = ["Package", "Repository", "Starred", "Owner"] if verbosity is not None: if verbosity >= 1: writer.headers += ["Depth"] if verbosity >= 2: writer.headers += ["URL"] writer.value_matrix = sorted(records, key=itemgetter(4, 0)) # sorted by depth writer.margin = 1 writer.register_trans_func(bool_to_checkmark) writer.set_style("Starred", Style(align="center")) writer.set_style("Owner", Style(align="center")) pager(writer.dumps())
def generate_md_table(data, headers): writer = MarkdownTableWriter() writer.headers = headers writer.column_styles = [Style(align="center", font_weight="bold") ] * len(headers) writer.value_matrix = data return writer.dumps()
def intent_table(): writer = MarkdownTableWriter() writer.table_name = "Intent Cross-Validation Results (5 folds)" with open('results/intent_report.json', 'r') as f: data = json.loads(f.read()) cols = ["support", "f1-score", "confused_with"] writer.headers = ["class"] + cols classes = list(data.keys()) classes.remove('accuracy') classes.sort(key=lambda x: data[x]['support'], reverse=True) def format_cell(data, c, k): if not data[c].get(k): return "N/A" if k == "confused_with": return ", ".join([f"{k}({v})" for k, v in data[c][k].items()]) else: return data[c][k] writer.value_matrix = [[c] + [format_cell(data, c, k) for k in cols] for c in classes] return writer.dumps()
def entity_table(): writer = MarkdownTableWriter() writer.table_name = "Entity Cross-Validation Results (5 folds)" with open('results/DIETClassifier_report.json', 'r') as f: data = json.loads(f.read()) cols = ["support", "f1-score", "precision", "recall"] writer.headers = ["entity"] + cols classes = list(data.keys()) classes.sort(key = lambda x: data[x]['support'], reverse=True) def format_cell(data, c, k): if not data[c].get(k): return "N/A" else: return data[c][k] writer.value_matrix = [ [c] + [format_cell(data, c, k) for k in cols] for c in classes ] return writer.dumps()
def make_table(result_dict): """Generate table of results.""" from pytablewriter import MarkdownTableWriter, LatexTableWriter md_writer = MarkdownTableWriter() latex_writer = LatexTableWriter() md_writer.headers = ["Task", "Version", "Metric", "Value", "", "Stderr"] latex_writer.headers = ["Task", "Version", "Metric", "Value", "", "Stderr"] values = [] for k, dic in result_dict["results"].items(): version = result_dict["versions"][k] for m, v in dic.items(): if m.endswith("_stderr"): continue if m + "_stderr" in dic: se = dic[m + "_stderr"] values.append([k, version, m, "%.4f" % v, "±", "%.4f" % se]) else: values.append([k, version, m, "%.4f" % v, "", ""]) k = "" version = "" md_writer.value_matrix = values latex_writer.value_matrix = values # todo: make latex table look good # print(latex_writer.dumps()) return md_writer.dumps()
def _repr_markdown_(self): """Outuput for Markdown.""" is_supergroup = self.__contains_groups() rsl = [] rsl += self._row_values(is_supergroup) if is_supergroup: total = str(self.total()) rsl.append(["", "", "", ""]) rsl.append([ "", f"**_Total {self.name}_**", f"**_{total}_**", "", ]) writer = MarkdownTableWriter( headers=[ "Pos.", "Bezeichnung", "Betrag", "Anmerkung", ], column_styles=[ TableStyle(align="right"), TableStyle(align="left"), TableStyle(align="right"), TableStyle(align="left"), ], value_matrix=rsl, margin=1, ) return writer.dumps()
def _generate_docs_for_raw_config( raw_file_config: DirectIngestRawFileConfig, ) -> str: """Generates documentation for the given raw file config and returns it as a string.""" file_columns = sorted(raw_file_config.columns, key=lambda col: col.name) primary_key_columns = [ col.upper() for col in raw_file_config.primary_key_cols ] def _is_primary_key(column: str) -> str: return "YES" if column.upper() in primary_key_columns else "" documentation = ( f"## {raw_file_config.file_tag}\n\n{raw_file_config.file_description}\n\n" ) table_matrix = [[ column.name, column.description or "<No documentation>", _is_primary_key(column.name), ] for column in file_columns] writer = MarkdownTableWriter( headers=["Column", "Column Description", "Part of Primary Key?"], value_matrix=table_matrix, margin=1, ) documentation += writer.dumps() return documentation
def _generate_raw_file_table( self, config_paths_by_file_tag: Dict[str, str], file_tags_with_raw_file_configs: List[str], views_by_raw_file: Dict[str, List[str]], ) -> str: """Generates a Markdown-formatted table of contents to be included in a raw file specification.""" table_matrix = [[ (f"[{file_tag}](#{file_tag})" if file_tag in file_tags_with_raw_file_configs else f"{file_tag}"), ",<br />".join(views_by_raw_file[file_tag]), self._get_last_updated(config_paths_by_file_tag[file_tag]), self._get_updated_by(config_paths_by_file_tag[file_tag]), ] for file_tag in sorted(config_paths_by_file_tag)] writer = MarkdownTableWriter( headers=[ "**Table**", "**Referencing Views**", "**Last Updated**", "**Updated By**", ], value_matrix=table_matrix, margin=1, ) return writer.dumps()
def create_section(section_list, name): counter = 0 section_str = '## [↑](#-table-of-contents) {}\n\n'.format(name) # header = ['Task', 'Dataset', 'SOTA', 'Metric', 'SOTA Acc', 'Our Acc', 'Our Model', '📝', 'Notebook'] # header = ['Task', 'Dataset', 'SOTA', 'Metric', 'SOTA Acc', 'Our Acc', '📝', 'Notebook'] header = [ 'Task', 'Dataset', 'SOTA', 'SOTA Acc', 'Our Acc', '📝', 'Notebook' ] values_matrix = [] for row in section_list: values_matrix.append([ row[0], '[{}]({})'.format(row[1], row[2]) if row[2] else row[1], '[{}]({})'.format(row[3], row[4]) if row[4] else row[3], # row[5], row[6], row[7], # row[8], '[📝]({} "Article")'.format(row[9]), '[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)]' '(https://colab.research.google.com/github/eugenesiow/practical-ml/blob/master/notebooks/{}' ' "Open in Colab")'.format(row[10]) ]) counter += 1 writer = MarkdownTableWriter( headers=header, value_matrix=values_matrix, ) section_str += writer.dumps() section_str += '\n\n' return section_str, counter
def mk_table(datasets): values = [] total_weight = sum([x[1] * x[0].size() for x in datasets]) train_chars = 1.2e12 for dataset, weight in datasets: size = dataset.size() relative_weight = size * weight / total_weight values.append([ dataset.name(), size, '{:.2%}'.format(relative_weight), train_chars / size * relative_weight, humanbytes(size / dataset.num_docs()) ]) values.sort(key=lambda x: -x[1]) values.append([ '**Total**', sum([x[1] for x in values]), "", "", humanbytes( sum([x[1] for x in values]) / sum(x[0].num_docs() for x in datasets)) ]) values = [[x[0], humanbytes(x[1]), x[2], x[3], x[4]] for x in values] writer = MarkdownTableWriter() writer.table_name = "The Pile™" writer.headers = [ "Component", "Size", "Weight", "Epochs (@1.2TB)", "Mean Document Size" ] writer.value_matrix = values return writer.dumps()
def overall_report(self): writer = MarkdownTableWriter() writer.title = 'overall' flattens = self._container.flatten writer.headers = ['metric', 'value'] values = [] for k, v in flattens.items(): values.append([k, v]) writer.value_matrix = values return writer.dumps()
def print_nodes_mark_down(node_type, nodes): writer = MarkdownTableWriter() writer.headers = [ 'ID', 'Link to GitHub', "Created date", "Status", 'Title', 'Assigned to' ] writer.value_matrix = filter_nodes_data(node_type, nodes) print(writer.dumps())
def _generate_docs_for_raw_config( raw_file_config: DirectIngestRawFileConfig, ) -> str: """Generates documentation for the given raw file config and returns it as a string.""" primary_key_columns = [col.upper() for col in raw_file_config.primary_key_cols] def _is_primary_key(column: str) -> str: return "YES" if column.upper() in primary_key_columns else "" def _get_enum_bullets(known_values: Optional[List[ColumnEnumValueInfo]]) -> str: if known_values is None: return "N/A" if not known_values: return "<No documentation>" list_contents = ", <br/>".join( [ f"<b>{enum.value}: </b> {enum.description if enum.description else 'Unknown'}" for enum in known_values ] ) return list_contents documentation = ( f"## {raw_file_config.file_tag}\n\n{raw_file_config.file_description}\n\n" ) table_matrix = [ [ column.name, column.description or "<No documentation>", _is_primary_key(column.name), _get_enum_bullets(column.known_values), ] for column in raw_file_config.columns ] writer = MarkdownTableWriter( headers=[ "Column", "Column Description", "Part of Primary Key?", "Distinct Values", ], value_matrix=table_matrix, # Margin values other than 0 have nondeterministic spacing. Do not change. margin=0, ) documentation += writer.dumps() return documentation
def __create_table(self, platform: str, username: str, changed_datas: Dict[str, Union[str, int]]) -> str: table = MarkdownTableWriter() table.table_name = f"{platform.title()}: {username}" table.headers = ["Metric", "Value", "Change"] table.value_matrix = [[ key, value["current"], value.get("change", "n/a") if value.get("change", False) else value["last"], ] for key, value in changed_datas.items()] table.margin = 1 return table.dumps()
def main(): random.seed(42) np.random.seed(42) lm = models.get_model(model).create_from_arg_string(model_args) if limit: print( "WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT." ) if not no_cache: lm = base.CachingLM( lm, 'lm_cache/' + model + '_' + model_args.replace('=', '-').replace(',', '_') + '.db') task_dict = tasks.get_task_dict([task]) for desc in fewshot_descriptions: custom_task_dict = { k: CustomDescTask(v, desc) for k, v in task_dict.items() } results = evaluator.evaluate(lm, custom_task_dict, True, num_fewshot, limit) dumped = json.dumps(results, indent=2) print('Description:', desc) print(dumped) # MAKE TABLE from pytablewriter import MarkdownTableWriter writer = MarkdownTableWriter() writer.headers = ["Task", "Metric", "Value"] values = [] for k, dic in results.items(): for m, v in dic.items(): values.append([k, m, '%.4f' % v]) k = "" writer.value_matrix = values print(writer.dumps())
def main(): args = parse_args() random.seed(args.seed) np.random.seed(args.seed) lm = models.get_model(args.model).create_from_arg_string(args.model_args) if args.limit: print( "WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT." ) if not args.no_cache: lm = base.CachingLM( lm, 'lm_cache/' + args.model + '_' + args.model_args.replace('=', '-').replace(',', '_') + '.db') if args.tasks == "all_tasks": task_names = tasks.ALL_TASKS else: task_names = args.tasks.split(",") task_dict = tasks.get_task_dict(task_names) results = evaluator.evaluate(lm, task_dict, args.provide_description, args.num_fewshot, args.limit) dumped = json.dumps(results, indent=2) print(dumped) if args.output_path: with open(args.output_path, "w") as f: f.write(dumped) # MAKE TABLE from pytablewriter import MarkdownTableWriter writer = MarkdownTableWriter() writer.headers = ["Task", "Metric", "Value"] values = [] for k, dic in results.items(): for m, v in dic.items(): values.append([k, m, '%.4f' % v]) k = "" writer.value_matrix = values print(writer.dumps())
def params_to_markdown(params_str): """ this converts the hyper-params into MarkDown text and we put it on Tensorboard subsequently """ params = process_text(text=params_str) writer = MarkdownTableWriter() writer.table_name = "Hyper-parameters" writer.headers = ["Item", "Value"] items = [] for key, value in params.items(): items.append([str(key), str(value)]) writer.value_matrix = items writer.margin = 1 # add a whitespace for both sides of each cell return writer.dumps()
def get_table_string(authors, dates, titles, topics): """ Returns an object ready to write a table :param authors: :param s: :param titles: :param topics: :return: """ writer = MarkdownTableWriter( table_name="Completed Articles", headers=["Author", "Title", "Year", "Topics"], value_matrix=[[a, tit, d, ", ".join(top)] for a, d, tit, top in zip(authors, dates, titles, topics) ], margin=1, # add a whitespace for both sides of each cell ) return writer.dumps()
def markdown_table(self, *columns): """Export a markdown-formatted version of the table Parameters ---------- *columns Names of columns to include in the exported table """ writer = MarkdownTableWriter() writer.header_list = ['Name'] + ([self.header[col] for col in columns] if columns else self.header) writer.value_matrix = [ [name] + ([self.dict[name][col] for col in columns] if columns else self.dict[name]) for name in sorted(self.dict.keys()) ] return writer.dumps()
def main(): lm = DryrunLM() task_list = "arc_challenge,arc_easy,boolq,cola,copa,headqa,hellaswag,lambada,logiqa,mathqa,mc_taco,mrpc,multirc,openbookqa,piqa,prost,pubmedqa,qnli,qqp,race,record,rte,sciq,sst,triviaqa,webqs,wic,wikitext,winogrande,wnli,wsc" values = [] for taskname in task_list.split(","): lm.tokencost = 0 evaluator.evaluate( lm=lm, task_dict={taskname: tasks.get_task(taskname)()}, num_fewshot=0, limit=None, bootstrap_iters=10, description_dict=None, ) print(taskname, lm.tokencost) values.append([ taskname, lm.tokencost, lm.tokencost / 1000 * 0.0008, lm.tokencost / 1000 * 0.0012, lm.tokencost / 1000 * 0.006, lm.tokencost / 1000 * 0.06, ]) from pytablewriter import MarkdownTableWriter writer = MarkdownTableWriter() writer.headers = ["Task", "Tokens", "Ada", "Babbage", "Curie", "Davinci"] values.sort(key=lambda x: -x[1]) totcost = sum([x[1] for x in values]) values.append([ "**Total**", totcost, totcost / 1000 * 0.0008, totcost / 1000 * 0.0012, totcost / 1000 * 0.006, totcost / 1000 * 0.06, ]) writer.value_matrix = values print(writer.dumps())
def mk_table(datasets, train_chars, print_latex=False): values = [] total_weight = sum([x[1] * x[0].size() for x in datasets]) for dataset, weight in datasets: size = dataset.size() relative_weight = size * weight / total_weight values.append([dataset.name(), size, '{:.2%}'.format(relative_weight), '{:.4f}'.format(train_chars / size * relative_weight), size * weight, humanbytes(size / dataset.num_docs(), 'KiB')]) values.sort(key=lambda x: -x[4]) values.append(['**Total**', "", "", "", sum([x[4] for x in values]), humanbytes(sum([x[1] for x in values]) / sum(x[0].num_docs() for x in datasets), 'KiB')]) values = [[x[0], humanbytes(x[1], 'GiB') if x[1] else "", x[2], x[3], humanbytes(x[4], 'GiB'), x[5]] for x in values] writer = MarkdownTableWriter() writer.table_name = "The Pile™" writer.headers = ["Component", "Raw Size", "Weight", "Epochs", "Effective Size", "Mean Document Size"] writer.value_matrix = values if print_latex: rows = [] for row in values[:-1]: rows.append(" " + " & ".join(map(lambda x: str(x).replace('%', r'\%'), row)) + r" \\") totalrow = " & ".join(map(lambda x: r'\textbf{%s}' % str(x).replace('%', r'\%') if x else "", values[-1][1:])) + r" \\" latex = r""" \begin{table*}[t!] \centering \begin{tabular}{l r r r r r} \toprule \textbf{Component} & \textbf{Raw Size} & \textbf{Weight} & \textbf{Copies} & \textbf{Effective Size} & \textbf{Mean Document Size} \\ \midrule """ + "\n".join(rows) + r""" \midrule \textbf{The Pile} & """ + totalrow + r""" \bottomrule \end{tabular} \caption{Overview of datasets in \textit{The Pile} before deduplication. The Pile is distributed with a predefined up/down-sampling of the different constituent datasets.} \label{table:pile_overview} \end{table*} """ print(latex) return writer.dumps()
def generate_markdown_table(input_params_yaml, output_markdown_file_path): try: with open(input_params_yaml, "r") as input: parameters = yaml.safe_load(input)["parameters"] parameter_fields = ["name", "description", "default"] writer = MarkdownTableWriter() writer.table_name = "Parameters" writer.styles = [ Style(align="left", font_weight="bold"), Style(align="left"), Style(align="left"), ] writer.headers = [field.capitalize() for field in parameter_fields] writer.margin = 1 writer.value_matrix = [ list([parameter.get(field) for field in parameter_fields]) for parameter in parameters ] if output_markdown_file_path: try: with open(output_markdown_file_path, "w") as output: writer.stream = output writer.write_table() except Exception as e: logging.error( f"Failed to output Markdown to {output_markdown_file_path}", e, ) return 1 else: print(writer.dumps()) return 0 except Exception as e: logging.error( f"Failed to generate Markdown from {input_params_yaml}", e ) return 1
def _get_metric_information(self, metric: Type[RecidivizMetric]) -> str: """Returns string contents for a metric markdown.""" metric_table_id = DATAFLOW_METRICS_TO_TABLES[metric] metric_type = DATAFLOW_TABLES_TO_METRIC_TYPES[metric_table_id].value state_infos_list = sorted( self.state_metric_calculations_by_metric[metric_type], key=lambda info: (info.name, info.month_count), ) headers = [ "**State**", "**Number of Months Calculated**", "**Calculation Frequency**", ] table_matrix = [[ f"[{state_info.name}](../../states/{self._normalize_string_for_path(state_info.name)}.md)", state_info.month_count if state_info.month_count else "N/A", state_info.frequency, ] for state_info in state_infos_list] writer = MarkdownTableWriter(headers=headers, value_matrix=table_matrix, margin=0) documentation = METRIC_DOCS_TEMPLATE.format( staging_link=BQ_LINK_TEMPLATE.format( project="recidiviz-staging", dataset_id="dataflow_metrics", table_id=metric_table_id, ), prod_link=BQ_LINK_TEMPLATE.format( project="recidiviz-123", dataset_id="dataflow_metrics", table_id=metric_table_id, ), metric_name=metric.__name__, description=metric.get_description(), metrics_cadence_table=writer.dumps(), metric_table_id=metric_table_id, ) return documentation
def _get_metrics_table_for_state(self, state_name: str) -> str: sorted_state_metric_calculations = self._get_sorted_state_metric_info() metric_names_to_tables = { metric.value: table for table, metric in DATAFLOW_TABLES_TO_METRIC_TYPES.items() } if state_name in sorted_state_metric_calculations: headers = [ "**Metric**", "**Number of Months Calculated**", "**Calculation Frequency**", ] table_matrix = [[ f"[{metric_info.name}](../metrics/{self.generic_types_by_metric_name[metric_names_to_tables[metric_info.name]].lower()}/{metric_names_to_tables[metric_info.name]}.md)", metric_info.month_count if metric_info.month_count else "N/A", metric_info.frequency, ] for metric_info in sorted_state_metric_calculations[state_name]] writer = MarkdownTableWriter(headers=headers, value_matrix=table_matrix, margin=0) return writer.dumps() return "_This state has no regularly calculated metrics._"
def pipeline_visualizer(num_stages, num_microbatches, include_all=False): stages = {} for stage_id in range(num_stages): steps = [i for i in TrainSchedule(micro_batches=num_microbatches, stages=num_stages - 1 , stage_id=stage_id).steps()] steps = expand(steps, include_all=include_all) stages[stage_id] = steps value_matrix = [v for k, v in stages.items()] headers = ['GPU ID'] + [str(i) for i in range(len(stages[0]))] value_matrix = [[f'GPU {i}'] + value_matrix[i] for i in range(len(value_matrix))] writer = MarkdownTableWriter( table_name=f"Pipe Schedule\n", headers=headers, value_matrix=value_matrix ) string = writer.dumps() all_steps = flatten(value_matrix) idle_time = len([i for i in all_steps if i is None]) print(all_steps) non_idle_time = len([i for i in all_steps if (i is not None and 'GPU' not in i)]) string += f'\nNum Devices: {num_stages}\nNum Microbatches: {num_microbatches} \n' \ f'Idle Time: {idle_time}\nNon Idle Time: {non_idle_time}' return string
def main(): lm = DryrunLM() values = [] for taskname in list(tasks.TASK_REGISTRY.keys()): lm.tokencost = 0 evaluator.evaluate(lm, {taskname: tasks.get_task(taskname)()}, False, 0, None) print(taskname, lm.tokencost) values.append([taskname, lm.tokencost, lm.tokencost / 1000 * 0.06]) from pytablewriter import MarkdownTableWriter writer = MarkdownTableWriter() writer.headers = ["Task", "Tokens", "Davinci Cost"] values.sort(key=lambda x: -x[1]) totcost = sum([x[1] for x in values]) values.append(["**Total**", totcost, totcost / 1000 * 0.06]) writer.value_matrix = values print(writer.dumps())