def what_are_we_waiting_for(command, sort_by='time', max_len=20, print_here=True): """ An easy way to show what is taking all the time when you run something. Taken from docs: https://docs.python.org/2/library/profile.html#module-cProfile :param command: A string python command :param sort_by: How to sort results. {'time', 'cumtime', 'calls', ...}. See https://docs.python.org/2/library/profile.html#pstats.Stats.sort_stats :param max_len: Maximum number of things to show in profile. :param print_here: Print the results here (instead of returning them). :return: A pstats.Stats object containing the profiling results. """ _, filepath = mkstemp() cProfile.run(command, filepath) p = pstats.Stats(filepath) os.remove(filepath) p.strip_dirs() p.sort_stats(sort_by) if print_here: print surround_with_header('Profile for "{}"'.format(command), width=100, char='=') p.print_stats(max_len) print '=' * 100 return p
def test_surround_with_header(): a = surround_with_header('abcd', width=40) assert len(a)==40 b = surround_with_header('abcde', width=40) assert len(b)==40 a = surround_with_header('abcd', width=41) assert len(a)==41 b = surround_with_header('abcde', width=41) assert len(b)==41 a = surround_with_header('abcd', width=2) assert len(a)==6
def what_are_we_waiting_for(command, sort_by ='time', max_len = 20, print_here = True): """ An easy way to show what is taking all the time when you run something. Taken from docs: https://docs.python.org/2/library/profile.html#module-cProfile :param command: A string python command :param sort_by: How to sort results. {'time', 'cumtime', 'calls', ...}. See https://docs.python.org/2/library/profile.html#pstats.Stats.sort_stats :param max_len: Maximum number of things to show in profile. :param print_here: Print the results here (instead of returning them). :return: A pstats.Stats object containing the profiling results. """ _, filepath = mkstemp() try: cProfile.run(command, filepath) finally: p = pstats.Stats(filepath) os.remove(filepath) p.strip_dirs() p.sort_stats(sort_by) if print_here: print(surround_with_header('Profile for "{}"'.format(command), width=100, char='=')) p.print_stats(max_len) print('='*100) return p
def launch(self): commands = { 'h': self.help, 'del': self.delete, 'cd': self.cd, 'sortby': self.sortby } redraw = True while True: if redraw: print(surround_with_header('Contents of {}'.format(self.dc.directory), width=100)) print(self._list_directory_contents()) redraw = True command = input('Enter Command, or h for help >>') cmd_and_args = command.split(' ') cmd, args = cmd_and_args[0], cmd_and_args[1:] if cmd=='': continue elif cmd=='q': break elif cmd not in commands: print('Command not found: {}'.format(cmd)) redraw=False else: commands[cmd](*args)
def get_record_full_string(record, show_info = True, show_logs = True, truncate_logs = None, show_result ='deep', show_exceptions=True, truncate_result = None, include_bottom_border = True, header_width=64): """ Get a human-readable string containing info about the experiment record. :param show_info: Show info about experiment (name, id, runtime, etc) :param show_logs: Show logs (True, False, or an integer character count to truncate logs at) :param show_result: Show the result. Options for display are: 'short': Print a one-liner (note: sometimes prints multiple lines) 'long': Directly call str 'deep': Use the deepstr function for a compact nested printout. :return: A string to print. """ assert show_result in (False, 'full', 'deep') full_info_string = surround_with_header(record.get_id(), width=header_width, char='=') + '\n' if show_info: full_info_string += '{}\n'.format(record.info.get_text()) if show_logs: log = record.get_log() if truncate_logs is not None and len(log)>truncate_logs: log = log[:truncate_logs-100] + '\n\n ... LOG TRUNCATED TO {} CHARACTERS ... \n\n'.format(truncate_logs) + log[-100:] full_info_string += section_with_header('Logs', log, width=header_width) error_trace = record.get_error_trace() if show_exceptions and error_trace is not None: full_info_string += section_with_header('Error Trace', record.get_error_trace(), width=header_width) if show_result: result_str = get_record_result_string(record, truncate_to=truncate_result, func=show_result) full_info_string += section_with_header('Result', result_str, width=header_width, bottom_char='=' if include_bottom_border else None) return full_info_string
def launch(self): commands = { 'h': self.help, 'del': self.delete, 'cd': self.cd, 'sortby': self.sortby } redraw = True while True: if redraw: print( surround_with_header('Contents of {}'.format( self.dc.directory), width=100)) print(self._list_directory_contents()) redraw = True command = input('Enter Command, or h for help >>') cmd_and_args = command.split(' ') cmd, args = cmd_and_args[0], cmd_and_args[1:] if cmd == '': continue elif cmd == 'q': break elif cmd not in commands: print('Command not found: {}'.format(cmd)) redraw = False else: commands[cmd](*args)
def get_record_full_string(record, show_info = True, show_logs = True, truncate_logs = None, show_result ='deep', show_exceptions=True, truncate_result = None, include_bottom_border = True, header_width=64, return_list = False): """ Get a human-readable string containing info about the experiment record. :param show_info: Show info about experiment (name, id, runtime, etc) :param show_logs: Show logs (True, False, or an integer character count to truncate logs at) :param show_result: Show the result. Options for display are: 'short': Print a one-liner (note: sometimes prints multiple lines) 'long': Directly call str 'deep': Use the deepstr function for a compact nested printout. :return: A string to print. """ parts = [surround_with_header(record.get_id(), width=header_width, char='=')] if show_info: parts.append(section_with_header('Info', record.info.get_text(), width=header_width)) if show_logs: log = record.get_log() if truncate_logs is not None and len(log)>truncate_logs: log = log[:truncate_logs-100] + '\n\n ... LOG TRUNCATED TO {} CHARACTERS ... \n\n'.format(truncate_logs) + log[-100:] # return section_with_header('Logs', log, width=header_width) parts.append(section_with_header('Logs', log, width=header_width)) if show_exceptions: error_trace = record.get_error_trace() error_trace_text = '' if error_trace is None else section_with_header('Error Trace', record.get_error_trace(), width=header_width) parts.append(error_trace_text) if show_result: assert show_result in (False, 'full', 'deep') result_str = get_record_result_string(record, truncate_to=truncate_result, func=show_result) parts.append(section_with_header('Result', result_str, width=header_width)) if return_list: return parts else: return '\n'.join(parts)
def get_experiment_list_str(self, exp_record_dict): headers = { 'full': [ ExpRecordDisplayFields.RUNS, ExpRecordDisplayFields.DURATION, ExpRecordDisplayFields.STATUS, ExpRecordDisplayFields.ARGS_CHANGED, ExpRecordDisplayFields.RESULT_STR, ExpRecordDisplayFields.NOTES ], 'results': [ExpRecordDisplayFields.RESULT_STR] }[self.view_mode] if self.remove_prefix: deprefixed_ids = deprefix_experiment_ids(exp_record_dict.keys()) exp_record_dict = OrderedDict( (k, v) for k, v in zip(deprefixed_ids, exp_record_dict.values())) row_func = _get_record_rows_cached if self.cache_result_string else _get_record_rows header_names = [h.value for h in headers] def remove_notes_if_no_notes(_record_rows): notes_column_index = headers.index( ExpRecordDisplayFields.NOTES ) if ExpRecordDisplayFields.NOTES in headers else None # Remove the notes column if there are no notes! if notes_column_index is not None and all( row[notes_column_index] == '' for row in _record_rows): for row in _record_rows: del row[notes_column_index] if self.display_format == 'nested': # New Display mode if self.show_args: _, argdiff = separate_common_items([ load_experiment(ex).get_args().items() for ex in exp_record_dict ]) argdiff = { k: args for k, args in izip_equal(exp_record_dict.keys(), argdiff) } # Build a list of experiments and a list of records. full_headers = ['#'] + header_names record_rows = [] experiment_rows = [] experiment_row_ixs = [] counter = 1 # Start at 2 because record table has the headers. for i, (exp_id, record_ids) in enumerate(exp_record_dict.items()): experiment_row_ixs.append(counter) exp_identifier = exp_id if not self.show_args else ','.join( '{}={}'.format(k, v) for k, v in argdiff[exp_id]) experiment_rows.append([i, exp_identifier]) for j, record_id in enumerate(record_ids): record_rows.append([j] + row_func( record_id, headers, raise_display_errors=self.raise_display_errors, truncate_to=self.truncate_result_to, ignore_valid_keys=self.ignore_valid_keys)) counter += 1 remove_notes_if_no_notes(record_rows) # Merge the experiments table and record table. record_table_rows = tabulate(record_rows, headers=full_headers, tablefmt="pipe").split('\n') del record_table_rows[1] # Get rid of that silly line. experiment_table_rows = tabulate( experiment_rows, numalign='left').split('\n')[ 1:-1] # First and last are just borders longest_row = max(max(len(r) for r in record_table_rows), max(len(r) for r in experiment_table_rows) + 4) if len(record_table_rows) > 0 else 0 record_table_rows = [ r if len(r) == longest_row else r[:-1] + ' ' * (longest_row - len(r)) + r[-1] for r in record_table_rows ] experiment_table_rows = [ ('=' if i == 0 else '-') * longest_row + '\n' + r + ' ' * (longest_row - len(r) - 1) + '|' for i, r in enumerate(experiment_table_rows) ] all_rows = [ surround_with_header( 'Experiments', width=longest_row, char='=') ] + insert_at(record_table_rows, experiment_table_rows, indices=experiment_row_ixs) + ['=' * longest_row] table = '\n'.join(all_rows) elif self.display_format == 'flat': # Display First record on same row full_headers = ['E#', 'R#', 'Experiment'] + header_names rows = [] for i, (exp_id, record_ids) in enumerate(exp_record_dict.items()): if len(record_ids) == 0: rows.append([str(i), '', exp_id, '<No Records>'] + ['-'] * (len(headers) - 1)) else: for j, record_id in enumerate(record_ids): rows.append([ str(i) if j == 0 else '', j, exp_id if j == 0 else '' ] + row_func( record_id, headers, raise_display_errors=self.raise_display_errors, truncate_to=self.truncate_result_to, ignore_valid_keys=self.ignore_valid_keys)) remove_notes_if_no_notes(rows) table = tabulate(rows, headers=full_headers) else: raise NotImplementedError(self.display_format) return table
def get_experiment_list_str(self, exp_record_dict): headers = { 'full': [ExpRecordDisplayFields.RUNS, ExpRecordDisplayFields.DURATION, ExpRecordDisplayFields.STATUS, ExpRecordDisplayFields.ARGS_CHANGED, ExpRecordDisplayFields.RESULT_STR, ExpRecordDisplayFields.NOTES], 'results': [ExpRecordDisplayFields.RESULT_STR] }[self.view_mode] if self.remove_prefix: deprefixed_ids = deprefix_experiment_ids(exp_record_dict.keys()) exp_record_dict = OrderedDict((k, v) for k, v in zip(deprefixed_ids, exp_record_dict.values())) row_func = _get_record_rows_cached if self.cache_result_string else _get_record_rows header_names = [h.value for h in headers] def remove_notes_if_no_notes(_record_rows, _record_headers): notes_column_index = _record_headers.index(ExpRecordDisplayFields.NOTES.value) if ExpRecordDisplayFields.NOTES.value in _record_headers else None # Remove the notes column if there are no notes! if notes_column_index is not None and all(row[notes_column_index]=='' or row[notes_column_index]=='-' for row in _record_rows): new_rows = [] for row in _record_rows: new_rows.append(row[:notes_column_index]+row[notes_column_index+1:]) new_headers = _record_headers[:notes_column_index]+_record_headers[notes_column_index+1:] else: new_rows = _record_rows new_headers = _record_headers return new_rows, new_headers if self.display_format=='nested': # New Display mode if self.show_args: _, argdiff = separate_common_items([load_experiment(ex).get_args().items() for ex in exp_record_dict]) argdiff = {k: args for k, args in izip_equal(exp_record_dict.keys(), argdiff)} # Build a list of experiments and a list of records. full_headers = ['#']+header_names record_rows = [] experiment_rows = [] experiment_row_ixs = [] counter = 1 # Start at 2 because record table has the headers. for i, (exp_id, record_ids) in enumerate(exp_record_dict.items()): experiment_row_ixs.append(counter) exp_identifier = exp_id if not self.show_args else ','.join('{}={}'.format(k, v) for k, v in argdiff[exp_id]) experiment_rows.append([i, exp_identifier]) for j, record_id in enumerate(record_ids): record_rows.append([j]+row_func(record_id, headers, raise_display_errors=self.raise_display_errors, truncate_to=self.truncate_result_to, ignore_valid_keys=self.ignore_valid_keys)) counter+=1 record_rows, full_headers = remove_notes_if_no_notes(record_rows, full_headers) # Merge the experiments table and record table. if self.table_package=='tabulate': record_table_rows = tabulate(record_rows, headers=full_headers, tablefmt="pipe").split('\n') del record_table_rows[1] # Get rid of that silly line. experiment_table_rows = tabulate(experiment_rows, numalign='left').split('\n')[1:-1] # First and last are just borders longest_row = max(max(len(r) for r in record_table_rows), max(len(r) for r in experiment_table_rows)+4) if len(experiment_table_rows)>0 and len(record_table_rows)>0 else 0 record_table_rows = [r if len(r)==longest_row else r[:-1] + ' '*(longest_row-len(r)) + r[-1] for r in record_table_rows] experiment_table_rows = [('=' if i==0 else '-')*longest_row+'\n'+r + ' '*(longest_row-len(r)-1)+'|' for i, r in enumerate(experiment_table_rows)] all_rows = [surround_with_header('Experiments', width=longest_row, char='=')] + (insert_at(record_table_rows, experiment_table_rows, indices=experiment_row_ixs) if len(experiment_table_rows)>0 else ['<No non-root Experiments>']) + ['='*longest_row] table = '\n'.join(all_rows) else: raise NotImplementedError(self.table_package) elif self.display_format=='flat': # Display First record on same row full_headers = ['E#', 'R#', 'Experiment']+header_names rows = [] for i, (exp_id, record_ids) in enumerate(exp_record_dict.items()): if len(record_ids)==0: rows.append([str(i), '', exp_id, '<No Records>'] + ['-']*(len(headers)-1)) else: for j, record_id in enumerate(record_ids): rows.append([str(i) if j==0 else '', j, exp_id if j==0 else '']+row_func(record_id, headers, raise_display_errors=self.raise_display_errors, truncate_to=self.truncate_result_to, ignore_valid_keys=self.ignore_valid_keys)) assert len(rows[0])==len(full_headers) rows, full_headers = remove_notes_if_no_notes(rows, full_headers) if self.table_package == 'pretty_table': from prettytable.prettytable import PrettyTable table = str(PrettyTable(rows, field_names=full_headers, align='l', max_table_width=self.max_width)) elif self.table_package == 'tabulate': table = tabulate(rows, headers=full_headers) else: raise NotImplementedError(self.table_package) else: raise NotImplementedError(self.display_format) return table