def processDir(dir, desc): ans = "" # COPY all images to the appropriate report folder and get their # filepaths (dict of test:filepath pairs) imgs = getFastqcLocal(directory=dir) # Get the status flags, which have been computed according to our custom rules # returns dict of test:status pairs statuses = parseFastqcSummary(dir) for test in opts["tests"]: # Relative to report.html in project directory dirs = imgs[test].split(os.sep) pathRel = os.sep.join(dirs[len(dirs) - 4:len(dirs)]) ans += tagFastqc(pathRel) + "\n" ans += captionFastqc(test, statuses[test]) sectionHead = htmltag.h3(desc) paragraph = htmltag.p(htmltag.HTML(ans)) div = htmltag.div(htmltag.HTML(paragraph)) return htmltag.div(htmltag.HTML(sectionHead + div), _class="accordion")
def captionFastqc(test, status): passfail = 1 if status == "PASS" else 0 caption = opts["qcCaptions"][test][passfail] if len(caption) == 0: return "" if status == "PASS": ans = htmltag.div( htmltag.HTML( htmltag.div( htmltag.HTML( htmltag.p( htmltag.HTML( htmltag.span( _class="ui-icon ui-icon-info", style="float: left; margin-right: .3em;", ) + caption))), _class="ui-state-highlight ui-corner-all", style="margin-top: 20px; padding: 0 .7em;", )), _class="ui-widget", ) else: ans = htmltag.div( htmltag.HTML( htmltag.div( htmltag.HTML( htmltag.p( htmltag.HTML( htmltag.span( _class="ui-icon ui-icon-info", style="float: left; margin-right: .3em;", ) + caption))), _class="ui-state-error ui-corner-all", style="margin-top: 20px; padding: 0 .7em;", )), _class="ui-widget", ) return ans
def plot(plot_spec): """ create a plot by storing the data in a json file and returning HTML for displaying the plot """ global NEXT_PLOT_INDEX # compute an ID for this plot plot_id = 'plot%d' % NEXT_PLOT_INDEX NEXT_PLOT_INDEX += 1 # process the dictionary of plot specifications plot_spec = prep_plot_spec(plot_spec) # create HTML h = htmltag.div('', id=plot_id) h += htmltag.script('var g_%s = %s;\ngeneratePlot("%s", g_%s);' % (plot_id, json.dumps(plot_spec), plot_id, plot_id)) return h
def plot(plot_spec): """ create a plot by storing the data in a json file and returning HTML for displaying the plot """ global next_plot_index # compute an ID for this plot id = 'plot%d' % next_plot_index next_plot_index += 1 plot_spec = prep_plot_spec(plot_spec) # create HTML h = htmltag.div('', id=id) h += htmltag.script('var g_%s = %s;\ngeneratePlot("%s", g_%s);' % (id, json.dumps(plot_spec), id, id)) return h
def plotly(data, layout, modebar=False): """ directly call Plotly.newPlot(data, layout) Args: data (list): list of dictionaries each dictionary has keys relevant to the plot e.g. x, y, mode, type layout (dict): dictionary to describe the overall layout e.g. title, xaxis.label Returns: HTML (str) """ # compute an ID for this plot global NEXT_PLOT_INDEX plot_id = 'plot%d' % NEXT_PLOT_INDEX NEXT_PLOT_INDEX += 1 # create HTML mode_bar_dict = {'displayModeBar': modebar} h = htmltag.div('', id=plot_id) h += htmltag.script('\nPlotly.newPlot(%s, %s, %s, %s);' % (plot_id, json.dumps(data), json.dumps(layout), json.dumps(mode_bar_dict))) return h
def _angled_html_table(column_headers, row_headers, row_data): rc = div(table(), _class="angled_table") return rc
def _header_entry(odd, data): cl = "" if odd: cl = "odd" return div(span(data), _class=cl)
def to_html(results): PREAMBLE_FILE = os.getenv('LSM_PREAMBLE_FILE', "") preamble = "" methods = [ 'capabilities', 'systems', 'plugin_info', 'pools', 'job_status', 'job_free', 'iscsi_chap_auth', 'volumes', 'volume_create', 'volume_delete', 'volume_resize', 'volume_replicate', 'volume_replicate_range_block_size', 'volume_replicate_range', 'volume_enable', 'volume_disable', 'disks', 'target_ports', 'volume_mask', 'volume_unmask', 'volume_child_dependency', 'volume_child_dependency_rm', 'access_groups', 'access_groups_granted_to_volume', 'access_group_create', 'access_group_delete', 'volumes_accessible_by_access_group', 'access_groups_granted_to_volume', 'access_group_initiator_add', 'access_group_initiator_delete', 'fs', 'fs_create', 'fs_delete', 'fs_resize', 'fs_clone', 'fs_file_clone', 'fs_snapshots', 'fs_snapshot_create', 'fs_snapshot_delete', 'fs_snapshot_restore', 'fs_child_dependency', 'fs_child_dependency_rm', 'export_auth', 'exports', 'export_fs', 'export_remove' ] ch = [] row_data = [] if os.path.isfile(PREAMBLE_FILE): with open(PREAMBLE_FILE, 'r') as pm: preamble = pm.read() #Build column header for r in results: ch.append(r['SYSTEM']['ID']) # Add overall pass/fail for unit tests pass_fail = ['Overall Pass/Fail result'] for r in results: if r['META']['ec'] == '0': pass_fail.append('P') else: pass_fail.append('F') row_data.append(pass_fail) # Append on link for error log error_log = ['Error log (click +)'] for r in results: error_log.append('<a href="%s">+</a>' % ('./' + os.path.basename(r['META']['error_file']))) row_data.append(error_log) for m in methods: row = [m] for r in results: row.append(get_result(r, m)) row_data.append(row) # Build HTML text = '<!DOCTYPE html>' text += str( html( head( link(rel="stylesheet", type="text/css", href="../../test.css"), title("libStorageMgmt test results"), ), body( HTML( h1("%s Results generated @ %s") % (preamble, time.strftime("%c"))), div(table(_table_header(ch), _table_body(row_data)), _class="angled_table"), div( pre(" Legend\n" " P = Pass (Method called and returned without error)\n" " F = Fail (Method call returned an error)\n" " U = Unsupported or unable to test due to other errors\n" " * = Unable to connect to array or provider totally unsupported\n" " + = hyper link to error log"))))) return bs(text).prettify()
def to_html(results): PREAMBLE_FILE = os.getenv('LSM_PREAMBLE_FILE', "") preamble = "" methods = ['capabilities', 'systems', 'plugin_info', 'pools', 'job_status', 'job_free', 'iscsi_chap_auth', 'volumes', 'volume_create', 'volume_delete', 'volume_resize', 'volume_replicate', 'volume_replicate_range_block_size', 'volume_replicate_range', 'volume_enable', 'volume_disable', 'disks', 'target_ports', 'volume_mask', 'volume_unmask', 'volume_child_dependency', 'volume_child_dependency_rm', 'access_groups', 'access_groups_granted_to_volume', 'access_group_create', 'access_group_delete', 'volumes_accessible_by_access_group', 'access_groups_granted_to_volume', 'access_group_initiator_add', 'access_group_initiator_delete', 'fs', 'fs_create', 'fs_delete', 'fs_resize', 'fs_clone', 'fs_file_clone', 'fs_snapshots', 'fs_snapshot_create', 'fs_snapshot_delete', 'fs_snapshot_restore', 'fs_child_dependency', 'fs_child_dependency_rm', 'export_auth', 'exports', 'export_fs', 'export_remove' ] ch = [] row_data = [] if os.path.isfile(PREAMBLE_FILE): with open(PREAMBLE_FILE, 'r') as pm: preamble = pm.read() #Build column header for r in results: ch.append(r['SYSTEM']['ID']) # Add overall pass/fail for unit tests pass_fail = ['Overall Pass/Fail result'] for r in results: if r['META']['ec'] == '0': pass_fail.append('P') else: pass_fail.append('F') row_data.append(pass_fail) # Append on link for error log error_log = ['Error log (click +)'] for r in results: error_log.append('<a href="%s">+</a>' % ('./' + os.path.basename(r['META']['error_file']))) row_data.append(error_log) for m in methods: row = [m] for r in results: row.append(get_result(r, m)) row_data.append(row) # Build HTML text = '<!DOCTYPE html>' text += str(html( head(link(rel="stylesheet", type="text/css", href="../../test.css"), title("libStorageMgmt test results"), ), body( HTML(h1("%s Results generated @ %s") % (preamble, time.strftime("%c"))), div(table(_table_header(ch), _table_body(row_data)), _class="angled_table"), div(pre( " Legend\n" " P = Pass (Method called and returned without error)\n" " F = Fail (Method call returned an error)\n" " U = Unsupported or unable to test due to other errors\n" " * = Unable to connect to array or provider totally unsupported\n" " + = hyper link to error log\n\n\n", HTML(' Source code for plug-in for this test run <a href=./smis.py.html>is here. </a>')))) )) return bs(text).prettify()
def vis(): def to_str(x): return '{:.2f}'.format(x.item()) assert FLAGS.dataset == 'mimic-cxr' phase = Phase.test dataset = test_dataset data_loader = test_loader if FLAGS.ckpt_path: logger.info(f'Loading model from {FLAGS.ckpt_path}') load_state_dict(model, torch.load(FLAGS.ckpt_path)) log = Log() converter = SentIndex2Report(index_to_word=Dataset.index_to_word) chexpert = CheXpert() working_dir = os.path.join(FLAGS.working_dir, 'vis', datetime.datetime.now().strftime('%Y-%m-%d-%H%M%S-%f')) image_dir = os.path.join(working_dir, 'imgs') os.makedirs(image_dir) reports = [] prog = tqdm.tqdm(enumerate(data_loader), total=len(data_loader)) for (num_batch, batch) in prog: for (key, value) in batch.items(): batch[key] = value.to(FLAGS.device) losses = {} metrics = {} with torch.no_grad(): batch = model(batch, phase=Phase.test, beam_size=FLAGS.beam_size) _log = {**losses, **metrics} log.update(_log) prog.set_description(', '.join( [f'[{phase:5s}]'] + [f'{key:s}: {log[key]:8.2e}' for key in sorted(losses.keys())] )) item_index = to_numpy(batch['item_index']) _text_length = batch['_text_length'] # (num_reports,) ( _stop, # (num_reports, max_num_sentences, 1) _temp, # (num_reports, max_num_sentences, 1) _attention, # (num_repoers, max_num_sentences, beam_size, max_num_words, 65) _score, # (num_reports, max_num_sentences, beam_size) _sum_log_probability, # (num_reports, max_num_sentences, beam_size) _text, # (num_reports, max_num_sentences, beam_size, max_num_words) _sent_length, # (num_reports, max_num_sentences, beam_size) ) = [ pad_packed_sequence(batch[key], length=_text_length) for key in ['_stop', '_temp', '_attention', '_score', '_sum_log_probability', '_text', '_sent_length'] ] (fig, ax) = plot.subplots(1, figsize=(6, 6), gridspec_kw={'left': 0, 'right': 1, 'bottom': 0, 'top': 1}) for num_report in range(len(_text_length)): num = num_batch * FLAGS.batch_size + num_report item = dataset.df.iloc[int(item_index[num_report])] image_path = mimic_cxr.image_path(dicom_id=item.dicom_id) sentences = [] for num_sentence in range(_text_length[num_report]): beams = [] for num_beam in range(FLAGS.beam_size): num_words = _sent_length[num_report, num_sentence, num_beam] _sentence = _text[num_report, num_sentence, num_beam] _words = Dataset.index_to_word[to_numpy(_sentence[:num_words])] texts = [] for num_word in range(num_words): _attention_path = os.path.join(image_dir, f'{num}-{num_sentence}-{num_beam}-{num_word}.png') print(_attention_path) # texts.append(span(_words[num_word])) _a = _attention[num_report, num_sentence, num_beam, num_word, :FLAGS.image_size * FLAGS.image_size] _a_sum = _a.sum() _a = _a - _a.min() _a = _a / _a.max() _a = _a.reshape(FLAGS.image_size, FLAGS.image_size) _a = F.interpolate(_a[None, None, :], scale_factor=(32, 32), mode='bilinear', align_corners=False)[0, 0] ''' ax.contourf(to_numpy(_a), cmap='gray') ax.set_axis_off() fig.savefig(_attention_path, bbox_inches=0) ax.cla() ''' _a = 255 * to_numpy(_a) PIL.Image.fromarray(_a.astype(np.uint8)).save(_attention_path) texts.append(span(_words[num_word], **{ 'data-toggle': 'tooltip', 'title': ( p('Total attention: ' + to_str(_a_sum)) + img(src=f'http://monday.csail.mit.edu/xiuming{_attention_path}', width='128', height='128') ).replace('"', '\''), })) beams.append({ 'score': to_str(_score[num_report, num_sentence, num_beam]), 'log_prob': to_str(_sum_log_probability[num_report, num_sentence, num_beam]), 'text': str('\n'.join(texts)), }) sentence = {} if mode & Mode.use_label_ce: sentence.update({ 'stop': to_str(_stop[num_report, num_sentence]), 'temp': to_str(_temp[num_report, num_sentence]), 'labels': [{ 'label': label_col, 'prob': to_str(_l), } for (label_col, _l) in zip(Dataset.label_columns, _label[num_report, num_sentence])], }) if mode & Mode.gen_text: sentence.update({ 'beams': beams, }) sentences.append(sentence) reports.append({ 'rad_id': str(dataset.df.rad_id.iloc[item_index[num_report]]), 'dicom_id': str(dataset.df.dicom_id.iloc[item_index[num_report]]), 'image': str(img(src=f'http://monday.csail.mit.edu/xiuming{image_path}', width='256')), 'generated text': sentences, 'ground truth text': converter(batch['text'][num_report], batch['sent_length'][num_report], batch['text_length'][num_report], is_eos=False)[0], }) if num_batch == 0: break s = json2html.convert(json=reports, table_attributes='class="table table-striped"', escape=False) s = ( head([ link(rel='stylesheet', href='https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css'), script(src='https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js'), script(src='https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js'), script('$(function () { $(\'[data-toggle="tooltip"]\').tooltip({placement: "bottom", html: true}); })', type="text/javascript"), ]) + body(div(div(div(HTML(s), _class='panel-body'), _class='panel panel-default'), _class='container-fluid')) ) with open(os.path.join(working_dir, 'index.html'), 'w') as f: f.write(s)
def pagebreak(): """ create a page break (will show up when printing to pdf) """ return htmltag.div('', style="page-break-after:always;")
def fastqc(pw): ans = "" script = "" rundirs = () imgDir = os.path.join(pw.proj("REPORT"), "FastQC") overwriteDir(imgDir) # "rundirs" should have all directories in the project directory p = Path(pw.pd) rundirs = [pw.proj(run) for run in args.runs] # REPORT is a keyword directory that is never considered a run directory # Might want to change this so the user can select specific directories as runs. runs = 0 for rundir in rundirs: # Get FastQC report from each run folder p = Path(rundir) subdirs = [x for x in p.iterdir() if x.is_dir()] # Get the directory names of the forward and reverse FastQC folders. fwdFastQc = [ os.path.join(x, "seqs_fastqc.zip") for x in subdirs if os.path.basename(x) in ["R1split", "fwdSplit"] and os.path.isfile(os.path.join(x, "seqs_fastqc.zip")) ] revFastQc = [ os.path.join(x, "seqs_fastqc.zip") for x in subdirs if os.path.basename(x) in ["R2split", "R4split", "revSplit"] and os.path.isfile(os.path.join(x, "seqs_fastqc.zip")) ] # for subdir in subdirs: # # if subdir == "R1split" or subdir == "fwdSplit": # fwdFastQc = os.path.join(subdir, "seqs_fastqc") # elif (subdir in ["R2split", "R4split", "revSplit"]): # revFastQc = os.path.join(subdir, "seqs_fastqc") ### UNTESTED ### if len(revFastQc) > 1: print( "Error: multiple FastQC directories for reverse reads in run " + os.path.basename(str(rundir)) + " found.") questions = [{ "type": "list", "name": "chooser", "message": "Please choose one:", "choices": revFastQc, }] revFastQc = [ PyInquirer.prompt(questions, style=examples.custom_style_2)["chooser"] ] if len(fwdFastQc) > 1: print( "Error: multiple FastQC directories for forward reads in run " + os.path.basename(str(rundir)) + " found.") questions = [{ "type": "list", "name": "chooser", "message": "Please choose one:", "choices": fwdFastQc, }] fwdFastQc = [ PyInquirer.prompt(questions, style=examples.custom_style_2)["chooser"] ] # Now fwdFastQc and revFastQc are lists containing zero or one filename to a zip folder if (len(fwdFastQc) == 1) and (len(revFastQc) == 1): runs += 1 fwdFastQc = str(fwdFastQc[0]) revFastQc = str(revFastQc[0]) for zipdir in (fwdFastQc, revFastQc): with zipfile.ZipFile(zipdir, "r") as zip_ref: zip_ref.extractall( os.path.dirname(zipdir[0:len(zipdir) - 4])) fwdFastQc = fwdFastQc[0:len(fwdFastQc) - 4] revFastQc = revFastQc[0:len(revFastQc) - 4] def processDir(dir, desc): ans = "" # COPY all images to the appropriate report folder and get their # filepaths (dict of test:filepath pairs) imgs = getFastqcLocal(directory=dir) # Get the status flags, which have been computed according to our custom rules # returns dict of test:status pairs statuses = parseFastqcSummary(dir) for test in opts["tests"]: # Relative to report.html in project directory dirs = imgs[test].split(os.sep) pathRel = os.sep.join(dirs[len(dirs) - 4:len(dirs)]) ans += tagFastqc(pathRel) + "\n" ans += captionFastqc(test, statuses[test]) sectionHead = htmltag.h3(desc) paragraph = htmltag.p(htmltag.HTML(ans)) div = htmltag.div(htmltag.HTML(paragraph)) return htmltag.div(htmltag.HTML(sectionHead + div), _class="accordion") fwd = processDir(fwdFastQc, "Forward reads") rev = processDir(revFastQc, "Reverse reads") sectionHead = htmltag.h2(os.path.basename(rundir)) ans += htmltag.div( htmltag.HTML(sectionHead + htmltag.div(htmltag.HTML(fwd + rev))), _class="accordion accordionRun", ) else: if len(fwdFastQc) == 0: checkedDirs = "\n".join([ os.path.join(str(rundir), subdir, "seqs_fastqc") for subdir in ["R1split", "fwdSplit"] ]) print("Warning: FastQC files for forward reads in run " + os.path.basename(str(rundir)) + " not found. Looked for:\n" + checkedDirs + "\n") if len(revFastQc) == 0: checkedDirs = "\n".join([ os.path.join(str(rundir), subdir) for subdir in ["R2split", "R4split", "revSplit"] ]) print("Warning: FastQC files for reverse reads in run " + os.path.basename(str(rundir)) + " not found. Looked for:\n" + checkedDirs + "\n") if len(ans) > 0: script = htmltag.HTML(""" <script> $( ".accordion" ).accordion({collapsible: true, heightStyle: "content"}); $( ".accordionRun" ).accordion( "option", "active", 0 ); </script>""") if runs == 1: script = script.append( """$( ".accordionRun" ).accordion( "option", "collapsible", false ); """) return ans + script