def get_stack_overflow(): """The Stack Overflow compontent writes the Questions, Results and a Distribution""" schema, results = get_data() questions_grid = stack_overflow_questions_grid(schema) items = [ get_stack_overflow_intro(), ip.Markdown("""## Stack Overflow Questions 2019"""), questions_grid, ip.Markdown("## Stack Overflow Results 2019"), stack_overflow_results_grid(results, questions_grid), respondents_per_country_component(results), ] return to_output_widget(items)
def progress_report(self): """ Reports the progress of the inference """ self.get_intermediate_results() dead_samples = self.intermediate_results['rejected_points'] live_samples = self.intermediate_results['live_points'] likelihood = self.intermediate_results['logLikelihood'] lnX = self.intermediate_results['lnX'] if not ((dead_samples is None) or (likelihood is None) or (lnX is None)): fig = visualization.trace_plot( parameter_names=self._active_parameters, samples=dead_samples, live_samples=live_samples, likelihood=likelihood, lnX=lnX) fig_filepath = os.path.join(self._run_directory, 'progress_report.pdf') msg = 'Saving progress report to {}'.format(fig_filepath) log.info(msg) fig.savefig(fig_filepath) if misc.is_notebook(): ipd.clear_output() ipd.display( ipd.Markdown( "\n**Progress report:**" '\nnumber of likelihood evaluations {}'.format( self._likelihood_evaluations_counter))) plt.show() else: print(msg)
def show(self, format=None): """show the data object content in Jupyter :param format: format to use (when there is no/wrong suffix), e.g. 'png' """ if not is_ipython: logger.warning( "Jupyter/IPython was not detected, .show() will only display inside Jupyter" ) return from IPython import display suffix = self.suffix.lower() if format: suffix = "." + format if suffix in [".jpg", ".png", ".gif"]: display.display(display.Image(self.get(), format=suffix[1:])) elif suffix in [".htm", ".html"]: display.display(display.HTML(self.get(encoding="utf-8"))) elif suffix in [".csv", ".pq", ".parquet"]: display.display(self.as_df()) elif suffix in [".yaml", ".txt", ".py"]: display.display(display.Pretty(self.get(encoding="utf-8"))) elif suffix == ".json": display.display(display.JSON(orjson.loads(self.get()))) elif suffix == ".md": display.display(display.Markdown(self.get(encoding="utf-8"))) else: logger.error(f"unsupported show() format {suffix} for {self.url}")
def hypothesis_testing(): import IPython.display as dp notes = """### Assumption Tests |Assumption test| Function | | --- | --- | | **Normality**| `scipy.stats.normaltest`| | **Equal Variance** | `scipy.stats.levene`| ### Hypothesis Tests | Parametric tests (means) | Function | Nonparametric tests (medians) | Function | | --- | --- | --- | --- | | **1-sample t test** |`scipy.stats.ttest_1samp()`| **1-sample Wilcoxon** |`scipy.stats.wilcoxon`| | **2-sample t test** |`scipy.stats.ttest_ind()` | **Mann-Whitney U test** |`scipy.stats.mannwhitneyu()`| | **One-Way ANOVA** | `scipy.stats.f_oneway()` | **Kruskal-Wallis** | `scipy.stats.kruskal` | ### Post-Hoc Tests/Calculations | Post-Hoc Tests/Calculatons|Function| |--- | --- | |**Tukey's Pairwise Comparisons** | `statsmodels.stats.multicomp.pairwise_tukeyhsd`| |**Effect Size**| `Cohens_d`| |**Statistical Power** | `statsmodels.stats.power`:<br> `TTestIndPower` , `TTestPower` """ return dp.Markdown(notes)
def explore(obj): ''' Interactive object explorer Args: obj: Object to introspect This widget displays an object and some information about it (eg: its signature, if it's callable) and provides a dropdown to select one of its attributes. The selected attribute is displayed in the same way (including a dropdown of the attribute's attributes.) ''' def show_markdown(markdown): display.display(display.Markdown(markdown.replace('>', '\\>'))) flag_predicates = {'callable': callable} flag_predicates.update({ k: v for k, v in vars(inspect).items() if k.startswith('is') and callable(v) }) show_markdown('### {typ}'.format(typ=type(obj).__name__)) display.display(obj) flag_names = sorted(k[2:] for k, v in flag_predicates.items() if v(obj)) if flag_names: show_markdown('_({})_'.format(', '.join(flag_names))) if callable(obj): try: sig = inspect.signature(obj) except: show_markdown('_No signature available_') sig = None if sig is not None: param_attrs = ['name', 'kind', 'default', 'annotation'] display.display(display.Markdown('##### Parameters')) def str_or_null(x): return '' if x is inspect._empty else str(x) display.display( display.HTML( str( html_table( [[str_or_null(getattr(p, a)) for a in param_attrs] for p in sig.parameters.values()], header_row=param_attrs)))) attr_descriptions = OrderedDict([('', None)]) retval_key = '<return value>' if callable(obj): attr_descriptions[retval_key] = retval_key attr_descriptions.update( ('{val} ({typ})'.format(typ=type(getattr(obj, k)).__name__, val=k), k) for k in dir(obj)) @ipywidgets.interact(attribute=attr_descriptions) def explore_attr(attribute): if attribute is not None: explore(obj() if attribute == retval_key else getattr(obj, attribute))
def evidence_report(self, sdigits=4): if misc.is_notebook(): ipd.display(ipd.Markdown("**Evidence report:**")) out = r"\log\mathcal{{ Z }} = " out += q2tex(self.log_evidence, self.log_evidence_err) ipd.display(ipd.Math(out)) else: print('Evidence report') print('logZ =', self.log_evidence, '±', self.log_evidence_err)
def _print_plain_override_for_ndarray(arg, p, cycle): """caller for pretty, for use in IPython 0.11""" import IPython.display as d if isinstance(arg, numpy.ndarray): if hasattr(arg.dtype, 'fields') and arg.dtype.fields is not None: mkd = 'structured array with fields: ' + ', '.join( [j[0] for j in arg.dtype.descr]) else: mkd = "array" if any(numpy.array(arg.shape) > 20): print("Matrix is too large (" + 'x'.join([str(j) for j in arg.shape]) + ") -- using text numpy print") print(arg) elif len(arg.shape) == 2: math_str = render_matrix(arg) d.display( d.Markdown("**numpy 2D " + mkd + "** represented as a matrix:")) d.display(d.Math(math_str)) elif len(arg.shape) == 1: d.display( d.Markdown("**numpy 1D " + mkd + "** represented as a row vector:")) d.display(d.Math(render_matrix(arg.reshape(1, -1)))) elif len(arg.shape) == 3: d.display( d.Markdown("***numpy 3D " + mkd + ",*** represented as a series of matrices:")) math_str = r'\begin{bmatrix}' + '\n' for j in range(arg.shape[0]): math_str += '\\text{Select slice with outermost dimension set to %d}' % j math_str += r'\\' + '\n' math_str += render_matrix(arg[j, :, :]) math_str += r'\\' + '\n' math_str += r'\end{bmatrix}' + '\n' d.display(d.Math(math_str)) elif len(arg.shape) > 3: d.display( d.Markdown( "***numpy ND array*** $N>3$ dimensions ($" + r'\times'.join(map(str, arg.shape)) + "$), so I'm just giving the text representation:")) d.display(str(arg))
def print_md(msg: str, color: str = None) -> None: """Print message with Markdown support. The message supports markdown syntax like **bold** or _italics_. :param msg: message to print. :param color: name of desired text color, e.g. 'red' """ color_msg = "<span style='color:{}'>{}</span>".format(color, msg) ipd.display(ipd.Markdown(color_msg))
def macro(code: str) -> Tuple[display.DisplayObject]: """ >>> url = "https://test.com" >>> assert macro(url) and macro(''' {} ... ... '''.format(url))[0].data.strip() == url """ lines = code.splitlines() if lines and lines[0].strip(): if len(lines) is 1 and lines[0][:1].strip(): type = mimetypes.guess_type(code)[0] is_image = type and type.startswith('image') disp = partial(display.Image, embed=True) if is_image else display.Markdown if fnmatch(code, "* [[]*[]](*)*"): url = code.rsplit(')', 1)[0].rsplit('(', 1)[1].split(' ', 1)[0] if url and url != '#': return (display.Markdown(data=code), *macro(url)) if fnmatch(code, 'http*://*'): return is_image and display.Image(url=code) or display.IFrame( code, width=600, height=400), return display.Markdown(data=code), return tuple()
def get_resources(): items = [ ip.Markdown("""## Introduction You can use Python and Voila as shown in the code below to create a web app! """), # Use https://gist.github.com/jiffyclub/5385501 to format code ip.Code( """ import ipython.display as ip ip.display( ip.Markdown( "For more info watch the ***4 minutes introduction*** to Streamlit" ), ip.YouTubeVideo("VtchVpoSdoQ") ) """, language="python3", ), ip.Markdown( "For more info watch the ***30 minutes introduction*** to Voila"), ip.YouTubeVideo("VtchVpoSdoQ"), ] return to_output_widget(items)
def posterior_report(self, sdigits=2, **kwargs): """ Displays the best fit values and 1-sigma errors for each active parameter. Also produces a corner plot of the samples, which is saved to the run directory. If running on a jupyter-notebook, a nice LaTeX display is used, and the plot is shown. Parameters ---------- sdigits : int The number of significant digits to be used """ out = '' for param, pdict in self.posterior_summary.items(): if misc.is_notebook(): # Extracts LaTeX representation from astropy unit object out += r"\\ \text{{ {0}: }}\; ".format(param) out += q2tex(*map(pdict.get, ['median', 'errup', 'errlo']), sdigits=sdigits) out += r"\\" else: out += r"{0}: ".format(param) md, errlo, errup = map(pdict.get, ['median', 'errlo', 'errup']) if isinstance(md, apu.Quantity): unit = str(md.unit) md, errlo, errup = map(lambda x: x.value, [md, errlo, errup]) else: unit = "" v, l, u = misc.adjust_error_intervals(md, errlo, errup, sdigits=sdigits) out += r'{0} (-{1})/(+{2}) {3}\n'.format(v, l, u, unit) fig = self.corner_plot(**kwargs) fig.savefig(os.path.join(self._run_directory, 'corner_plot.pdf')) if misc.is_notebook(): ipd.display(ipd.Markdown("\n**Posterior report:**")) plt.show() ipd.display(ipd.Math(out)) else: # Restores linebreaks and prints print('Posterior report') print(out.replace(r'\n', '\n'))
def get_stack_overflow_intro(): items = [ ip.Markdown(f""" ## Stack Overflow 2019 You will be analyzing and providing insights from the Stack Overflow 2019 survey <a href="{stack_overflow.SURVEY_2019_URL}" target="_blank"> <img src="{stack_overflow.IMAGE_2019_URL}"style="width=100%;"> </a> Results: [{stack_overflow.SURVEY_2019_URL}]({stack_overflow.DATA_URL}) Data: [{stack_overflow.DATA_URL}]({stack_overflow.DATA_URL}) """) ] return to_output_widget(items)
def run_markdown(cpp_filename): """ Run execution file from the cpp file and present the output as markdown cpp_filename : ex) test or test.cpp """ basename, ext = os.path.splitext(cpp_filename) # https://stackoverflow.com/questions/4760215/running-shell-command-from-python-and-capturing-the-output # https://stackoverflow.com/questions/35160256/how-do-i-output-lists-as-a-table-in-jupyter-notebook # Run executable while capturing output result = subprocess.run( [os.path.join(os.curdir, basename)], stdout=subprocess.PIPE, check=True, ) # present output as a markdown disp.display(disp.Markdown(result.stdout.decode()))
def prob_combinations(): import IPython.display as dp combinations=\ r"""### Combinations > ***How many ways can we create a subset $k$ out of $n$ objects?*** - Combinations are unordered. - **The # of combinations equals number of variations, over the number of permutations:** $$ \large C_{p}^{n} = \frac{V_{p}^{n}}{P_p} = \frac{n!}{p!(n-p)!} $$ - Also represented as: $$\displaystyle\binom{n}{k} = \dfrac{P_{k}^{n}}{k!}=\dfrac{ \dfrac{n!}{(n-k)!}}{k!} = \dfrac{n!}{(n-k)!k!}$$ """ dp.display(dp.Markdown(combinations))
def progress_report(self): """ Reports the progress of the inference """ # Try to call get_intermediate_results try: self.get_intermediate_results() # If this method is not implemented, show error instead and continue except NotImplementedError: print("Progress reports can only be made if the " "'get_intermediate_results()'-method is overridden! " "Skipping report.") # If this method is implemented, create progress report else: if mpirank!=0: return dead_samples = self.intermediate_results['rejected_points'] live_samples = self.intermediate_results['live_points'] likelihood = self.intermediate_results['logLikelihood'] lnX = self.intermediate_results['lnX'] if dead_samples is not None: fig = visualization.trace_plot( parameter_names=self._active_parameters, samples=dead_samples, live_samples=live_samples, likelihood=likelihood, lnX=lnX) fig_filepath = os.path.join(self._run_directory, 'progress_report.pdf') msg = 'Saving progress report to {}'.format(fig_filepath) log.info(msg) fig.savefig(fig_filepath) if misc.is_notebook(): ipd.clear_output() ipd.display(ipd.Markdown( "\n**Progress report:**\nnumber of likelihood " "evaluations {}".format( self._likelihood_evaluations_counter))) plt.show() else: print(msg)
def prob_permutations(): from IPython.display import display,Markdown,HTML permutations=\ r"""### Permutations - **Permutations have to do with arranging objects.** - Order is important ### Permutations of $n$: > ***How many ways to arrange $n$ objects?*** $$P(n) = n!$$ - where $n$ = total number of elements ### Permutations without replacement (the $k$-permutation of $n$) > ***How many ways to select $k$ elements out of a pool of $n$ objects?*** $$ \large P_{k}^{n}= \dfrac{n!}{(n-k)!}$$ <center> this is known as a $k$-permutation of $n$. </center> ### Permutations with Replacement: $$ \large P_{j}^{n} = n^j $$ - where $n$ = total number of elements - $j$ = number of positions to fill #### Sometimes called **Variations** with notation: $$ \large \bar{V}_{j}^{n} = n^j$$ ### Permutations with repeated elements - e.g. selecting random letters from the word TENNESEE $$\dfrac{n!}{n_1!n_2!\ldots n_j!}$$ """ import IPython.display as dp display(dp.Markdown(permutations))
def stack_overflow_results_grid( results, questions_grid: qgrid.QGridWidget) -> widgets.Widget: """This component writes the Stack Overflow Developer Survey Questions Arguments: results {[type]} -- A DataFrame of the Results questions_grid {qgrid.QGridWidget} -- The table of questions """ def get_selected_questions(questions_grid): return list(questions_grid.get_selected_df()["Column"]) results_grid = qgrid.show_grid( results[get_selected_questions(questions_grid)].head(10), column_options=styles.RESULTS_GRID_COL_OPTIONS, column_definitions=styles.RESULTS_GRID_COL_DEFS, ) no_results_grid = ip.Markdown( "**Select one or more questions in the table above to show the results!**" ) def update_results_grid(): selected_questions = list(questions_grid.get_selected_df()["Column"]) if selected_questions: results_to_show = results results_to_show = results_to_show[selected_questions] results_grid.df = results_to_show else: results_grid.df = pd.DataFrame() def questions_grid_handler(change): update_results_grid() questions_grid.observe(handler=questions_grid_handler, names="_selected_rows") return to_output_widget([no_results_grid, results_grid])
def respondents_per_country_component(results): """This component writes a bar chart showing number of Respondants per Country Arguments: results {[type]} -- A DataFrame of the Results """ distributions = ((results[["Country", "Respondent" ]].groupby("Country").count().reset_index() ).sort_values("Respondent").tail(50)) fig = px.bar( distributions, x="Respondent", y="Country", title="Count", orientation="h", height=800, width=1200, ) return to_output_widget([ ip.Markdown("""### Respondents per Countrys You can plot using matplot, seaborn, vega lite, plotly and other. Here we have chosen plotly """), go.FigureWidget(fig), ])
def freespaceImageAnalysis(fids, guesses=None, fit=True, bgInput=None, bgPcInput=None, shapes=[None], zeroCorrection=0, zeroCorrectionPC=0, keys=None, fitModule=bump, extraPicDictionaries=None, newAnnotation=False, onlyThisPic=None, pltVSize=5, plotSigmas=False, plotCounts=False, manualColorRange=None, calcTemperature=False, clearOutput=True, dataRange=None, guessTemp=10e-6, trackFitCenter=False, picsPerRep=1, startPic=0, binningParams=None, win=pw.PictureWindow(), transferAnalysisOpts=None, tferBinningParams=None, tferWin=pw.PictureWindow(), extraTferAnalysisArgs={}, emGainSetting=300, lastConditionIsBackGround=True, showTferAnalysisPlots=True, show2dFitsAndResiduals=True, plotFitAmps=False, indvColorRanges=False, fitF2D=gaussian_2d.f_notheta, rmHighCounts=True, useBase=True, weightBackgroundByLoading=True, returnPics=False, forceNoAnnotation=False): """ returnPics is false by default in order to conserve memory. The total picture arrays of lots of experiments can take up a lot of RAM and are usually unnecessary, """ fids = [fids] if type(fids) == int else fids keys = [None for _ in fids] if keys is None else keys sortedStackedPics = {} initThresholds = [None] picsForBg = [] bgWeights = [] isAnnotatedList = [] for filenum, fid in enumerate(fids): if transferAnalysisOpts is not None: res = ta.stage1TransferAnalysis(fid, transferAnalysisOpts, useBase=useBase, **extraTferAnalysisArgs) (initAtoms, tferAtoms, initAtomsPs, tferAtomsPs, key, keyName, initPicCounts, tferPicCounts, repetitions, initThresholds, avgPics, tferThresholds, initAtomImages, tferAtomImages, basicInfoStr, ensembleHits, groupedPostSelectedPics, isAnnotated) = res isAnnotatedList.append(isAnnotated) # assumes that you only want to look at the first condition. for varPics in groupedPostSelectedPics: # don't remember why 0 works if false... picsForBg.append( varPics[-1 if lastConditionIsBackGround else 0]) bgWeights.append(len(varPics[0])) allFSIPics = [ varpics[0][startPic::picsPerRep] for varpics in groupedPostSelectedPics ] if showTferAnalysisPlots: fig, axs = plt.subplots(1, 2) mp.makeAvgPlts(axs[0], axs[1], avgPics, transferAnalysisOpts, ['r', 'g', 'b']) allFSIPics = [win.window(np.array(pics)) for pics in allFSIPics] allFSIPics = ah.softwareBinning(binningParams, allFSIPics) elif type(fid) == int: ### For looking at either PGC imgs or FSI imgs with exp.ExpFile(fid) as file: # I think this only makes sense if there is a specific bg pic in the rotation picsForBg.append(list(file.get_pics())) allFSIPics = file.get_pics()[startPic::picsPerRep] _, key = file.get_key() if len(np.array(key).shape) == 2: key = key[:, 0] file.get_basic_info() allFSIPics = win.window(allFSIPics) allFSIPics = ah.softwareBinning(binningParams, allFSIPics) allFSIPics = np.reshape( allFSIPics, (len(key), int(allFSIPics.shape[0] / len(key)), allFSIPics.shape[1], allFSIPics.shape[2])) else: ### Assumes given pics have the same start pic and increment (picsPerRep). # doesn't combine well w/ transfer analysis picsForBg.append(fid) allFSIPics = fid[startPic::picsPerRep] print( "Assuming input is list of all pics, then splices to get FSI pics. Old code assumed the given were FSI pics." ) allFSIPics = win.window(allFSIPics) allFSIPics = ah.softwareBinning(binningParams, allFSIPics) allFSIPics = np.reshape( allFSIPics, (len(key), int(allFSIPics.shape[0] / len(key)), allFSIPics.shape[1], allFSIPics.shape[2])) # ############## if keys[filenum] is not None: key = keys[filenum] for i, keyV in enumerate(key): keyV = misc.round_sig_str(keyV) sortedStackedPics[keyV] = np.append( sortedStackedPics[keyV], allFSIPics[i], axis=0) if (keyV in sortedStackedPics) else allFSIPics[i] if lastConditionIsBackGround: bgInput, pcBgInput = getBgImgs( picsForBg, startPic=startPic, picsPerRep=picsPerRep, rmHighCounts=rmHighCounts, bgWeights=bgWeights, weightBackgrounds=weightBackgroundByLoading) elif bgInput == 'lastPic': bgInput, pcBgInput = getBgImgs( picsForBg, startPic=picsPerRep - 1, picsPerRep=picsPerRep, rmHighCounts=rmHighCounts, bgWeights=bgWeights, weightBackgrounds=weightBackgroundByLoading) if bgInput is not None: # was broken and not working if not given bg bgInput = win.window(bgInput) bgInput = ah.softwareBinning(binningParams, bgInput) if bgPcInput is not None: bgPcInput = win.window(bgPcInput) bgPcInput = ah.softwareBinning(binningParams, bgPcInput) if extraPicDictionaries is not None: if type(extraPicDictionaries) == dict: extraPicDictionaries = [extraPicDictionaries] for dictionary in extraPicDictionaries: for keyV, pics in dictionary.items(): sortedStackedPics[keyV] = (np.append( sortedStackedPics[keyV], pics, axis=0) if keyV in sortedStackedPics else pics) sortedStackedKeyFl = [float(keyStr) for keyStr in sortedStackedPics.keys()] sortedKey, sortedStackedPics = ah.applyDataRange( dataRange, sortedStackedPics, list(sorted(sortedStackedKeyFl))) numVars = len(sortedStackedPics.items()) if len(np.array(shapes).shape) == 1: shapes = [shapes for _ in range(numVars)] if guesses is None: guesses = [[None for _ in range(4)] for _ in range(numVars)] if len(np.array(bgInput).shape) == 2 or bgInput == None: bgInput = [bgInput for _ in range(numVars)] if len(np.array(bgPcInput).shape) == 2 or bgPcInput == None: bgPcInput = [bgPcInput for _ in range(numVars)] datalen, avgFitSigmas, images, hFitParams, hFitErrs, vFitParams, vFitErrs, fitParams2D, fitErrs2D = [ {} for _ in range(9) ] titles = ['Bare', 'Photon-Count', 'Bare-mbg', 'Photon-Count-mbg'] assert (len(sortedKey) > 0) for vari, keyV in enumerate(sortedKey): keyV = misc.round_sig_str(keyV) if vari == 0: initKeyv = keyV varPics = sortedStackedPics[keyV] # 0 is init atom pics for post-selection on atom number... if we wanted to. expansionPics = rmHighCountPics(varPics, 7000) if rmHighCounts else varPics datalen[keyV] = len(expansionPics) expPhotonCountImage = photonCounting(expansionPics, 120)[0] / len(expansionPics) bgPhotonCountImage = np.zeros( expansionPics[0].shape ) if bgPcInput[vari] is None else bgPcInput[vari] expAvg = np.mean(expansionPics, 0) bgAvg = np.zeros(expansionPics[0].shape) if ( bgInput[vari] is None or len(bgInput[vari]) == 1) else bgInput[vari] if bgPhotonCountImage is None: print('no bg photon', expAvg.shape) bgPhotonCount = np.zeros(photonCountImage.shape) avg_mbg = expAvg - bgAvg avg_mbgpc = expPhotonCountImage - bgPhotonCountImage images[keyV] = [expAvg, expPhotonCountImage, avg_mbg, avg_mbgpc] hFitParams[keyV], hFitErrs[keyV], vFitParams[keyV], vFitErrs[ keyV], fitParams2D[keyV], fitErrs2D[keyV] = [[] for _ in range(6)] for imnum, (im, guess) in enumerate(zip(images[keyV], guesses[vari])): if fit: # fancy guess_x and guess_y values use the initial fitted value, typically short time, as a guess. _, pictureFitParams2d, pictureFitErrors2d, v_params, v_errs, h_params, h_errs = ah.fitPic( im, guessSigma_x=5, guessSigma_y=5, showFit=False, guess_x=None if vari == 0 else fitParams2D[initKeyv][imnum][1], guess_y=None if vari == 0 else fitParams2D[initKeyv][imnum][2], fitF=fitF2D) fitParams2D[keyV].append(pictureFitParams2d) fitErrs2D[keyV].append(pictureFitErrors2d) hFitParams[keyV].append(h_params) hFitErrs[keyV].append(h_errs) vFitParams[keyV].append(v_params) vFitErrs[keyV].append(v_errs) # conversion from the num of pixels on the camera to microns at the focus of the tweezers cf = 16e-6 / 64 mins, maxes = [[], []] imgs_ = np.array(list(images.values())) for imgInc in range(4): if indvColorRanges: mins.append(None) maxes.append(None) elif manualColorRange is None: mins.append(min(imgs_[:, imgInc].flatten())) maxes.append(max(imgs_[:, imgInc].flatten())) else: mins.append(manualColorRange[0]) maxes.append(manualColorRange[1]) numVariations = len(images) if onlyThisPic is None: fig, axs = plt.subplots(numVariations, 4, figsize=(20, pltVSize * numVariations)) if numVariations == 1: axs = np.array([axs]) bgFig, bgAxs = plt.subplots(1, 2, figsize=(20, pltVSize)) else: numRows = int(np.ceil((numVariations + 3) / 4)) fig, axs = plt.subplots(numRows, 4 if numVariations > 1 else 3, figsize=(20, pltVSize * numRows)) avgPicAx = axs.flatten()[-3] avgPicFig = fig bgAxs = [axs.flatten()[-1], axs.flatten()[-2]] bgFig = fig if show2dFitsAndResiduals: fig2d, axs2d = plt.subplots( *((2, numVariations) if numVariations > 1 else (1, 2))) keyPlt = np.zeros(len(images)) (totalSignal, hfitCenter, hFitCenterErrs, hSigmas, hSigmaErrs, h_amp, hAmpErrs, vfitCenter, vFitCenterErrs, vSigmas, vSigmaErrs, v_amp, vAmpErrs, hSigma2D, hSigma2dErr, vSigma2D, vSigma2dErr) = [np.zeros((len(images), 4)) for _ in range(17)] for vari, ((keyV, ims), hParamSet, hErr_set, vParamSet, vErr_set, paramSet2D, errSet2D) in enumerate( zip( images.items(), *[ dic.values() for dic in [ hFitParams, hFitErrs, vFitParams, vFitErrs, fitParams2D, fitErrs2D ] ])): for which in range(4): if onlyThisPic is None: (im, ax, title, min_, max_, hparams, hErrs, vparams, vErrs, param2d, err2d) = [ obj[which] for obj in (ims, axs[vari], titles, mins, maxes, hParamSet, hErr_set, vParamSet, vErr_set, paramSet2D, errSet2D) ] else: which = onlyThisPic ax = axs.flatten()[vari] (im, title, min_, max_, hparams, hErrs, vparams, vErrs, param2d, err2d) = [ obj[which] for obj in (ims, titles, mins, maxes, hParamSet, hErr_set, vParamSet, vErr_set, paramSet2D, errSet2D) ] h_amp[vari][which], hfitCenter[vari][which], hSigmas[vari][ which] = hparams[0], hparams[1], hparams[2] * cf * 1e6 hAmpErrs[vari][which], hFitCenterErrs[vari][which], hSigmaErrs[ vari][which] = hErrs[0], hErrs[1], hErrs[2] * cf * 1e6 v_amp[vari][which], vfitCenter[vari][which], vSigmas[vari][ which] = vparams[0], vparams[1], vparams[2] * cf * 1e6 vAmpErrs[vari][which], vFitCenterErrs[vari][which], vSigmaErrs[ vari][which] = vErrs[0], vErrs[1], vErrs[2] * cf * 1e6 hSigma2D[vari][which], hSigma2dErr[vari][which], vSigma2D[vari][ which], vSigma2dErr[vari][which] = [ val * cf * 1e6 for val in [param2d[-3], err2d[-3], param2d[-2], err2d[-2]] ] totalSignal[vari][which] = np.sum(im.flatten()) keyPlt[vari] = keyV res = mp.fancyImshow(fig, ax, im, imageArgs={ 'cmap': dark_viridis_cmap, 'vmin': min_, 'vmax': max_ }, hFitParams=hparams, vFitParams=vparams, fitModule=fitModule, flipVAx=True, fitParams2D=param2d) ax.set_title( keyV + ': ' + str(datalen[keyV]) + ';\n' + title + ': ' + misc.errString(hSigmas[vari][which], hSigmaErrs[vari][which]) + r'$\mu m$ sigma, ' + misc.round_sig_str(totalSignal[vari][which], 5), fontsize=12) if show2dFitsAndResiduals: X, Y = np.meshgrid(np.arange(len(im[0])), np.arange(len(im))) data_fitted = fitF2D((X, Y), *param2d) fitProper = data_fitted.reshape(im.shape[0], im.shape[1]) ax1 = axs2d[0] if numVariations == 1 else axs2d[0, vari] ax2 = axs2d[1] if numVariations == 1 else axs2d[1, vari] imr = ax1.imshow(fitProper, vmin=min_, vmax=max_) mp.addAxColorbar(fig2d, ax1, imr) ax1.contour(np.arange(len(im[0])), np.arange(len(im)), fitProper, 4, colors='w', alpha=0.2) imr = ax2.imshow(fitProper - im) mp.addAxColorbar(fig2d, ax2, imr) ax2.contour(np.arange(len(im[0])), np.arange(len(im)), fitProper, 4, colors='w', alpha=0.2) if onlyThisPic is not None: break mp.fancyImshow(avgPicFig, avgPicAx, np.mean([img[onlyThisPic] for img in images.values()], axis=0), imageArgs={'cmap': dark_viridis_cmap}, flipVAx=True) avgPicAx.set_title('Average Over Variations') ### Plotting background and photon counted background mp.fancyImshow(bgFig, bgAxs[0], bgAvg, imageArgs={'cmap': dark_viridis_cmap}, flipVAx=True) bgAxs[0].set_title('Background image (' + str(len(picsForBg) / picsPerRep) + ')') mp.fancyImshow(bgFig, bgAxs[1], bgPhotonCountImage, imageArgs={'cmap': dark_viridis_cmap}, flipVAx=True) bgAxs[1].set_title('Photon counted background image (' + str(len(picsForBg) / picsPerRep) + ')') fig.subplots_adjust(left=0, right=1, bottom=0.1, hspace=0.2, **({ 'top': 0.7, 'wspace': 0.4 } if (onlyThisPic is None) else { 'top': 0.9, 'wspace': 0.3 })) disp.display(fig) temps, tempErrs, tempFitVs, = [], [], [] if calcTemperature: for sigmas, sigmaerrs in zip( [hSigmas, vSigmas, hSigma2D, vSigma2D], [hSigmaErrs, vSigmaErrs, hSigma2dErr, vSigma2dErr]): mbgSigmas = np.array([elt[2] for elt in sigmas]) mbgSigmaErrs = np.array([elt[2] for elt in sigmaerrs]) myGuess = [0.0, min((mbgSigmas) * 1e-6), guessTemp] temp, fitV, cov = ah.calcBallisticTemperature( keyPlt * 1e-3, (mbgSigmas) * 1e-6, guess=myGuess, sizeErrors=mbgSigmaErrs) error = np.sqrt(np.diag(cov)) temps.append(temp) tempErrs.append(error[2]) tempFitVs.append(fitV) numAxisCol = int(plotSigmas) + int(plotCounts) + int(trackFitCenter) if numAxisCol != 0: fig2, axs = plt.subplots(1, numAxisCol, figsize=(15, 5)) fig2.subplots_adjust(top=0.75, wspace=0.4) colors = ['b', 'k', 'c', 'purple'] if plotSigmas: ax = (axs if numAxisCol == 1 else axs[0]) stdStyle = dict(marker='o', linestyle='', capsize=3) if onlyThisPic is not None: ax.errorbar(keyPlt, hSigmas[:, onlyThisPic], hSigmaErrs[:, onlyThisPic], color=colors[0], label='h ' + titles[onlyThisPic], **stdStyle) ax.errorbar(keyPlt, hSigma2D[:, onlyThisPic], hSigma2dErr[:, onlyThisPic], color=colors[1], label='2dh ' + titles[onlyThisPic], **stdStyle) ax.errorbar(keyPlt, vSigmas[:, onlyThisPic], vSigmaErrs[:, onlyThisPic], color=colors[2], label='v ' + titles[onlyThisPic], **stdStyle) ax.errorbar(keyPlt, vSigma2D[:, onlyThisPic], vSigma2dErr[:, onlyThisPic], color=colors[3], label='2dv ' + titles[onlyThisPic], **stdStyle) else: for whichPic in range(4): ax.errorbar(keyPlt, hSigmas[:, whichPic], hSigmaErrs[:, whichPic], color='b', label='h ' + titles[whichPic], **stdStyle) ax.errorbar(keyPlt, vSigmas[:, whichPic], vSigmaErrs[:, whichPic], color='c', label='v ' + titles[whichPic], **stdStyle) ax.set_ylim(max(0, ax.get_ylim()[0]), min([ax.get_ylim()[1], 5])) ax.set_ylabel(r'Fit Sigma ($\mu m$)') if calcTemperature: # converting time to s, hSigmas in um xPoints = np.linspace(min(keyPlt), max(keyPlt)) * 1e-3 for num, fitV in enumerate(tempFitVs): #ax.plot(xPoints*1e3, LargeBeamMotExpansion.f(xPoints, *myGuess)*1e6, label = 'guess') ax.plot(xPoints * 1e3, LargeBeamMotExpansion.f(xPoints, *fitV) * 1e6, color=colors[num]) ax.legend() if plotFitAmps: ax = (axs if numAxisCol == 1 else axs[0]) ampAx = ax.twinx() if onlyThisPic is not None: ampAx.errorbar(keyPlt, h_amp[:, onlyThisPic], hAmpErrs[:, onlyThisPic], label='h ' + titles[onlyThisPic], color='orange', **stdStyle) ampAx.errorbar(keyPlt, v_amp[:, onlyThisPic], vAmpErrs[:, onlyThisPic], label='v ' + titles[onlyThisPic], color='r', **stdStyle) else: for whichPic in range(4): ampAx.errorbar(keyPlt, h_amp[:, whichPic], hAmpErrs[:, whichPic], label='h ' + titles[whichPic], color='orange', **stdStyle) ampAx.errorbar(keyPlt, v_amp[:, whichPic], vAmpErrs[:, whichPic], label='v ' + titles[whichPic], color='r', **stdStyle) [tick.set_color('red') for tick in ampAx.yaxis.get_ticklines()] [tick.set_color('red') for tick in ampAx.yaxis.get_ticklabels()] ampAx.set_ylabel(r'Fit h_amps', color='r') hTotalPhotons, vTotalPhotons = None, None if plotCounts: # numAxCol = 1: ax = axs # numAxCol = 2: plotSigmas + plotCounts -- ax = axs[1] # numAxCol = 2: plotCounts + trackFitCenter -- ax = axs[0] # numAxCol = 3: ax = axs[1] if numAxisCol == 1: ax = axs elif numAxisCol == 2: ax = axs[1 if plotSigmas else 0] else: ax = axs[1] # Create axis to plot photon counts ax.set_ylabel(r'Integrated signal') photon_axis = ax.twinx() # This is not currently doing any correct for e.g. the loading rate. countToCameraPhotonEM = 0.018577 / (emGainSetting / 200 ) # the float is is EM200. countToScatteredPhotonEM = 0.018577 / 0.07 / (emGainSetting / 200) if onlyThisPic is not None: # calculate number of photons hamp = h_amp[:, onlyThisPic] * len( expansionPics[0][0] ) # Horizontal "un"normalization for number of columns begin averaged. vamp = v_amp[:, onlyThisPic] * len(expansionPics[0]) hsigpx = hSigmas[:, onlyThisPic] / ( 16 / 64) # Convert from um back to to pixels. vsigpx = vSigmas[:, onlyThisPic] / (16 / 64) htotalCountsPerPic = bump.area_under(hamp, hsigpx) vtotalCountsPerPic = bump.area_under(vamp, vsigpx) hTotalPhotons = countToScatteredPhotonEM * htotalCountsPerPic vTotalPhotons = countToScatteredPhotonEM * vtotalCountsPerPic ax.plot(keyPlt, totalSignal[:, onlyThisPic], marker='o', linestyle='', label=titles[onlyThisPic]) photon_axis.plot(keyPlt, hTotalPhotons, marker='o', linestyle='', color='r', label='Horizontal') photon_axis.plot(keyPlt, vTotalPhotons, marker='o', linestyle='', color='orange', label='Vertical') else: for whichPic in range(4): # See above comments amp = h_amp[:, whichPic] * len(expansionPics[0][0]) sig = hSigmas[:, whichPic] / (16 / 64) totalCountsPerPic = bump.area_under(amp, sig) hTotalPhotons = countToScatteredPhotonEM * totalCountsPerPic ax.plot(keyPlt, totalSignal[:, whichPic], marker='o', linestyle='', label=titles[whichPic]) photon_axis.plot(keyPlt, hTotalPhotons, marker='o', linestyle='', color=['red', 'orange', 'yellow', 'pink'][whichPic]) ax.legend() photon_axis.legend() [tick.set_color('red') for tick in photon_axis.yaxis.get_ticklines()] [tick.set_color('red') for tick in photon_axis.yaxis.get_ticklabels()] photon_axis.set_ylabel(r'Fit-Based Avg Scattered Photon/Img', color='r') if trackFitCenter: #numaxcol = 1: ax = axs #numaxcol = 2: trackfitcenter + plothSigmas: ax = axs[1] #numaxcol = 2: trackfitcenter + plotCounts: ax = axs[1] #numaxcol = 3: ax = axs[2] ax = (axs if numAxisCol == 1 else axs[-1]) if onlyThisPic is not None: #ax.errorbar(keyPlt, hfitCenter[:,onlyThisPic], hFitCenterErrs[:,onlyThisPic], marker='o', linestyle='', capsize=3, label=titles[onlyThisPic]); ax.errorbar(keyPlt, vfitCenter[:, onlyThisPic], vFitCenterErrs[:, onlyThisPic], marker='o', linestyle='', capsize=3, label=titles[onlyThisPic]) #def accel(t, x0, a): # return x0 + 0.5*a*t**2 #accelFit, AccelCov = opt.curve_fit(accel, keyPlt*1e-3, hfitCenter[:,onlyThisPic], sigma = hFitCenterErrs[:,onlyThisPic]) #fitx = np.linspace(keyPlt[0], keyPlt[-1])*1e-3 #fity = accel(fitx, *accelFit) #ax.plot(fitx*1e3, fity) else: for whichPic in range(4): ax.errorbar(keyPlt, hfitCenter[:, whichPic], hFitCenterErrs[:, whichPic], marker='o', linestyle='', capsize=3, label=titles[whichPic]) #accelErr = np.sqrt(np.diag(AccelCov)) fig2.legend() ax.set_ylabel(r'Fit Centers (pix)') ax.set_xlabel('time (ms)') if numAxisCol != 0: disp.display(fig2) if not forceNoAnnotation: for fid, isAnnotated in zip(fids, isAnnotatedList): if not isAnnotated: if type(fid) == int or type(fid) == type(''): if newAnnotation or not exp.checkAnnotation( fid, force=False, quiet=True, useBase=useBase): exp.annotate(fid, useBase=useBase) if clearOutput: disp.clear_output() if calcTemperature: for temp, err, label in zip(temps, tempErrs, ['Hor', 'Vert', 'Hor2D', 'Vert2D']): print(label + ' temperature = ' + misc.errString(temp * 1e6, err * 1e6) + 'uk') for fid in fids: if type(fid) == int: expTitle, _, lev = exp.getAnnotation(fid) expTitle = ''.join( '#' for _ in range(lev)) + ' File ' + str(fid) + ': ' + expTitle disp.display(disp.Markdown(expTitle)) with exp.ExpFile(fid) as file: file.get_basic_info() if trackFitCenter: pass #print('Acceleration in Mpix/s^2 = ' + misc.errString(accelFit[1], accelErr[1])) if transferAnalysisOpts is not None and showTferAnalysisPlots: colors, colors2 = misc.getColors( len(transferAnalysisOpts.initLocs()) + 2) #, cmStr=dataColor) pltShape = (transferAnalysisOpts.initLocsIn[-1], transferAnalysisOpts.initLocsIn[-2]) # mp.plotThresholdHists([initThresholds[0][0],initThresholds[1][0]], colors, shape=pltShape) mp.plotThresholdHists([initThresholds[0][0], initThresholds[0][0]], colors, shape=[1, 2]) returnDictionary = { 'images': images, 'fits': hFitParams, 'errs': hFitErrs, 'hSigmas': hSigmas, 'sigmaErrors': hSigmaErrs, 'dataKey': keyPlt, 'hTotalPhotons': hTotalPhotons, 'tempCalc': temps, 'tempCalcErr': tempErrs, 'initThresholds': initThresholds[0], '2DFit': fitParams2D, '2DErr': fitErrs2D, 'bgPics': picsForBg, 'dataLength': datalen } if returnPics: returnDictionary['pics'] = sortedStackedPics return returnDictionary
def simple(self, obj): return display.display(display.Markdown(obj))
'insertedAt', 'updatedAt', 'articleTitle'] corpus = corpus[to_keep] #### superintendent !pip install superintendent !jupyter nbextension enable --py --sys-prefix ipyevents #### from superintendent import ClassLabeller import pandas as pd from IPython import display labelling_widget = ClassLabeller( features=headlines, display_func=lambda x: display.display(display.Markdown("# " + x)), options=['professional', 'not professional'], ) labelling_widget ################3 obj = corpus.to_json(orient='records') jdata = json.loads(obj) #for d in jdata:
def md_print(markdown_text): """ Print Markdown text so that it renders correctly in the cell output. """ IPDisplay.display(IPDisplay.Markdown(markdown_text))
def bold(string): display(iplay.Markdown(string))
def mdown(x): import IPython.display as d d.display(d.Markdown(x))
def display_md(md: str): ip_disp.display(ip_disp.Markdown(md))
def show_markdown(markdown): display.display(display.Markdown(markdown.replace('>', '\\>')))
def print_md(string): """Render a string as Markdown to the notebook's output cell.""" ipd.display(ipd.Markdown(string))
def display_md(md_str: str): escaped_str = re.sub('(?<!\\\\)\$', '\$', md_str) display(ipd.Markdown(escaped_str))
#data = np.random.rand(10,4) # Columns A, B, C, D #columns = [chr(x) for x in range(65,69)] # Create the dataframe data = [rick, scrook, scrook, smbaer, emarder, markram] df = pd.DataFrame(data=data, columns=columns) # Optionally give the dataframe's index a name #df.index.name = "my_index" # Create the markdown string md = tabulate(df, headers='keys', tablefmt='pipe') # Fix the markdown string; it will not render with an empty first table cell, # so if the dataframe's index has no name, just place an 'x' there. md = md.replace('| |', '| %s |' % (df.index.name if df.index.name else 'x')) # Create the Markdown object result = d.Markdown(md) # Display the markdown object (in a Jupyter code cell) result #print('the scores are:',np.min(rick,sharon)) ''' bench = metrics(bench) pm = metrics(pm) hss = metrics(hss) winners = [('rgerkin',rick),('scrook',scrook),('upgoer5_corpus',bench),('the readability of science decr over time', hss), ('peter',pm)] with open('results.p','wb') as f: pickle.dump(winners,f) winners = sorted([(w[1],w[0]) for w in winners]) print(winners) try:
def display_markdown(text: str) -> None: display.display(display.Markdown(text))