def continuous_stagewise_separate():
arrsum = ArraySummary("Continuous vs Stage-wise vs Separate")
with arrsum.create(tex.Section("Overview")):
arrsum.add_tabular(get_tab_data(data, parameters, by_exp_type, by_reward, get_std, name="STD"))
for reward_type in ["minimize", "maximize", "target", "range"]:
grouped_by_reward_params = group_parameters_by(parameters_by_reward_type[reward_type], lambda x: tuple(x[reward_type + "_pred_err"]) if type(x[reward_type + "_pred_err"]) == list else x[reward_type + "_pred_err"])
for k in grouped_by_reward_params:
with arrsum.create(tex.Section(reward_type + str(k))):
groups = group_parameters_by(grouped_by_reward_params[k], lambda x: "stages" if x["continuous"] is None else "continuous")
with arrsum.create(tex.Subsection("Stages")):
sub_groups = group_parameters_by(groups["stages"], lambda x: "separate" if x["separate"] else "joint")
with arrsum.create(tex.Subsubsection("Separate")):
for p in sub_groups["separate"]:
arrsum.add_experiment(p)
with arrsum.create(tex.Subsubsection("Joint")):
for p in sub_groups["joint"]:
arrsum.add_experiment(p)
arrsum.clearpage()
with arrsum.create(tex.Subsection("Continuous")):
sub_groups = group_parameters_by(groups["continuous"], lambda x: "separate" if x["separate"] else "joint")
with arrsum.create(tex.Subsubsection("Separate")):
for p in sub_groups["separate"]:
arrsum.add_experiment(p)
with arrsum.create(tex.Subsubsection("Joint")):
for p in sub_groups["joint"]:
arrsum.add_experiment(p)
arrsum.clearpage()
arrsum.generate_pdf(filepath=path_to_array + "new_impl", clean_tex=False)
def main():
if not os.path.exists("report"):
os.mkdir("report")
global tumor
global max_distance
global e_method
for t in tumors:
print(t)
doc = pylatex.Document('report/results' + t)
doc.append(pylatex.NewPage())
with doc.create(pylatex.Section(t)):
for m in e_methods:
for d in max_distances:
tumor = t
max_distance = d
e_method = m
with doc.create(
pylatex.Subsection("Score = " + m.tag +
", Max dist. = " + str(d))):
do_analysis(doc)
doc.generate_pdf(clean_tex=False)
def subsection(doc: pylatex.Document) -> pylatex.Subsection:
result = pylatex.Subsection('Sensitivity and Cadence')
result.escape = False
result.append(
r"""
Count rates for \ESIS\ are estimated using the expected component throughput from Section~\ref{
subsec:CoatingsandFilters} and the \CCD\ \QE\ listed in Table~\ref{table:prescription}. Line intensities are derived
from \citet{Vernazza78} (V\&R) \roy{\VR} and the \SOHO/\CDS\ \citep{Harrison95} data, and are given in a variety of
solar contexts: \QS, \CHs, and \ARs. The \SI{100}{\percent} duty cycle of \ESIS\ (\S\,\ref{subsec:Cameras}) gives us
the flexibility to use the shortest exposures that are scientifically useful. So long as the shot noise dominates
over read noise (which is true even for our coronal hole estimates at \SI{10}{\second} exposure length), we can stack
exposures without a significant \SNR\ penalty. Table~\ref{table:count} shows that \ESIS\ is effectively shot noise
limited with a \SI{10}{\second} exposure. The signal requirement in Table~\ref{table:scireq} is met by stacking
exposures. Good quality images ($\sim300$ counts) in active regions can be obtained by stacking \SI{30}{\second}
worth of exposures. This cadence is sufficient to observe explosive events, but will not resolve torsional Alfv\'en
waves described in \S\,\ref{sec:ScienceObjectives}. However, by stacking multiple \SI{10}{\second} exposures,
sufficient \SNR\ \emph{and} temporal resolution of torsional Alfv\'en wave oscillations can be obtained. \roy{Just
delete these next three sentences?} \jake{Assuming the table and sentences above have been updated to reflect the
vignetted system, yes}. We also note that the count rates given here are for an unvignetted system which is limited
by the baffling of this design. While not explored here, there is the possibility of modifying the instrument
baffling (\S\,\ref{subsec:AperturesandBaffles}) to increase throughput. Thus, a faster exposure cadence may be
obtained by accepting some vignetting in the system.
"""
)
# result.append(tables.counts.table_old)
result.append(tables.counts.table(doc))
return result
def subsection() -> pylatex.Subsection:
result = pylatex.Subsection('Cameras')
result.escape = False
result.append(figures.cameras.figure())
result.append(
r"""
The \ESIS\ \CCD\ cameras were designed and constructed by \MSFC\ and are the latest in a
series of camera systems developed specifically for use on solar space flight instruments.
The \ESIS\ camera heritage includes those flown on both the \CLASP~\citep{Kano12,Kobayashi12} and \HiC~\citep{Kobayashi2014}.
The \ESIS\ detectors are CCD230-42 \roy{\detectorName} astro-process \CCDs\ from E2V \roy{\detectorManufacturer}.
For each camera, the \CCD\ is operated in a split frame transfer mode with each of the four ports read out by a 16-bit \roy{\detectorAnalogToDigitalBits-bit} A/D
converter.
The central $2048 \times 1024$ \roy{$\detectorPixelsX \times \detectorPixelsY$} pixels of the $2k\times2k$ device are used for imaging, while the outer two regions are
used for storage.
Two \roy{\DetectorNumOverscanColumnWords} overscan columns on either side of the imaging area and eight extra rows in each storage region will monitor read
noise and dark current.
When the camera receives the trigger signal, it transfers the image from the imaging region to the storage regions and
starts image readout.
The digitized data are sent to the \DACS\ through a SpaceWire interface immediately,
one line at a time.
The frame transfer takes $<$\SI{60}{\milli\second} \roy{\detectorFrameTransferTime}, and readout takes \SI{1.1}{\second} \roy{\detectorReadoutTime}.
The cadence is adjustable from 2-\SI{600}{\second} \roy{\detectorExposureLengthRange} in increments of \SI{100}{\milli\second} \roy{\detectorExposureLengthIncrement}, to satisfy the requirement
listed in Table~\ref{table:scireq}.
Because the imaging region is continuously illuminated, the action of frame transfer (transferring the image from the
imaging region to the storage regions) also starts the next exposure without delay.
Thus the exposure time is controlled by the time period between triggers.
Camera 1 \roy{\detectorTriggerIndex} (Fig.~\ref{F-cameras}) generates the sync trigger, which is fed back into Camera 1's \roy{\detectorTriggerIndex's} trigger input and provides
independently buffered triggers to the remaining three cameras.
The trigger signals are synchronized to better than $\pm$\SI{1}{\milli\second} \roy{$\pm$\detectorSynchronizationError}.
Shutterless operation allows \ESIS\ to observe with a \SI{100}{\percent} duty cycle.
The cadence is limited only by the 1.1\,s \roy{\detectorReadoutTime} readout time.
\MSFC\ custom designed the camera board, enclosure, and mounting structure for \ESIS\ to fit the unique packaging
requirements of this experiment (Fig~\ref{F-cameras}).
The front part of the camera is a metal block which equalizes the temperature across the \CCD\ while fastening it in
place.
The carriers of all cameras are connected to a central two-piece copper (\SI{3}{\kilo\gram}) and aluminum
(\SI{1}{\kilo\gram}) thermal reservoir (cold block) by flexible copper cold straps.
The flexible cold straps allow individual cameras to be translated parallel to the optical axis (by means of shims) up
to $\sim$\SI{13}{\milli\meter} \roy{$\sim$\detectorFocusAdjustmentRange} to adjust focus in each channel prior to launch.
The centrally located cold block will be cooled by LN2 \roy{\LN} flow from outside the payload until just before launch.
The LN2 \roy{\LN} flow will be controlled automatically by a Ground Support Equipment (GSE) \roy{\GSE} computer so that all cameras are
maintained above survival temperature but below the target temperature of \SI{-55}{\celsius} \roy{\detectorTemperatureTarget} to insure a negligible dark
current level.
The gain, read noise, and dark current of the four cameras were measured at \MSFC\ using an ${}^{55}$Fe radioactive
source.
Cameras are labeled 1, 2, 3, and 4 \roy{\channelNames} with associated serial numbers SN6, SN7, SN9, and SN10 \roy{\detectorSerialNumbers} respectively in
Fig.~\ref{F-cameras}. Gain ranges from 2.5-\SI{2.6}{e^- \per DN} \roy{\detectorGainRange} in each quadrant of all four cameras.
Table~\ref{T-cameras} lists gain, read noise, and dark current by quadrant for each camera.
The \QE\ of the \ESIS\ \CCDs\ will not be measured before flight.
Similar astro-process \CCDs\ with no AR \roy{antireflection (because AR is already used for active region)} coating are used in the \SXI\ aboard the \GOES\ N and O.
A \QE\ range of 43\% at 583\AA\ \roy{\detectorQuantumEfficiencyHeI\ at \HeI} to 33\% at 630\AA\ \roy{\detectorQuantumEfficiency\ at \OV} is expected for the \ESIS\ \CCDs, based on \QE\ measurements by
\citet{Stern04} for \GOES\ \SXI\ instruments."""
)
# result.append(tables.cameras.table_old())
result.append(tables.cameras.table())
return result
def subsection() -> pylatex.Subsection:
result = pylatex.Subsection('Avionics')
result.escape = False
result.append(
r"""
The \ESIS\ \DACS\ are based on the designs used for both \CLASP~\citep{Kano12,Kobayashi12} and \HiC~\citep{Kobayashi2014}.
The electronics are a combination of \MOTS\ hardware and custom designed components.
The \DACS\ is a 6-slot, 3U, open VPX PCIe architecture conduction cooled system using an AiTech C873 single board
computer.
The data system also include a \MOTS\ PCIe switch card, \MSFC\ parallel interface card, and two \MOTS\ Spacewire cards.
A slot for an additional Spacewire card is included to accommodate two more cameras for the next \ESIS\ flight.
The C873 has a \SI{2.4}{\giga\hertz} Intel i7 processor with \SI{16}{\giga b} of memory.
The operating temperature range for the data system is -40 to +85 C.
The operating system for the flight data system is Linux Fedora 23.
The \DACS\ is responsible for several functions;
it controls the \ESIS\ experiment, responds to timers and uplinks, acquires and stores image data from the cameras,
downlinks a subset of images through telemetry, and provides experiment health and status.
The \DACS\ is housed with the rest of the avionics (power supply, analog signal conditioning system) in a
0.56-\SI{0.43}{\meter} transition section outside of the experiment section.
This relaxes the thermal and cleanliness constraints placed on the avionics.
Custom DC/DC converters are used for secondary voltages required by other electronic components.
The use of custom designed converters allowed additional ripple filtering for low noise."""
)
return result
def sum_sample(self):
"""Summarize the analysis data and plots in a report
"""
for i in range(len(self.infiles)):
self.logger.info('\nCreating pdf for sample {} results.\n'.format(
self.sample[i]))
geometry_options = {
'tmargin': '3cm',
'bmargin': '3cm',
'rmargin': '3cm',
'lmargin': '3cm'
}
doc = px.Document(documentclass='article',
geometry_options=geometry_options)
doc.preamble.append(
px.Command('title',
'Sequencing results for sample ' + self.sample[i]))
doc.preamble.append(px.Command('date', px.NoEscape(r'\today')))
doc.append(px.NoEscape(r'\maketitle'))
with doc.create(px.Section('Genome coverage')):
#include table of results with statistics of coverage
with doc.create(px.Subsection('Coverage results')):
with doc.create(px.Tabular(table_spec='l l')) as table:
with open(self.stats_file, 'r') as stats:
table.add_hline()
stats_data = pd.read_csv(stats, sep='\t')
for num in range(len(stats_data.iloc[0])):
table.add_row([
stats_data.columns[num],
stats_data.iloc[0][num]
])
table.add_hline()
#include coverage plot
with doc.create(px.Figure(position='htb!')) as plot:
plot.add_image(self.cov_plot[i],
width=px.NoEscape(r'\linewidth'))
plot.add_caption(
'Genome coverage for sample ' + self.sample[i] +
'. Calculated using samtools depth with zero-coverage positions included.'
)
#include mismatch plot comparing the sample to the reference
with doc.create(px.Section('Comparison to reference genome')):
with doc.create(px.Figure(position='htb!')) as plot:
plot.add_image(self.basefreq_plot + '_' + self.sample[i] +
'.png',
width=px.NoEscape(r'\linewidth'))
plot.add_caption(
'Mismatch fraction per position for sample ' +
self.sample[i] +
'. Calculated compared to reference {}.'.format(
self.config['folder_locations']['ref_fasta']))
doc.generate_pdf(
filepath=os.path.join(self.outputdir, self.sample_info[i] +
'.Report'))
self.logger.info(
'\nDone creating pdf for sample {} results.\n'.format(
self.sample[i]))
def finalize(self):
doc = util.create_doc(
f"Correlator Fits: {self.ensemble_name} - {self.task_name}")
for operator_set, operator_fits in self.operator_fits.items():
with doc.create(pylatex.Section(str(operator_set))):
for operator, fits in operator_fits.items():
with doc.create(pylatex.Subsection(str(operator))):
for fit, fit_infos in fits.items():
# normal fits
logfile = self.logfile(operator_set, operator,
fit.name)
fit_log = sigmond_info.sigmond_log.FitLog(logfile)
if self.fit_plots:
plotdir = self.fit_plotdir(
operator_set, operator, fit.name)
util.dirGrace2pdf(plotdir)
section_title = f"{fit.name} - Model: {fit.model.short_name}"
with doc.create(
pylatex.Subsubsection(section_title)):
self._add_fits(doc, fit_log, fit.name,
operator_set, fit.ratio,
fit.model.has_gap,
fit.model.has_const)
# tmin fits
if self.tmin_plots:
plotdir = self.tmin_fit_plotdir(
operator_set, operator, fit.name)
util.dirGrace2pdf(plotdir)
tmin_fit_infos = list()
for fit_info in fit_infos['tmin']:
plotfile = self.tmin_fit_plotfile(
operator_set,
fit.name,
fit_info,
extension=util.PlotExtension.pdf)
if os.path.isfile(plotfile):
tmin_fit_infos.append(fit_info)
if len(tmin_fit_infos) == 0:
continue
tmin_fit_infos.sort(
key=lambda fit_info: fit_info.tmax)
section_title = f"$t_{{\\rm min}}$ plots - {fit.name} - Model: {fit.model.short_name}"
with doc.create(
pylatex.Subsubsection(
pylatex.NoEscape(section_title))):
self._add_tmins(doc, tmin_fit_infos,
fit.name, operator_set,
fit.ratio)
results_dir = self.results_dir
os.makedirs(results_dir, exist_ok=True)
filename = os.path.join(results_dir, self.task_name)
util.compile_pdf(doc, filename, self.latex_compiler)
def _export_pdf(self, fname: str):
geometry_options = {"margin": "1in"}
doc = pylatex.Document(geometry_options=geometry_options)
header = pylatex.PageStyle("header")
with doc.create(pylatex.MiniPage(align="c")):
doc.append(
pylatex.HugeText(pylatex.utils.bold("Practice Sessions")))
doc.append(pylatex.LineBreak())
with header.create(pylatex.Foot("L")):
header.append("@2019 Musicavis")
with header.create(pylatex.Foot("R")):
header.append(pylatex.simple_page_number())
doc.preamble.append(header)
doc.change_document_style("header")
for practice in self.practices:
with doc.create(
pylatex.Section(
f"{practice.instrument.name.title()} ({practice.date:%a, %B %m %Y})"
)):
self._add_list_itemize_latex(doc, "Goals",
practice.goals.all())
with doc.create(pylatex.Subsection("Exercises")):
with doc.create(pylatex.Itemize()) as itemize:
for x in practice.exercises.all():
itemize.add_item(
f"{x.name} at {x.bpm_start}-{x.bpm_end}bpm for {x.minutes}m"
)
self._add_list_itemize_latex(doc, "Improvements",
practice.improvements.all())
self._add_list_itemize_latex(doc, "Positives",
practice.positives.all())
with doc.create(pylatex.Subsection("Notes:")):
if practice.notes:
for note in practice.notes.split("\r\n"):
doc.append(note) if note else doc.append(
pylatex.NewLine())
doc.generate_pdf(fname.split(".pdf")[0], clean=True)
def subsection() -> pylatex.Subsection:
result = pylatex.Subsection('Pointing System')
result.escape = False
result.append(r"""
The imaging target will be selected prior to launch, the morning of the day of flight.
During flight, pointing will be maintained by the \SPARCS\ \citep{Lockheed69}.
Images from Camera 1 will be downlinked and displayed in real time on the \SPARCS\ control system console at intervals of
$\sim$\SI{16}{\second} to verify pointing is maintained during flight.""")
return result
def generate_latex(
self, sort_by="graphs"
): # graphs = [] #todo allow for setting which graphs to output
if sort_by == "graphs":
for graph_type, state_dict in self.tex_dict.items():
# todo map graph type to human readable section name
with self.doc.create(pyl.Section(graph_type)):
for state, tex_list in state_dict.items():
with self.doc.create(pyl.Subsection(state)):
[tex_func(doc=self.doc) for tex_func in tex_list]
else:
raise NotImplementedError("Sorting by protein state not implemented")
def subsection() -> pylatex.Subsection:
result = pylatex.Subsection('Optics')
result.escape = False
result.append(r"""
Figure~\ref{fig:schematic}a shows the relative layout of the optics and detectors for a single
\ESIS\ channel.
Here we give specific details of the primary mirror and gratings (Fig.~\ref{fig:schematic}b and \ref{fig:schematic}c, respectively).
The features of the field stop have been described previously in Sec.~\ref{subsec:ESISFeatures}, while the \CCD\ and
cameras are covered in Sec.~\ref{subsec:Cameras}. """)
result.append(figures.schematic.figure())
result.append(r"""
The primary mirror is octagonal in shape.
The octagonal shape of the primary allows dynamic clearance for filter tubes that are arranged radially around the
mirror (\S\,\ref{subsec:CoatingsandFilters}).
The mirror is attached to a backing plate by three \textit{bipods}: thin titanium structures that are flexible in the radial
dimension, perpendicular to the mirror edge, but rigid in the other two dimensions.
The bipods form a kinematic mount, isolating the primary mirror figure from mounting stress.
The mirror will have to maintain its figure under direct solar illumination, so a Corning \ULE\ substrate was used.
The transparency of \ULE, in conjunction with the transparency of the mirror coating in visible and near-IR wavelengths
(\eg, Table~\ref{table:prescription} and \S\,\ref{subsec:CoatingsandFilters}), helps minimize
the heating of the mirror.
Surface figure specifications for the \ESIS\ optics are described in Sec.~\ref{subsec:OptimizationandTolerancing}.
The spherical gratings (Fig.~\ref{fig:schematic}c) re-image light from the field stop to form dispersed images at the
\CCDs.
Each grating is individually mounted to a backing plate in a similar fashion as the primary mirror.
For these much smaller optics, lightweight bipods were photo-chemically etched from thin titanium sheet.
The bipods are bonded to both the grating and backing plate along the three long edges of each grating.
The individual mounts allow each grating to be adjusted in tip and tilt to center the image on the \CCD. """
)
result.append(figures.field_stop_projections.figure())
result.append(r"""
The gratings have a varied line space ruling pattern optimized to provide, in principle,
pixel-limited imaging from the field stop to the \CCDs.
The pitch at the center of the grating is $d_0=\text{\gratingRulingSpacing}$ resulting in a dispersion of
\dispersionDoppler\ at the center of the \OV\ \FOV.
The groove profile is optimized for the $m=1$ order, so that each grating serves only a single \CCD.
The modeled grating groove efficiency in this order is \SI{36}{\percent} \roy{We said \SI{39}{\percent} above, need to
find out which it is, I get \gratingGrooveEfficiency} at \OV.
Figure specification and groove profile are not well controlled near the edges of the gratings. Therefore,
\jake{an uncoated section of mirror was left around the edge of the grating when applying the multilayer coating,
minimizing reflection in EUV.} Fig.~\ref{fig:schematic}c
The \ESIS\ passband is defined through a combination of the field stop, the grating dispersion, and the \CCD\ size.
The passband includes the \HeI\ spectral line through \MgXion\ (\MgXwavelength\ and \MgXdimWavelength) to \OV.
Figure~\ref{fig:projections} shows where images of each of the strong spectral lines will fall on the \CCD.
The instrument dispersion satisfies the spectral resolution requirement in Table~\ref{table:scireq} and ensures that the
spectral images are well-separated; Figure~\ref{fig:projections} shows that \HeI\ will be completely
separated from the target \OV\ line.""")
return result
def _add_string_notes(document: pylatex.Document, texts: dict,
images: dict) -> None:
document.append(
pylatex.Subsection(
title=texts["strings"]["title"],
label=False,
numbering=False,
))
document.append(
pylatex.Subsubsection(
title=texts["strings"]["subtitle0"],
label=False,
numbering=False,
))
document.append(texts["strings"]["text0"])
document.append(
pylatex.Subsubsection(
title=texts["microtonal_notation"]["title"],
label=False,
numbering=False,
))
document.append(texts["microtonal_notation"]["text0"])
document.append(_make_img(images["twelfth_tone_explanation"]))
document.append(texts["microtonal_notation"]["text1"])
for instrument in ("violin", "viola", "cello"):
document.append(_make_img(images["scale_{}".format(instrument)]))
document.append(
_make_img(
images["scale_{}_artificial_harmonics".format(instrument)]))
document.append(
pylatex.Command("hspace", arguments=[pylatex.NoEscape("5mm")]))
document.append(texts["microtonal_notation"]["text2"])
document.append(
pylatex.Subsubsection(
title=texts["strings"]["subtitle1"],
label=False,
numbering=False,
))
document.append(_make_img(images["ornamentation"], width=0.25))
document.append(texts["strings"]["text1"])
document.append(_make_img(images["glissando"], width=0.28))
document.append(texts["strings"]["text2"])
def section() -> pylatex.Section:
result = pylatex.Section('Mission Profile')
result.escape = False
result.append(r"""
\ESIS\ will be launched aboard a sub-orbital Terrier Black Brant sounding rocket from White Sands Missile Range.
The experiment is currently scheduled for launch in August, 2019.
Trajectory will follow a roughly parabolic path, with $>$\SI{270}{\second} solar observing time above
\SI{160}{\kilo\meter}.
\ESIS\ will begin continuously taking exposures at a fixed cadence immediately after launch, terminating just before the
payload impacts the upper atmosphere.
Exposure length will be determined by the target selected for launch day.
Exposures taken while the payload shutter door is closed ($<$ \SI{160}{\kilo\meter}) will be used for dark calibration.
Data will be stored on board and downloaded after recovery, however a limited amount of data will be transmitted to the
ground station via high speed telemetry as a safeguard against payload loss or destruction.
A parachute will slow the descent of the payload after it enters the atmosphere, and recovery will be accomplished by
helicopter after the payload is located on the ground.""")
with result.create(
pylatex.Subsection(
pylatex.NoEscape('\ESIS\ Mission Update'))) as mission_update:
mission_update.escape = False
mission_update.append(r"""
Since the time of writing \ESIS\ launched and was recovered successfully from White Sands Missile Range on
September 30, 2019.
Unfortunately, due to failure of the mechanical shutter, no \MOSES\ data was obtained during this flight.
A paper is forthcoming that will document the \ESIS\ instrument in its as-flown configuration~\citep{Courrier_inprep}.
A companion paper will describe \ESIS\ first results~\citep{Parker_inprep}.
Two significant changes, one to the \ESIS\ instrument and one to our alignment procedures, were made prior to launch and
are summarized below.
The transfer from visible to \EUV\ grating alignment was completed by an alternative means.
The apparatus described by~\citet{Johnson18} was not able to maintain sufficient repeatability during test runs on
diffraction grating surfaces.
To maintain the launch schedule, a phase shifting interferometer was used to transfer the alignment of the visible
gratings to the \EUV\ flight gratings.
A trade study was conducted, and it was decided to remove the primary aperture stop. The advantage was an increase in
sensitivity.
The disadvantage was to sacrifice the unvignetted design described in Section \ref{subsec:AperturesandBaffles}.
The effective aperture is increased by a factor of 1.7 to 2.7 as a function of \FOV\ in the radial dimension.
The corresponding signal gradient is oriented along the dispersion direction of each channel;
vignetting increases (and signal decreases) when moving towards blue wavelengths
(\ie\,moving to the left in Figure~\ref{fig:projections}).
This gradient is due almost entirely to vignetting by the central obscuration, and is linear across the entire \FOV.
The principal challenge is that the images cannot be corrected directly;
rather, since the gradient is repeated for each of the overlapping spectral line images, the vignetting can only be
accounted for by forward modeling.
Since forward modeling is required for all of the inversion procedures under consideration for \ESIS\ data analysis, the
vignetting was deemed low risk to the mission science.""")
return result
def subsection() -> pylatex.Subsection:
result = pylatex.Subsection('Vignetting')
result.escape = False
result.append(r"""
The original design of \ESIS\ had no vignetting thanks to an stop placed at the primary mirror that was designed to
perfectly fill the grating with the same amount of light for each point in the \FOV.
This is the \ESIS\ design that was used for the optimization procedure of the grating parameters described in
Section~\ref{subsec:OptimizationandTolerancing}, for example.
All other results described in the paper use the fully-open system.
Before flight, we decided to remove the primary aperture stop to increase the sensitivity of the instrument at the
expense of introducing vignetting to the \ESIS\ \FOV.
This was acceptable since the vignetting was found to be a simple linear field as shown in Figure~\ref{fig:vignetting},
and could be removed in the post-processing phase.""")
result.append(figures.vignetting.figure())
return result
def addPlotsToPDF(self, doc, data_files, operators, name):
obs_handler, _ = util.get_obs_handlers(data_files, self.bins_info,
self.sampling_info)
corr_plotsdir = self.correlator_plotdir(name)
energy_plotsdir = self.energy_plotdir(name)
util.dirGrace2pdf(corr_plotsdir)
util.dirGrace2pdf(energy_plotsdir)
off_diag_corrs = list()
for op_src in operators:
for op_snk in operators:
if op_src == op_snk:
continue
corr = sigmond.CorrelatorInfo(op_snk.operator_info,
op_src.operator_info)
if not self.data_handler.hasCorrelator(corr):
continue
off_diag_corrs.append(corr)
with doc.create(pylatex.Subsection("Diagonal Correlators")):
for operator in operators:
corr = sigmond.CorrelatorInfo(operator.operator_info,
operator.operator_info)
if self.data_handler.hasCorrelator(corr):
with doc.create(pylatex.Subsubsection(str(operator))):
util.add_correlator(doc, self, corr, name, obs_handler)
if self.off_diagonal and off_diag_corrs:
with doc.create(pylatex.Subsection("Off-Diagonal Correlators")):
for corr in off_diag_corrs:
with doc.create(pylatex.Subsubsection(corr.corr_str())):
util.add_correlator(doc, self, corr, name, obs_handler)
def subsection() -> pylatex.Subsection:
result = pylatex.Subsection('Mechanical')
result.escape = False
result.append(r"""
\ESIS\ and \MOSES\ are mounted on opposite sides of a composite optical table structure originally developed for the
\SPDE~\citep{Bruner95lock}.
The layered carbon fiber structure features a convenient, precisely coplanar array of threaded inserts with precision
counterbores.
The carbon fiber layup is designed to minimize the longitudinal coefficient of thermal expansion.
The optical table is housed in two \SI{0.56}{\meter} diameter skin sections, with a total length of \SI{3}{\meter}.
A ball joint and spindle assembly on one end and flexible metal aperture plate on the other hold the optical table in
position inside the skin sections.
The kinematic mounting system isolates the optical table from bending or twisting strain of the skins."""
)
return result
def subsection() -> pylatex.Subsection:
result = pylatex.Subsection('Magnetic Reconnection Events')
result.escape = False
result.append(r"""
Magnetic reconnection describes the re-arrangement of the magnetic topology wherein magnetic energy
is converted to kinetic energy resulting in the acceleration of plasma particles.
Reconnection is implicated in many dynamic, high energy solar events.
Solar flares are a well studied example (\eg\,\citet{Priest02} and the references therein), however we have little hope
of pointing in the right place at the right time to observe a significant flare event in a rocket flight lasting only
five minutes.
Instead, we will search for signatures of magnetic reconnection in \TR\ spectral lines.
%label to track table 1 references
\phantomsection
\label{t1_2}
A particular signature of reconnection in the \TR\ is the explosive energy release by ubiquitous, small scale events.
These \EEs\ are characterized as spatially compact ($\approx$\SI{1.5}{\mega\meter} length~\citep{Dere94}) line
broadenings on the order of \SI{100}{\kilo\meter\per\second}~\citep{Dere91}.
They are observed across a range of \TR\ emission lines that span temperatures of \SI{20000}{}--\SI{250000}{\kelvin}
(C\,\textsc{ii}--O\,\textsc{v})~\citep{1994Moses}.
The typical lifetime of an \EE\ is 60-\SI{90}{\second}~\citep{1994Moses,Dere94,Dere91}.
Due to their location near quiet sun magnetic network elements, and the presence of supersonic flows near the Alfv\`en
speed, \citet{Dere91} first suggested that \EEs\ may result from the model of fast Petschek~\citep{Petschek64}
reconnection.
The spectral line profile of \EEs\ may indicate the type of reconnection that is occurring in the \TR\
(\eg\,\citet{Rust17}).
For example, the Petschek model of reconnection predicts a `bi-directional jet' line profile with highly Doppler
shifted wings, but little emission from the line core~\citep{Innes99}.
\citet{Innes15} developed a reconnection model resulting from a plasmoid instability~\citep{Bhattacharjee09}.
In contrast to the bi-directional jet, this modeled line profile has bright core emission and broad wings.
Both types of profile are seen in slit spectrograph data (\eg, \citet{Innes97,Innes15}, and the references therein),
however \MOSES\ observed \EEs\ with more complicated morphologies than either of these two models
suggest~\citep{Fox10,Rust17}.
It is unclear whether the differing observations are a function of wavelength and temperature, a result of a limited
number of observations, or because the morphology of the event is difficult to ascertain from slit spectrograph data.
%label to track table 1 references
\phantomsection
\label{t1_01}
\ESIS\ will observe magnetic reconnection in the context of \EEs, by extending the technique pioneered by \MOSES\ to
additional \TR\ lines.
Explosive events are well suited to sounding rocket observations;
a significant portion of their temporal evolution can be captured in $>$\SI{150}{\second} (\eg\,the analysis by
\citet{Rust17}) and they are sufficiently common to provide a statistically meaningful sample in a 5-minute rocket
flight (\eg,~\citet{Dere89,Dere91}).
In similarity with \MOSES, we seek a \TR\ line for \ESIS\ that is bright and well enough isolated from neighboring
emission lines so as to be easily distinguished.""")
return result
def _add_setup_notes(document: pylatex.Document, texts: dict,
images: dict) -> None:
document.append(
pylatex.Subsection(
title=texts["setup"]["title"],
label=False,
numbering=False,
))
document.append(texts["setup"]["text0"])
document.append(_make_img(images["stage-setup"], width=0.8))
document.append(texts["setup"]["text1"])
document.append(
pylatex.Center(data=[
_make_img(
images["transducer"], width=0.3, position="h", add_figure=True)
]))
def subsection() -> pylatex.Subsection:
result = pylatex.Subsection('Distortion')
result.escape = False
result.append(r"""
The distortion is due to two factors: first, the tilt of the detector as needed to maintain good focus over the \FOV
\citep{Poletto04}; second, the anamorphic magnification of the grating (see \cite{Schweizer1979})."""
)
result.append(r"""
\begin{equation}
\begin{split}
\left(x', y'\right) &= \C + \C_x x + \C_y y + \C_\lambda \lambda \\
&+ \C_{xx} x^2 + \C_{xy} x y + \C_{y \lambda} x \lambda \\
&+ \C_{yy} y^2 + \C_{y \lambda} y \lambda + \C_{\lambda \lambda} \lambda^2
\end{split}
\end{equation}""")
result.append(tables.distortion.table())
result.append(figures.distortion.figure())
result.append(figures.distortion_residual.figure())
return result
def finalize(self):
if self.write_operators and os.path.isfile(self.op_yaml_file):
os.remove(self.op_yaml_file)
self.data_handler.findAveragedData()
doc = util.create_doc(
f"Averaged Correlators and Effective Energies: {self.ensemble_name} - {self.task_name}"
)
for channel in self.averaged_channels:
data_files = self.data_files + self.data_handler.getChannelDataFiles(
channel)
operators = self.data_handler.getChannelOperators(channel)
if self.write_operators:
operator_info.operator_set.write_operators(
self.op_file(channel), operators, True, False)
operator_info.operator_set.write_operators_to_yaml(
self.op_yaml_file, repr(channel), operators, True)
result_operators, original_operators, coefficients = self.final_info[
channel]
with doc.create(pylatex.Section(str(channel))):
with doc.create(pylatex.Subsection("Operators averaged")):
for result_operator in result_operators:
with doc.create(pylatex.Itemize()) as itemize:
itemize.add_item(result_operator.op_str())
with doc.create(pylatex.Itemize()) as sub_itemize:
for original_operator in original_operators:
the_op = original_operator[result_operator]
the_coeff = coefficients.get(the_op, 1.)
sub_itemize.add_item(
f"{the_coeff} {the_op.op_str()}")
doc.append(pylatex.NoEscape(r"\newpage"))
self.addPlotsToPDF(doc, data_files, operators, repr(channel))
filename = os.path.join(self.results_dir,
util.str_to_file(self.task_name))
util.compile_pdf(doc, filename, self.latex_compiler)
def subsection() -> pylatex.Subsection:
result = pylatex.Subsection('Science Requirements')
result.escape = False
result.append(
r"""
\ESIS\ will investigate two science targets;
reconnection in explosive events, and the transport of mass and energy through the transition region.
The latter may take many forms, from \MHD\ waves of various modes to \EUV\ jets or macro-spicules.
To fulfill these goals, \ESIS\ will obtain simultaneous intensity, Doppler shift and line width images of the \OV\ line
in the solar transition region at rapid cadence.
This is a lower \TR\ line (\SI{.25}{\mega\kelvin}).
The bright, optically thin \OVion\ emission line is well isolated except for the two coronal \MgXion\ lines.
These coronal lines can be viewed as contamination or as a bonus;
we expect that with the \numChannelsWords\ \ESIS\ projections it will be possible to separate the \OVion\ emission from
that of \MgXion.
From the important temporal, spatial, and velocity scales referenced Sections~\ref{subsec:MagneticReconnectionEvents}
and \ref{subsec:EnergyTransfer} we define the instrument requirements in Table~\ref{table:scireq} that are needed to
meet our science goals."""
)
result.append(figures.bunch.figure())
result.append(tables.requirements.table())
return result
def _add_keyboard_notes(document: pylatex.Document, texts: dict,
images: dict) -> None:
document.append(
pylatex.Subsection(
title=texts["keyboard"]["title"],
label=False,
numbering=False,
))
document.append(texts["keyboard"]["text0"])
document.append(pylatex.NoEscape(r"\vspace{5mm}"))
table = pylatex.Tabular(r" l | l | p{9cm} ")
# table.add_hline()
table.add_row(*tuple(
pylatex.MediumText(column_title) for column_title in (
"Bereich",
"Beschreibung der Klänge",
"verwendete Lautsprecher",
)))
for zone_idx in range(3):
table.add_hline()
table.add_row(
_make_img(images["zone_{}".format(zone_idx)],
width=0.22,
add_figure=False),
texts["keyboard"]["zone{}sound".format(zone_idx)],
texts["keyboard"]["zone{}speaker".format(zone_idx)],
)
# table.add_hline()
document.append(pylatex.Table(data=table, position="h!"))
document.append(texts["keyboard"]["text1"])
# document.append(pylatex.NoEscape(r"\vspace{3mm}"))
document.append(texts["keyboard"]["text2"])
def describeOneAdditiveComponent(self, term):
"""
Generate a subsection that describes one additive component.
:param term: term to be analysed
:type term: integer
"""
ker, cum = self.kers[term - 1], self.cums[term - 1]
data = ker.data
kdims = ker.getActiveDims()
error = cum.error()
if term > 1: delta = self.cums[term - 2].error() - error
nlml = cum.getNLML()
doc = self.doc
with doc.create(pl.Subsection("Component {0}".format(term))):
if term == 1:
s = r"With only one additive component, the GP classifier can achieve " \
+ r"a cross-validated classification error of {0:.2f}\%. ".format(error * 100) \
+ r"The corresponding negative log marginal likelihood is {0:.2f}. ".format(nlml)
doc.append(ut.NoEscape(s))
s = r"This component operates on " + dims2text(kdims, data) + ", " \
+ r"as shown in Figure {0}. ".format(self.fignum)
doc.append(ut.NoEscape(s))
else:
s = r"With {0} additive components, the cross-validated classification error ".format(term) \
+ r"can be reduced by {0:.2f}\% to {1:.2f}\%. ".format(delta * 100, error * 100) \
+ r"The corresponding negative log marginal likelihood is {0:.2f}. ".format(nlml)
doc.append(ut.NoEscape(s))
s = r"The additional component operates on " + dims2text(kdims, data) + ", " \
+ r"as shown in Figure {0}. ".format(self.fignum)
doc.append(ut.NoEscape(s))
self.makeInteractionFigure(ker, cum, term)
def subsection() -> pylatex.Subsection:
result = pylatex.Subsection(
pylatex.NoEscape(r'Limitations of the \MOSES\ Design'))
result.escape = False
result.append(r"""
The \MOSES\ design features a single concave diffraction grating forming images on three \CCD\
detectors~\citep{Fox10} (Figure~\ref{fig:mosesSchematic}).
The optical path is folded in half by a single flat secondary mirror (omitted in Figure~\ref{fig:mosesSchematic}).
Provided that the three cameras are positioned correctly, this arrangement allows the entire telescope to be brought
into focus using only the central (undispersed) order and a visible light source.
Unfortunately this design uses volume inefficiently for two reasons.
First, the lack of magnification by the secondary mirror limits the folded length of the entire telescope to be no less
than half of the \SI{5}{\meter} focal length of the grating~\citep{Fox10,Fox11}.
Second, the dispersion of the instrument is controlled by the placement of the cameras.
To achieve the maximum dispersion of \SI{29}{\kilo\meter\per\second}~\citep{Fox10}, the outboard orders are imaged as
far apart as possible in the $\sim\text{\skinDiameter}$ diameter envelope of the rocket payload.
The resulting planar dispersion poorly fills the cylindrical volume of the payload, leaving much unused space along the
orthogonal planes.""")
result.append(figures.schematic_moses.figure())
result.append(r"""
Furthermore, the monolithic secondary, though it confers the focus advantage noted above, does not
allow efficient placement of the $m=\pm1$ \CCDs.
For all practical purposes, the diameter of the payload (\skinDiameter) can only accommodate three diffraction
orders ($m=-1, 0, +1$).
Therefore, \textit{\MOSES\ can only collect, at most, three pieces of information at each point in the field of view.}
From this, it is not reasonable to expect the reconstruction of more than three degrees of freedom for each spectral line,
except in the case very compact, isolated features such as those described by \citet{Fox10} and \citet{Rust17}.
Consequently, it is a reasonable approximation to say that \MOSES\ is sensitive primarily to spectral line intensities,
shifts, and widths \citep{KankThom01}.
With any tomographic apparatus, the degree of detail that can be resolved in the object depends critically on the
number of viewing angles~\citep{Kak88,Descour97,Hagen08}.
So it is with the spectrum we observe with \MOSES: more dispersed images are required to confer sensitivity to finer
spectral details such as higher moments of the spectral line shape.
A related issue stems from the use of a single grating, with a single dispersion plane.
Since the solar corona and transition region are structured by magnetic fields, the scene tends to be dominated by
\sout{field aligned} \roy{field-aligned} structures such as loops~\citep{Rosner78,Bonnet80}.
When the \MOSES\ dispersion direction happens to be aligned nearly perpendicular to the magnetic field, filamentary
structures on the transition region serve almost as spectrograph slits unto themselves.
The estimation of Doppler shifts then becomes a simple act of triangulation, and broadenings are also readily
diagnosed~\citep{Fox10,Courrier18}.
A double-peaked profile can also be observed with sufficiently isolated features~\citep{Rust17}.
Unfortunately, solar magnetic fields in the transition region are quite complex and do not have a global preferred
direction.
In cases where the field is nearly parallel to the instrument dispersion, spectral shifts and broadenings are not
readily apparent.
The single diffraction grating also leads to a compromise in the optical performance of the instrument.
Since the \MOSES\ grating forms images in three orders simultaneously, \sout{aberration cannot be simultaneously optimized for}
\roy{there aren't enough degrees of freedom in the optical system to achieve sub-pixel aberrations in} all three of
those spectral orders.
A result of this design is that the orientations (\ie\,the major axis) of the \PSF\ varies
order to order~\citep{Rust17}.
During the first mission, \MOSES\ was flown with a small amount of defocus~\citep{Rust17}, which exacerbated the
inter-order \PSF\ variation and caused the individual \PSFs\ to span several
pixels~\citep{Rust17,Atwood18}.
The combination of these two effects results in spurious spectral features that
require additional consideration~\citep{Atwood18} and further increase the complexity of the inversion
process~\citep{Rust17,Courrier18}.
\jake{Another complication with \MOSES\ is that the image in each spectral order contains a different combination of
spatial and spectral information.
This stems from the fact that \MOSES\ lacks a field stop to define a wavelength independent \FOV\ and that it uses an
undispersed channel. In this configuration, intensity from wavelengths off the primary observing wavelength,
but within the \MOSES\ passband, will be imaged in the zero order, but may be dispersed off the detector in the outboard orders.
In the opposite sense, features outside of the \FOV\ of the zeroth order may be dispersed onto either of the outboard
order detectors. \citet{Parker2022} compared synthetic \MOSES\ images to the real data and found that approximately
ten percent of the intensity in the zeroth order image originated from more than ten dims lines in the \MOSES\ passband,
most of which are too dim to be visible in the dispersed images.
This study revealed that undispered channels, although attractive due to their lack of spatial-spectral ambiguity,
can provide misleading intensity information which limits their utility in inversion without careful forward modeling.
Also, that the \FOV\ should be clearly defined, and the same, for each wavelength so that the spectral contribution to a
given pixel is clearly defined.}
Finally, the exposure cadence of \MOSES\ is hindered by an $\sim$\SI{6}{\second} readout time for the \CCDs~\citep{
Fox11}.
The observing interval for a solar sounding rocket flight is very short, typically about five minutes.
Consequently, every second of observing time is precious, both to achieve adequate exposure time and to catch the
full development of dynamical phenomena.
The \MOSES\ observing duty cycle is $\sim$\SI{50}{\percent} since it is limited by the readout time of its \CCDs.
Thus, valuable observing time is lost. The readout data gap compelled us to develop a \MOSES\ exposure sequence with
exposures ranging from $0.25$-\SI{24}{\second}, a careful trade-off between deep and fast exposures.
In summary, our experience leads us to conclude that the \MOSES\ design has the following primary limitations:
\begin{enumerate}
\item inefficient use of volume \label{item-length} %(x and y direction)
\item dispersion constrained by payload dimensions \label{item-disp_con}
\item too few dispersed images (orders) \label{item-orders}
\item single dispersion plane \label{item-dispersion}
\item insufficient degrees of freedom to control aberrations \label{item-PSF}
\item poorly-defined and wavelength dependent \FOV\ \label{item-FOV}
\item misleading intensity in $m=0$ order image
\item low duty cycle \label{item-CAD}
\end{enumerate}
In designing \ESIS, we have sought to improve upon each of these points.""")
return result
def finalize(self):
doc = util.create_doc(f"Rotated Correlators and Effective Energies: {self.task_name} - {self.ensemble_name}")
for operator_basis in self.operator_bases:
logfile = self.logfile(repr(operator_basis))
rotation_log = sigmond_info.sigmond_log.RotationLog(logfile)
if rotation_log.failed:
logging.warning(f"Rotation {operator_basis.name} failed")
continue
corr_plotsdir = self.correlator_plotdir(operator_basis)
energy_plotsdir = self.energy_plotdir(operator_basis)
util.dirGrace2pdf(corr_plotsdir)
util.dirGrace2pdf(energy_plotsdir)
data_files = self.data_handler.getRotatedDataFiles(operator_basis)
obs_handler, _ = util.get_obs_handlers(data_files, self.bins_info, self.sampling_info)
with doc.create(pylatex.Section(f"{operator_basis.channel!s} - {operator_basis.name}")):
with doc.create(pylatex.Subsection("Rotation Info")):
with doc.create(pylatex.Center()) as centered:
with centered.create(
pylatex.LongTabu("X[c]|X[c]|X[c]|X[c]|X[c]|X[3,c]|X[3,c]|X[3,c]|X[3,c]|X[3,c]",
to=r"\linewidth")) as param_table:
header_row = [
pylatex.NoEscape(r"$N_{op}$"),
pylatex.NoEscape(r"$N_{\text{d}}$"),
pylatex.NoEscape(r"$\tau_N$"),
pylatex.NoEscape(r"$\tau_0$"),
pylatex.NoEscape(r"$\tau_D$"),
pylatex.NoEscape(r"$\xi_{cn}$ (max)"),
pylatex.NoEscape(r"$\xi_{cn}^C$ (input)"),
pylatex.NoEscape(r"$\xi_{cn}^C$ (retain)"),
pylatex.NoEscape(r"$\xi_{cn}^G$ (input)"),
pylatex.NoEscape(r"$\xi_{cn}^G$ (retain)"),
]
param_table.add_row(header_row, mapper=[pylatex.utils.bold])
param_table.add_hline()
param_table.end_table_header()
value_row = [
operator_basis.num_operators,
operator_basis.num_operators - rotation_log.number_levels,
operator_basis.pivot_info.norm_time,
operator_basis.pivot_info.metric_time,
operator_basis.pivot_info.diagonalize_time,
operator_basis.pivot_info.max_condition_number,
rotation_log.metric_condition(False),
rotation_log.metric_condition(True),
rotation_log.matrix_condition(False),
rotation_log.matrix_condition(True),
]
param_table.add_row(value_row)
doc.append(pylatex.NoEscape(r"\textbf{Metric Null Space Check:} " + \
rotation_log.metric_null_space_message))
with doc.create(pylatex.Subsubsection("Input Operators")):
with doc.create(pylatex.Center()) as centered:
with centered.create(
pylatex.LongTabu("X[2,c] X[c] X[c]", row_height=1.5)) as op_table:
header_row = [
"Operator",
pylatex.NoEscape(r"$\delta C(\tau_0)$"),
pylatex.NoEscape(r"$\delta C(\tau_D)$")
]
op_table.add_row(header_row, mapper=[pylatex.utils.bold])
op_table.add_hline()
op_table.end_table_header()
for op, errors in rotation_log.diagonal_correlator_errors.items():
row = [
op,
errors.metric,
errors.matrix,
]
op_table.add_row
op_table.add_row(row)
with doc.create(pylatex.Subsubsection("Diagonal Deviations From Zero")):
with doc.create(pylatex.Center()) as centered:
with centered.create(
pylatex.LongTabu("X[c] X[4,c] X[3,c] X[3,c] X[3,c] X[3,c] X[2,c]")) as deviation_table:
header_row = [
"time",
pylatex.NoEscape(r"$\delta 0_{max}$"),
pylatex.NoEscape(r"$\% > 1 \sigma$"),
pylatex.NoEscape(r"$\% > 2 \sigma$"),
pylatex.NoEscape(r"$\% > 3 \sigma$"),
pylatex.NoEscape(r"$\% > 4 \sigma$"),
"Status",
]
deviation_table.add_row(header_row, mapper=[pylatex.utils.bold])
deviation_table.add_hline()
deviation_table.end_table_header()
for time, deviation in rotation_log.deviations_from_zero.items():
row = [
time,
deviation.max,
deviation.one,
deviation.two,
deviation.three,
deviation.four,
deviation.status,
]
deviation_table.add_row(row)
doc.append(pylatex.NoEscape(r"\newpage"))
operators = self.data_handler.getRotatedOperators(operator_basis)
with doc.create(pylatex.Subsection("Correlators/Effective Energies")):
for operator in operators:
with doc.create(pylatex.Subsubsection(str(operator))):
corr = sigmond.CorrelatorInfo(operator.operator_info, operator.operator_info)
util.add_correlator(doc, self, corr, operator_basis, obs_handler)
results_dir = self.results_dir
os.makedirs(results_dir, exist_ok=True)
filename = os.path.join(results_dir, self.task_name)
util.compile_pdf(doc, filename, self.latex_compiler)
def _add_data(doc: pl.Document, ds: Dataset, nr: NonRedundantization,
meth: MLMethod):
name = f'{ds.name}_{nr.name}_{meth.name}'
directory = ds.name
aVp_graph = f'{name}.jpg'
angle_dist_graph = f'{name}_angledistribution.jpg'
error_dist_graph = f'{name}_errordistribution.jpg'
sqerror_graph = f'{name}_sqerror_vs_actual.jpg'
stats_csv_all = f'{name}_stats_all.csv'
stats_csv_out = f'{name}_stats_out.csv'
actualVpred_file = os.path.join(directory, aVp_graph)
ang_dist_file = os.path.join(directory, angle_dist_graph)
error_dist_file = os.path.join(directory, error_dist_graph)
sqerror_file = os.path.join(directory, sqerror_graph)
df_all = pd.read_csv(os.path.join(directory, stats_csv_all))
df_out = pd.read_csv(os.path.join(directory, stats_csv_out))
with doc.create(pl.Section(f'Method: {ds.name}, {nr.name}, {meth.name}')):
with doc.create(pl.Subsection('Summary of method:')):
doc.append(f'Dataset: {ds.name}')
doc.append(f'\nNon-redundantization: {nr.name}')
doc.append(f'\nType of machine learning used: {meth.name}')
with doc.create(pl.Subsection('Summary of the data:')):
with doc.create(pl.Figure(position='!htbp')) as actualVpred:
actualVpred.add_image(actualVpred_file, width='300px')
actualVpred.add_caption(
'Graph showing the predicted packing angle against the actual packing angle, when using the above specified methods of non-redundetization and machine learning.'
)
with doc.create(pl.Table(position='!htbp')) as table:
table.add_caption('Summary of results for all data')
table.append(pl.Command('centering'))
table.append(pl.NoEscape(df_all.to_latex(escape=False)))
with doc.create(pl.Table(position='!htbp')) as table:
table.add_caption('Summary of results for outliers.')
table.append(pl.Command('centering'))
table.append(pl.NoEscape(df_out.to_latex(escape=False)))
with doc.create(pl.Figure(position='!htbp')) as graphs:
with doc.create(
pl.SubFigure(position='!htbp',
width=pl.NoEscape(
r'0.30\linewidth'))) as ang_dist_graph:
ang_dist_graph.add_image(ang_dist_file,
width=pl.NoEscape(r'\linewidth'))
ang_dist_graph.add_caption(
'Frequency distribution of the packing angle.')
with doc.create(
pl.SubFigure(position='!htbp',
width=pl.NoEscape(
r'0.33\linewidth'))) as error_dist_graph:
error_dist_graph.add_image(error_dist_file,
width=pl.NoEscape(r'\linewidth'))
error_dist_graph.add_caption(
'Distribution of errors calculated as the difference between the predicted and actual interface angle.'
)
with doc.create(
pl.SubFigure(position='!htbp',
width=pl.NoEscape(
r'0.33\linewidth'))) as sqerror_graph:
sqerror_graph.add_image(sqerror_file,
width=pl.NoEscape(r'\linewidth'))
sqerror_graph.add_caption(
'Squared error in predicted packing angle against actual packing angle.'
)
graphs.add_caption('Graphs for further metrics.')
def subsection(doc: kgpy.latex.Document) -> pylatex.Subsection:
result = pylatex.Subsection('Optimization and Tolerancing')
result.escape = False
result.append(r"""
The science resolution requirement of \angularResolutionRequirement (Table~\ref{table:scireq}) was
flowed down to specifications for the \ESIS\ optics.
To ensure that \ESIS\ meets this requirement, an imaging error budget was developed to track parameters that
significantly influence instrument resolution.
The budget is roughly divided into two categories;
the first includes `variable' parameters that can be directly controlled (\eg, the figure and finish of the optics,
grating radius and ruling, placement of the elements in the system, and the accuracy to which the instrument is
focused).
The second category consists of `fixed' contributions (\eg, \CCD\ charge diffusion, pointing stability, and diffraction
from the entrance aperture).
In this sub-section we describe the optimization of the first category to balance the contributions of the second.
Figure and surface roughness specifications for the primary mirror and gratings were developed first by a rule of thumb
and then validated through a Fourier optics based model \roy{Fourier-optics-based model} and Monte Carlo simulations.
Surface figure errors were randomly generated, using a power law distribution in frequency.
The model explored a range of power spectral distributions for the surface figure errors, with power laws ranging from
0.1 to 4.0.
For each randomly generated array of optical figure errors, the amplitude was adjusted to yield a target \MTF\
degradation factor, as compared to the diffraction limited \roy{diffraction-limited} \MTF.
For the primary mirror, the figure of merit was a \MTF\ degradation of 0.7 \roy{\primaryMtfDegradationFactor} at \angularResolutionRequirement\ resolution.
Though the grating is smaller and closer to the focal plane, it was allocated somewhat more significant \MTF\
degradation of 0.6 \roy{\gratingMtfDegradationFactor} based on manufacturing capabilities.
The derived requirements are described in table~\ref{table:error}.
Note that this modeling exercise was undertaken before the baffle designs were finalized.
The estimated diffraction \MTF\ and aberrations were therefore modeled for a rough estimate of the \ESIS\ single sector
aperture.""")
# result.append(tables.surface_error.table_old())
result.append(tables.surface_error.table())
result.append(r"""
The initial grating radius of curvature, $R_g$, and ruling pattern of the \ESIS\ gratings were
derived from the analytical equations developed by \citet{Poletto04} for stigmatic spectrometers.
A second order polynomial describes the ruling pattern,
\begin{equation} \label{Eq-d}
d = d_0 + d_1 r + d_2 r^2 \, ,
\end{equation}
where $r$ runs radially outward from the optical axis with its origin at the center of the grating \roy{shouldn't we be talking about $x$ here?}
(Fig.~\ref{fig:schematic}c).
The parameters of Equation~\ref{Eq-d} and $R_g$ were chosen so that the spatial and spectral focal curves intersect at
the center of the O\,\textsc{v} \roy{\OV} image on the \CCD.
Starting from the analytically-derived optical prescription, a model of the system was developed in ray-trace \roy{raytrace} software.
Since the instrument is radially symmetric, only one grating and its associated lightpath was analyzed. \roy{delete previous sentence, all lightpaths were analyzed}
In the ray trace model, $R_g$, $d_1$, $d_2$, grating cant angle, \CCD\ cant angle, and focus position were then
optimized to minimize the RMS spot at select positions in the O\,\textsc{v} \roy{\OV} \FOV, illustrated in Fig.~\ref{fig:psf}.
The optical prescription derived from the ray trace is listed in Table~\ref{table:prescription} and
Figure~\ref{fig:schematic}. """)
result.append(figures.psf.figure())
result.append(figures.spot_size.figure())
result.append(figures.focus_curve.figure())
result.append(r"""
The ray trace model was also used to quantify how mirror and positional tolerances affect the
instrument's spatial resolution.
Each element of the model was individually perturbed, then a compensation applied to adjust the image on the \CCD.
The compensation optimized grating tip/tilt angle and \CCD\ focus position, so that the image was re-centered and RMS
spot size minimized at the positions in Fig.~\ref{F-spot} \roy{minimized at the vertices of the field stop and the central field angle}.
We then computed the maximum change in RMS spot size over all spot positions between the optimized and perturbed models.
The computed positional tolerances for each element in the \ESIS\ optical system are listed in Table~\ref{table:errorBudget}.
The imaging error budget is displayed in Table~\ref{table:errorBudget}.
For the primary mirror and grating surface figure contributions, we choose the \MTF\ figures of merit from the surface
roughness specifications described earlier.
To quantify the remaining entries, we assume that each term can be represented by a gaussian function of width
$\sigma^2$ that ``blurs'' the final image.
The value of $\sigma$ then corresponds to the maximum change in RMS spot size for each term as it is perturbed in the
tolerance analysis described above.
The value of the \MTF\ in the right-most column of Table~\ref{table:errorBudget} is computed from
each of the gaussian blur terms at the Nyquist frequency (\SI{0.5}{cycles\per arcsecond}).
From Table~\ref{table:errorBudget}, we estimate the total \MTF\ of \ESIS\ to be $0.109$ at the Nyquist frequency.
Compared to, for example, the Rayleigh criterion of \SI{0.09}{cycles\per arcsecond}~\citep{Rayleigh_1879} we estimate
the resolution of \ESIS\ to be essentially pixel limited.
Since \ESIS\ pixels span \SI{0.76}{\arcsecond} \roy{\plateScaleMean}, the resolution target in Table~\ref{table:scireq} is obtained by this
design.""")
# result.append(pylatex.NoEscape(tables.error_budget.table_old))
result.append(tables.error_budget.table(doc))
return result
def subsection() -> pylatex.Subsection:
result = pylatex.Subsection(pylatex.NoEscape(r'\ESIS\ Features'))
result.escape = False
result.append(figures.layout.figure())
result.append(
r"""
The layout of \ESIS\ (Figure~\ref{fig:layout}) is a modified form of Gregorian telescope.
Incoming light is brought to focus at an octagonal field stop by a parabolic primary mirror.
In the \ESIS\ layout, the secondary mirror of a typical Gregorian telescope is replaced by a segmented, octagonal array
of diffraction gratings.
From the field stop, the gratings re-image to \CCD\ detectors arranged radially around the primary mirror.
The gratings are blazed for first order, so that each \CCD\ is fed by a single corresponding grating, and all the
gratings are identical in design.
The features of this new layout address all of the limitations described in
Section~\ref{subsec:LimitationsoftheMOSESDesign}, and are summarized here.
Replacing the secondary mirror with an array of concave diffraction gratings confers several advantages to \ESIS\
over \MOSES.
First, the concavity of the gratings creates magnification in the \ESIS\ optical system, which results in a shorter axial
length than \MOSES, without sacrificing spatial or spectral resolution.
Second, the magnification and tilt of an individual grating controls the position of the dispersed image with respect to
the optical axis, so that the spectral resolution is not as constrained by the payload dimensions.
Third, the radial symmetry of the design places the cameras closer together, resulting in a more compact instrument.
Furthermore, by arranging the detectors around the optical axis, more dispersed grating orders can be populated; up to
eight gratings can be arrayed around the \ESIS\ primary mirror (up to six with the current optical table).
This contrasts the three image orders available in the planar symmetry of \MOSES.
Taken together, these three design features make \ESIS\ more compact than \MOSES\ \sout{(\S\,\ref{subsec:LimitationsoftheMOSESDesign}
item~\ref{item-length})} \roy{(Limitation~\ref{item-length})}, improve spectral resolution \sout{(\S\,
\ref{subsec:LimitationsoftheMOSESDesign} item~\ref{item-disp_con})} \roy{(Limitation~\ref{item-disp_con})} and allow
the collection of more projections to better constrain the interpretation of the data \sout{(\S\,
\ref{subsec:LimitationsoftheMOSESDesign} item~\ref{item-orders})} \roy{(Limitation~\ref{item-orders})}.
The \ESIS\ gratings are arranged in a segmented array, clocked in \SI{45}{\degree} increments, so that there are
\numChannelsWords\ distinct dispersion planes.
This will greatly aid in reconstructing spectral line profiles since the dispersion space of \ESIS\ occupies a
3D volume rather than a 2D plane as with \MOSES. For \ESIS, there will always be a dispersion plane within
\SI{22.5}{\degree} of the normal to any loop-like feature in the solar atmosphere.
As discussed in Section~\ref{subsec:LimitationsoftheMOSESDesign}, a nearly perpendicular dispersion plane
allows a filamentary structure to serve like a spectrographic slit, resulting in a clear presentation of the
spectrum.
This feature addresses \sout{\S\,\ref{subsec:LimitationsoftheMOSESDesign} item~\ref{item-dispersion}} \roy{
Limitation~\ref{item-dispersion}}.
Rather than forming images at three spectral orders from a single grating, each \ESIS\ imaging channel has a
dedicated grating.
Aberrations are controlled by optimizing the grating design to form images in first order,
over a narrow range of ray deviation angles.
This design controls aberration well enough to allow pixel-limited imaging, avoiding the \PSF\ mismatch problems
inherent to the \MOSES\ design (\S\,\ref{subsec:LimitationsoftheMOSESDesign} item \ref{item-PSF}).
In its flight configuration with gratings optimized around a \OVwavelength\ wavelength, the instrument cannot be aligned and
focused in visible light like \MOSES.
Visible gratings and an special alignment transfer procedure(\S\,\ref{subsec:AlignmentandFocus}) must be used for the
alignment and focus of \ESIS.
The \ESIS\ design also includes an octagonal field stop placed at prime focus.
This confers two advantages.
First, the field stop fully defines the instrument \FOV, so that \ESIS\ is not susceptible to the spectral confusion
observed in \MOSES\ data (\S\,\ref{subsec:LimitationsoftheMOSESDesign} limitation~\ref{item-FOV}).
Second, each spectral image observed by \ESIS\ will be bordered by the outline of the field stop
(\eg\,\S\,\ref{subsec:Optics}).
This aids the inversion process since outside of this sharp edge the intensity is zero for any look angle through an
\ESIS\ data cube.
Additionally, the symmetry of the field stop gives multiple checkpoints where the edge inversion is duplicated in the
dispersed images produced by adjacent orders.
The size and octagonal shape of the field stop are defined by the requirement that all \CCDs\ must see the entire \FOV\
from edge to edge, while leaving a small margin for alignment.
Lastly, in contrast to \MOSES, \ESIS\ employs frame transfer \CCDs\ to make optimum use of our five minutes of observing
time.
The \ESIS\ design is shutterless, so that each detector is always integrating.
The result is a \SI{100}{\percent} duty cycle.
The lack of downtime for readout also allows \ESIS\ to operate at a fixed, rapid cadence of $\sim$\SI{3}{\second}.
Longer integration times can be achieved for faint features by exposure stacking
(\S\,\ref{subsec:LimitationsoftheMOSESDesign} item~\ref{item-CAD}).
In summary, the \ESIS\ concept addresses all the limitations of the \MOSES\ design enumerated in
\S\,\ref{subsec:LimitationsoftheMOSESDesign}.
The volume of the \ESIS\ optical layout is smaller than \MOSES\ by almost a factor of two, yet with a smaller \PSF,
improved spectral resolution, and faster exposure cadence.
\ESIS\ offers several features to improve the recovery of spectral information, including more channels, crossed
dispersion planes, and a field stop."""
)
return result
def subsection() -> pylatex.Subsection:
result = pylatex.Subsection('Energy Transfer')
result.escape = False
result.append(r"""
Tracking the mass and energy flow through the solar atmosphere is a long-standing goal in solar physics.
Bulk mass flow is evidenced by Doppler shifts or skewness in spectral lines.
However, the observed non-thermal broadening of \TR\ spectral lines may result from a variety of physical processes,
including \MHD\ waves~\citep{DePontieu15, DePontieu07}, high-speed evaporative up-flows (\eg\,nanoflares,
\citet{Patsourakos06}), turbulence, and other sources (\eg\,\citet{Mariska1992}).
This is a broad topic which \ESIS\ can address in many ways.
Here we will focus on a single application;
\ESIS\ will search for sources of Alfv\'en waves in the solar atmosphere by observing line broadening as the
spectroscopic signature of these waves.
Alfv\'en waves in coronal holes are observed to carry an energy flux of
\SI{7e5}{erg\per\centi\square\meter\per\second}, enough to energize the fast solar wind \citep{Hahn2012,Hahn2013}.
The source and frequency spectrum of these waves is unknown.
Here, we hypothesize that \MHD\ waves are similarly ubiquitous in quiet Sun and active regions, and play an important
role in the energization of the quiescent corona.
%label to track table 1 references
\phantomsection
\label{t1_1}
The magnitude of non-thermal broadening of optically thin spectral lines is a direct measure of the wave
amplitude~\citep{Banerjee09,Hahn2012,Hahn2013}.
We may estimate a lower limit on the non-thermal velocity to be observed as follows.
We assume that the magnetic field is constant for small changes in scale height in the \TR\ and that line of sight
effects are negligible for observations sufficiently far from disk center.
Since the solar wind is not accelerated to an appreciable fraction of the Alfv\'en wave velocity at altitudes below
$R \leq 1.15R_\odot$~\citep{Cranmer05}, the wave amplitude, $v_{nt}$, depends only weakly on electron density, $n_e$, so
that $v_{nt} \propto n_e^{-1/4}$~\citep{Hahn2013,Moran01}.
Assuming pressure balance between the low corona and transition zone, we may infer non-thermal velocities in the \TR\ by
scaling according to the temperature drop, $v_{nt} \propto T^{1/4}$.
The measured non-thermal velocity of \SI{24}{\kilo\meter\per\second} for Si\,\textsc{viii}~\citep{Doyle98}
(\SI{0.8}{\mega\kelvin}~\citep{Moran03}) near the limb should, neglecting damping, correspond to velocities of at least
\SI{21}{\kilo\meter\per\second} in mid \TR\ Ne\,\textsc{vii}, and \SI{18}{\kilo\meter\per\second} in the lower
O\,\textsc{v} (\SI{0.25}{\mega\kelvin}) line.
The above non-thermal velocities are arrived at assuming both O\,\textsc{v} and Ne\,\textsc{vii} are formed near their
ionization equilibrium temperatures.
For O\,\textsc{v}, the thermal width is $\sim$\SI{11}{\kilo\meter\per\s} at \SI{0.25}{\mega\kelvin} which means the
total linewidth is primarily due to the non-thermal component.
More recently, ~\citet{Srivastava17} observed torsional Alfv\'en waves with amplitude
$\sim$\SI{20}{\kilo\meter\per\second} and period $\sim$\SI{30}{\second} in the chromosphere.
Modeling shows that these torsional waves can transfer a significant amount of energy to the corona~\citep{Kudoh99}.
The torsional motion will be observed as Doppler shifts when viewed from the side.
The oscillation period is long enough to be well resolved but short enough to see $\sim$\SI{10}{} cycles in a single
rocket flight.
An \ESIS-like instrument is therefore well suited to observations of torsional Alfv\'en wave propagation over multiple
heights in the \TR.
By mapping Doppler velocities over a wide field of view in the \TR, \ESIS\ can address questions about both the origin
of waves and whether they are able to propagate upward into the corona.
Independent of the two propagation modes discussed above, there is a range of possible sources for Alfv\'en
(and other \MHD) waves in the solar atmosphere.
Three potential scenarios are: \begin{inparaenum}[(1)] \item Waves originate in the chromosphere or below and propagate
through the \TR\ at a spatially uniform intensity; \label{wave-1}
\item Intense sources are localized in the \TR, but fill only a fraction of the surface\label{wave-2}; and \item Weak
sources are localized in the \TR, but cover the surface densely enough to appear like the first case\label{wave-3}.
\end{inparaenum}
The resulting non-thermal widths for localized sources will be significantly higher than the
$\sim$\SI{20}{\kilo\meter\per\second} mean derived above.
The concentration of non-thermal energy observed by \ESIS\ will serve as an indicator of source density.
Comparison of Doppler maps captured at different temperatures by \ESIS\ will indicate whether a uniform source density originates in the
chromosphere or below (scenario~\ref{wave-1}) or is associated with spatially distributed \TR\ phenomena
(scenario~\ref{wave-3}) such as explosive events, or macrospicules.
Comparison with a wider selection of ground and space based imagery will allow us to determine whether intense,
localized sources (scenario~\ref{wave-2}) are associated with converging or emerging magnetic bipoles, type \textsc{ii}
spicules, spicule bushes, or other sources beneath the \TR.
For these comparisons, we need only to localize, rather than resolve, wave sources.
A spatial resolution of $\sim$\SI{2}{\mega\meter} will be sufficient to localize sources associated with magnetic flux
tubes that are rooted in photospheric inter-granular network lanes (\eg\,\citet{Berger95ApJ})."""
)
return result
def generate_roga(seq_lsts_dict, genus, lab, source, work_dir, amendment_flag,
amended_id):
"""
Generates PDF
:param seq_lsts_dict: Dict of SeqIDs;LSTSIDs
:param genus: Expected Genus for samples (Salmonella, Listeria, Escherichia, or Vibrio)
:param lab: ID for lab report is being generated for
:param source: string input for source that strains were derived from, i.e. 'ground beef'
:param work_dir: bio_request directory
:param amendment_flag: determined if the report is an amendment type or not (True/False)
:param amended_id: ID of the original report that the new report is amending
"""
# RETRIEVE DATAFRAMES FOR EACH SEQID
seq_list = list(seq_lsts_dict.keys())
metadata_reports = extract_report_data.get_combined_metadata(seq_list)
gdcs_reports = extract_report_data.get_gdcs(seq_list)
gdcs_dict = extract_report_data.generate_gdcs_dict(gdcs_reports)
# Create our idiot proofing list. There are a bunch of things that can go wrong that should make us not send
# out reports. As we go through data retrieval/report generation, add things that are wrong to the list, and users
# will get a message saying what's wrong, no report will be generated unless user adds the FORCE flag.
idiot_proofing_list = list()
# DATE SETUP
date = datetime.today().strftime('%Y-%m-%d')
year = datetime.today().strftime('%Y')
# Follow our fiscal year - anything before April is actually previous year.
if datetime.now().month < 4:
year = int(year) - 1
# PAGE SETUP
geometry_options = {
"tmargin": "2cm",
"lmargin": "1cm",
"rmargin": "1cm",
"headsep": "1cm"
}
doc = pl.Document(page_numbers=False, geometry_options=geometry_options)
header = produce_header_footer()
doc.preamble.append(header)
doc.change_document_style("header")
# DATABASE HANDLING
report_id = update_db(date=date,
year=year,
genus=genus,
lab=lab,
source=source,
amendment_flag=amendment_flag,
amended_id=amended_id)
# MARKER VARIABLES SETUP
all_uida = False
all_vt = False
all_mono = False
all_enterica = False
all_vibrio = False
some_vt = False
vt_sample_list = []
# SECOND VALIDATION SCREEN
if genus == 'Escherichia':
validated_ecoli_dict = extract_report_data.validate_ecoli(
seq_list, metadata_reports)
vt_list = []
uida_list = []
hlya_list = []
for key, value in validated_ecoli_dict.items():
ecoli_uida_present = validated_ecoli_dict[key][0]
ecoli_vt_present = validated_ecoli_dict[key][1]
ecoli_hlya_present = validated_ecoli_dict[key][2]
hlya_list.append(ecoli_hlya_present)
uida_list.append(ecoli_uida_present)
vt_list.append(ecoli_vt_present)
# For the AMR table so only vt+ samples are shown
if ecoli_vt_present is True:
vt_sample_list.append(key)
if not ecoli_uida_present:
print(
'WARNING: uidA not present for {}. Cannot confirm E. coli.'
.format(key))
idiot_proofing_list.append(
'uidA not present in {}. Cannot confirm E. coli'.format(
key))
if not ecoli_vt_present:
print('WARNING: vt probe sequences not detected for {}. '
'Cannot confirm strain is verotoxigenic.'.format(key))
idiot_proofing_list.append(
'VTX not present in {}. Cannot confirm strain is verotoxigenic'
.format(key))
if False not in uida_list:
all_uida = True
if False not in vt_list:
all_vt = True
if True in vt_list:
some_vt = True
elif genus == 'Listeria':
validated_listeria_dict = extract_report_data.validate_listeria(
seq_list, metadata_reports)
mono_list = []
for key, value in validated_listeria_dict.items():
mono_list.append(value)
if value is False:
idiot_proofing_list.append(
'Could not confirm {} as L. monocytogenes'.format(key))
if False not in mono_list:
all_mono = True
elif genus == 'Salmonella':
validated_salmonella_dict = extract_report_data.validate_salmonella(
seq_list, metadata_reports)
enterica_list = []
for key, value in validated_salmonella_dict.items():
enterica_list.append(value)
if value is False:
idiot_proofing_list.append(
'Could not confirm {} as S. enterica'.format(key))
if False not in enterica_list:
all_enterica = True
elif genus == 'Vibrio':
validated_vibrio_dict = extract_report_data.validate_vibrio(
seq_list, metadata_reports)
vibrio_list = list()
for key, value in validated_vibrio_dict.items():
vibrio_list.append(value)
if value is False:
idiot_proofing_list.append(
'Could not confirm {} as Vibrio'.format(key))
if False not in vibrio_list:
all_vibrio = True
# MAIN DOCUMENT BODY
with doc.create(
pl.Section('Report of Genomic Analysis: ' + genus,
numbering=False)):
# REPORT ID AND AMENDMENT CHECKING
if amendment_flag:
doc.append(bold('Report ID: '))
doc.append(report_id)
doc.append(italic(' (This report is an amended version of '))
doc.append(amended_id)
doc.append(italic(')'))
doc.append('\n')
doc.append(
pl.Command('TextField',
options=[
"name=rdimsnumberbox", "multiline=false",
pl.NoEscape("bordercolor=0 0 0"),
pl.NoEscape("width=1.1in"), "height=0.2in"
],
arguments=bold('RDIMS ID: ')))
doc.append(bold('\nReporting laboratory: '))
doc.append(lab)
doc.append('\n\n')
# LAB SUMMARY
with doc.create(pl.Tabular('lcr', booktabs=True)) as table:
table.add_row(bold('Laboratory'), bold('Address'),
bold('Tel #'))
table.add_row(lab, lab_info[lab][0], lab_info[lab][1])
# AMENDMENT FIELD
with doc.create(
pl.Subsubsection('Reason for amendment:',
numbering=False)):
with doc.create(Form()):
doc.append(pl.Command('noindent'))
doc.append(
pl.Command('TextField',
options=[
"name=amendmentbox", "multiline=true",
pl.NoEscape("bordercolor=0 0 0"),
pl.NoEscape("width=7in"),
"height=0.43in"
],
arguments=''))
else:
doc.append(bold('Report ID: '))
doc.append(report_id)
doc.append('\n')
doc.append(
pl.Command('TextField',
options=[
"name=rdimsnumberbox", "multiline=false",
pl.NoEscape("bordercolor=0 0 0"),
pl.NoEscape("width=1.1in"), "height=0.2in"
],
arguments=bold('RDIMS ID: ')))
doc.append(bold('\nReporting laboratory: '))
doc.append(lab)
doc.append('\n\n')
# LAB SUMMARY
with doc.create(pl.Tabular('lcr', booktabs=True)) as table:
table.add_row(bold('Laboratory'), bold('Address'),
bold('Tel #'))
table.add_row(lab, lab_info[lab][0], lab_info[lab][1])
# TEXT SUMMARY
with doc.create(
pl.Subsection('Identification Summary',
numbering=False)) as summary:
summary.append('Whole-genome sequencing analysis was conducted on '
'{} '.format(len(metadata_reports)))
summary.append(italic('{} '.format(genus)))
if len(metadata_reports) == 1:
summary.append('strain isolated from "{}". '.format(
source.lower()))
else:
summary.append('strains isolated from "{}". '.format(
source.lower()))
if genus == 'Escherichia':
if all_uida:
summary.append('The following strains are confirmed as ')
summary.append(italic('Escherichia coli '))
summary.append(
'based on 16S sequence and the presence of marker gene '
)
summary.append(italic('uidA. '))
elif not all_uida:
summary.append(
'Some of the following strains could not be confirmed to be '
)
summary.append(italic('Escherichia coli '))
summary.append('as the ')
summary.append(italic('uidA '))
summary.append('marker gene was not detected. ')
if all_vt:
summary.append(
'All strain(s) are confirmed to be VTEC based on detection of probe sequences '
'indicating the presence of verotoxin genes.')
elif genus == 'Listeria':
if all_mono:
summary.append(
'The following strains are confirmed to be ')
summary.append(italic('Listeria monocytogenes '))
summary.append('based on GeneSeekr analysis: ')
else:
summary.append(
'Some of the following strains could not be confirmed to be '
)
summary.append(italic('Listeria monocytogenes.'))
elif genus == 'Salmonella':
if all_enterica:
summary.append(
'The following strains are confirmed to be ')
summary.append(italic('Salmonella enterica '))
summary.append('based on GeneSeekr analysis: ')
else:
summary.append(
'Some of the following strains could not be confirmed to be '
)
summary.append(italic('Salmonella enterica.'))
elif genus == 'Vibrio':
if all_vibrio:
summary.append(
'The following strains are confirmed to be ')
summary.append(italic('Vibrio parahaemolyticus '))
summary.append('based on GeneSeekr analysis: ')
else:
summary.append(
'Some of the following strains could not be confirmed to be '
)
summary.append(italic('Vibrio parahaemolyticus.'))
# VIBRIO TABLE
if genus == 'Vibrio':
genesippr_table_columns = (
bold('ID'),
bold(pl.NoEscape(r'R72H{\footnotesize \textsuperscript {a}}')),
bold(
pl.NoEscape(r'groEL{\footnotesize \textsuperscript {a}}')),
bold(pl.NoEscape(r'Virulence Profile')),
bold(pl.NoEscape(r'MLST')),
bold(pl.NoEscape(r'rMLST')),
)
with doc.create(
pl.Subsection('GeneSeekr Analysis',
numbering=False)) as genesippr_section:
with doc.create(pl.Tabular('|c|c|c|c|c|c|')) as table:
# Header
table.add_hline()
table.add_row(genesippr_table_columns)
# Rows
for sample_id, df in metadata_reports.items():
table.add_hline()
# ID
# lsts_id = df.loc[df['SeqID'] == sample_id]['SampleName'].values[0]
lsts_id = seq_lsts_dict[sample_id]
# Genus
genus = df.loc[df['SeqID'] ==
sample_id]['Genus'].values[0]
# MLST/rMLST
mlst = str(df.loc[df['SeqID'] == sample_id]
['MLST_Result'].values[0]).replace(
'-', 'New')
rmlst = str(df.loc[df['SeqID'] == sample_id]
['rMLST_Result'].values[0]).replace(
'-', 'New')
# Markers
marker_list = df.loc[df['SeqID'] == sample_id][
'GeneSeekr_Profile'].values[0]
(r72h, groel) = '-', '-'
if 'r72h' in marker_list:
r72h = '+'
if 'groEL' in marker_list:
groel = '+'
# Virulence
virulence = ''
if 'tdh' in marker_list:
virulence += 'tdh;'
if 'trh' in marker_list:
virulence += 'trh;'
if ';' in virulence:
virulence = virulence[:-1]
if virulence == '':
virulence = '-'
table.add_row(
(lsts_id, r72h, groel, virulence, mlst, rmlst))
table.add_hline()
create_caption(
genesippr_section, 'a', "+ indicates marker presence : "
"- indicates marker was not detected")
# ESCHERICHIA TABLE
if genus == 'Escherichia':
genesippr_table_columns = (
bold('ID'),
bold(pl.NoEscape(r'uidA{\footnotesize \textsuperscript {a}}')),
bold(pl.NoEscape(r'Serotype')),
bold(pl.NoEscape(r'Verotoxin(s)')),
bold(pl.NoEscape(r'hlyA{\footnotesize \textsuperscript {a}}')),
bold(pl.NoEscape(r'eae{\footnotesize \textsuperscript {a}}')),
bold(pl.NoEscape(r'aggR{\footnotesize \textsuperscript {a}}')),
bold(pl.NoEscape(r'MLST')),
bold(pl.NoEscape(r'rMLST')),
)
with doc.create(
pl.Subsection('GeneSeekr Analysis',
numbering=False)) as genesippr_section:
with doc.create(pl.Tabular('|c|c|c|c|c|c|c|c|c|')) as table:
# Header
table.add_hline()
table.add_row(genesippr_table_columns)
# Rows
for sample_id, df in metadata_reports.items():
table.add_hline()
# ID
# lsts_id = df.loc[df['SeqID'] == sample_id]['SampleName'].values[0]
lsts_id = seq_lsts_dict[sample_id]
# Genus (pulled from 16S)
genus = df.loc[df['SeqID'] ==
sample_id]['Genus'].values[0]
# Serotype
serotype = df.loc[df['SeqID'] == sample_id][
'E_coli_Serotype'].values[0]
# Remove % identity
fixed_serotype = remove_bracketed_values(serotype)
# Verotoxin
verotoxin = df.loc[df['SeqID'] == sample_id][
'Vtyper_Profile'].values[0]
# MLST/rMLST
mlst = str(df.loc[df['SeqID'] == sample_id]
['MLST_Result'].values[0]).replace(
'-', 'New')
rmlst = str(df.loc[df['SeqID'] == sample_id]
['rMLST_Result'].values[0]).replace(
'-', 'New')
marker_list = df.loc[df['SeqID'] == sample_id][
'GeneSeekr_Profile'].values[0]
(uida, eae, hlya, aggr) = '-', '-', '-', '-'
if 'uidA' in marker_list:
uida = '+'
if 'eae' in marker_list:
eae = '+'
if 'hlyA' in marker_list:
hlya = '+'
if 'aggR' in marker_list:
aggr = '+'
table.add_row(
(lsts_id, uida, fixed_serotype, verotoxin, hlya,
eae, aggr, mlst, rmlst))
table.add_hline()
create_caption(
genesippr_section, 'a', "+ indicates marker presence : "
"- indicates marker was not detected")
# LISTERIA TABLE
if genus == 'Listeria':
genesippr_table_columns = (
bold('ID'),
bold(pl.NoEscape(r'IGS{\footnotesize \textsuperscript {a}}')),
bold(pl.NoEscape(r'hlyA{\footnotesize \textsuperscript {a}}')),
bold(pl.NoEscape(r'inlJ{\footnotesize \textsuperscript {a}}')),
bold(pl.NoEscape(r'MLST')),
bold(pl.NoEscape(r'rMLST')),
)
with doc.create(
pl.Subsection('GeneSeekr Analysis',
numbering=False)) as genesippr_section:
with doc.create(pl.Tabular('|c|c|c|c|c|c|')) as table:
# Header
table.add_hline()
table.add_row(genesippr_table_columns)
# Rows
for sample_id, df in metadata_reports.items():
table.add_hline()
# ID
# lsts_id = df.loc[df['SeqID'] == sample_id]['SampleName'].values[0]
lsts_id = seq_lsts_dict[sample_id]
# Genus
genus = df.loc[df['SeqID'] ==
sample_id]['Genus'].values[0]
# MLST/rMLST
mlst = str(df.loc[df['SeqID'] == sample_id]
['MLST_Result'].values[0]).replace(
'-', 'New')
rmlst = str(df.loc[df['SeqID'] == sample_id]
['rMLST_Result'].values[0]).replace(
'-', 'New')
# Markers
marker_list = df.loc[df['SeqID'] == sample_id][
'GeneSeekr_Profile'].values[0]
(igs, hlya, inlj) = '-', '-', '-'
if 'IGS' in marker_list:
igs = '+'
if 'hlyA' in marker_list:
hlya = '+'
if 'inlJ' in marker_list:
inlj = '+'
table.add_row((lsts_id, igs, hlya, inlj, mlst, rmlst))
table.add_hline()
create_caption(
genesippr_section, 'a', "+ indicates marker presence : "
"- indicates marker was not detected")
# SALMONELLA TABLE
if genus == 'Salmonella':
genesippr_table_columns = (
bold('ID'),
bold(
pl.NoEscape(
r'Serovar{\footnotesize \textsuperscript {a}}')),
bold(
pl.NoEscape(
r'Serogroup{\footnotesize \textsuperscript {a,b}}')),
bold(pl.NoEscape(r'H1{\footnotesize \textsuperscript {a}}')),
bold(pl.NoEscape(r'H2{\footnotesize \textsuperscript {a}}')),
bold(pl.NoEscape(r'invA{\footnotesize \textsuperscript {b}}')),
bold(pl.NoEscape(r'stn{\footnotesize \textsuperscript {b}}')),
bold(pl.NoEscape(r'MLST')),
bold(pl.NoEscape(r'rMLST')),
)
with doc.create(
pl.Subsection('GeneSeekr Analysis',
numbering=False)) as genesippr_section:
with doc.create(
pl.Tabular('|c|p{2cm}|c|c|c|c|c|c|c|')) as table:
# Header
table.add_hline()
table.add_row(genesippr_table_columns)
# Rows
for sample_id, df in metadata_reports.items():
table.add_hline()
# ID
# lsts_id = df.loc[df['SeqID'] == sample_id]['SampleName'].values[0]
lsts_id = seq_lsts_dict[sample_id]
# MLST/rMLST
mlst = str(df.loc[df['SeqID'] == sample_id]
['MLST_Result'].values[0]).replace(
'-', 'New')
rmlst = str(df.loc[df['SeqID'] == sample_id]
['rMLST_Result'].values[0]).replace(
'-', 'New')
# Serovar
serovar = df.loc[df['SeqID'] ==
sample_id]['SISTR_serovar'].values[0]
# If the serovar is particularly long, tables end up being longer than the page.
# To fix, try to find a space somewhere near the middle of the serovar string and insert a
# newline there.
if len(serovar) > 12:
# First, find what index a space is that we can change.
starting_index = int(len(serovar) / 2)
index_to_change = 999
for i in range(starting_index, len(serovar)):
if serovar[i] == ' ':
index_to_change = i
break
if index_to_change != 999:
serovar_with_newline = ''
for i in range(len(serovar)):
if i == index_to_change:
serovar_with_newline += '\\newline '
else:
serovar_with_newline += serovar[i]
serovar = pl.NoEscape(r'' +
serovar_with_newline)
# SISTR Serogroup, H1, H2
sistr_serogroup = df.loc[df['SeqID'] == sample_id][
'SISTR_serogroup'].values[0]
sistr_h1 = df.loc[df['SeqID'] == sample_id][
'SISTR_h1'].values[0].strip(';')
sistr_h2 = df.loc[df['SeqID'] == sample_id][
'SISTR_h2'].values[0].strip(';')
# Markers
marker_list = df.loc[df['SeqID'] == sample_id][
'GeneSeekr_Profile'].values[0]
(inva, stn) = '-', '-'
if 'invA' in marker_list:
inva = '+'
if 'stn' in marker_list:
stn = '+'
table.add_row(
(lsts_id, serovar, sistr_serogroup, sistr_h1,
sistr_h2, inva, stn, mlst, rmlst))
table.add_hline()
create_caption(
genesippr_section, 'a',
"Predictions conducted using SISTR "
"(Salmonella In Silico Typing Resource)")
create_caption(
genesippr_section, 'b', "+ indicates marker presence : "
"- indicates marker was not detected")
# AMR TABLE (VTEC and Salmonella only)
create_amr_profile = False # only create if an AMR profile exists for one of the provided samples
amr_samples = [] # keep track of which samples to create rows for
# Grab AMR profile as a pre-check to see if we should even create the AMR Profile table
for sample_id, df in metadata_reports.items():
profile = df.loc[df['SeqID'] == sample_id]['AMR_Profile'].values[0]
parsed_profile = extract_report_data.parse_amr_profile(profile)
if parsed_profile is not None:
if genus == 'Salmonella':
amr_samples.append(sample_id)
create_amr_profile = True
elif genus == 'Escherichia':
if sample_id in vt_sample_list: # vt_sample_list contains all vt+ sample IDs
amr_samples.append(sample_id)
create_amr_profile = True
elif genus == 'Vibrio':
amr_samples.append(sample_id)
create_amr_profile = True
# Create table
if (genus == 'Salmonella' or some_vt is True
or genus == 'Vibrio') and create_amr_profile is True:
with doc.create(
pl.Subsection('Antimicrobial Resistance Profiling',
numbering=False)):
with doc.create(pl.Tabular('|c|c|c|c|')) as table:
amr_columns = (bold('ID'),
bold(pl.NoEscape(r'Resistance')),
bold(pl.NoEscape(r'Gene')),
bold(pl.NoEscape(r'Percent Identity')))
# Header
table.add_hline()
table.add_row(amr_columns)
# Keep track of what previous id and resistance were so we know how far to draw lines across
# table. Initialize to some gibberish.
previous_id = 'asdasdfasdfs'
previous_resistance = 'akjsdhfasdf'
# For the AMR table, don't re-write sample id if same sample has multiple resistances
# Also, don't re-write resistances if same resistance has multiple genes.
for sample_id, df in metadata_reports.items():
if sample_id in amr_samples:
# Grab AMR profile
profile = df.loc[df['SeqID'] == sample_id][
'AMR_Profile'].values[0]
# Parse and iterate through profile to generate rows
parsed_profile = extract_report_data.parse_amr_profile(
profile)
if parsed_profile is not None:
# Rows
for value in parsed_profile:
# ID
resistance = value.resistance
res_to_write = resistance
lsts_id = seq_lsts_dict[sample_id]
# If sample we're on is different from previous sample, line goes all the
# way across the table.
if lsts_id != previous_id:
table.add_hline()
id_to_write = lsts_id
# If sample is same and resistance is same, only want lines for gene and percent
# identity columns. Don't write out id or resistance again.
elif resistance == previous_resistance:
table.add_hline(start=3, end=4)
id_to_write = ''
res_to_write = ''
# Finally, if resistance is different, but id is same, need line across for
# resistance, gene, and percent id. Write out everything but id
else:
table.add_hline(start=2, end=4)
id_to_write = ''
previous_id = lsts_id
previous_resistance = resistance
# Gene
gene = value.gene
# Identity
identity = value.percent_id
# Add row
table.add_row((id_to_write, res_to_write,
gene, identity))
# Close off table
table.add_hline()
# SEQUENCE TABLE
with doc.create(
pl.Subsection('Sequence Quality Metrics', numbering=False)):
with doc.create(pl.Tabular('|c|c|c|c|c|')) as table:
# Columns
sequence_quality_columns = (
bold('ID'),
bold(pl.NoEscape(r'Total Length')),
bold(pl.NoEscape(r'Coverage')),
bold(pl.NoEscape(r'GDCS')),
bold(pl.NoEscape(r'Pass/Fail')),
)
# Header
table.add_hline()
table.add_row(sequence_quality_columns)
# Rows
for sample_id, df in metadata_reports.items():
table.add_hline()
# Grab values
# lsts_id = df.loc[df['SeqID'] == sample_id]['SampleName'].values[0]
lsts_id = seq_lsts_dict[sample_id]
total_length = df.loc[df['SeqID'] ==
sample_id]['TotalLength'].values[0]
average_coverage_depth = df.loc[df['SeqID'] == sample_id][
'AverageCoverageDepth'].values[0]
# Fix coverage
average_coverage_depth = format(
float(str(average_coverage_depth).replace('X', '')),
'.0f')
average_coverage_depth = str(average_coverage_depth) + 'X'
# Matches
matches = gdcs_dict[sample_id][0]
passfail = gdcs_dict[sample_id][1]
if passfail == '+':
passfail = 'Pass'
elif passfail == '-':
passfail = 'Fail'
idiot_proofing_list.append(
'{} failed GDCS validation'.format(sample_id))
# Add row
table.add_row((lsts_id, total_length,
average_coverage_depth, matches, passfail))
table.add_hline()
# PIPELINE METADATA TABLE
pipeline_metadata_columns = (bold('ID'), bold('Seq ID'),
bold('Pipeline Version'),
bold('Database Version'))
with doc.create(pl.Subsection('Pipeline Metadata', numbering=False)):
with doc.create(pl.Tabular('|c|c|c|c|')) as table:
# Header
table.add_hline()
table.add_row(pipeline_metadata_columns)
# Rows
for sample_id, df in metadata_reports.items():
table.add_hline()
# ID
# lsts_id = df.loc[df['SeqID'] == sample_id]['SampleName'].values[0]
lsts_id = seq_lsts_dict[sample_id]
# Pipeline version
pipeline_version = df.loc[
df['SeqID'] == sample_id]['PipelineVersion'].values[0]
database_version = pipeline_version
# Add row
table.add_row((lsts_id, sample_id, pipeline_version,
database_version))
table.add_hline()
# 'VERIFIED BY' FIELD
with doc.create(pl.Subsubsection('Verified by:', numbering=False)):
with doc.create(Form()):
doc.append(pl.Command('noindent'))
doc.append(
pl.Command('TextField',
options=[
"name=verifiedbybox", "multiline=false",
pl.NoEscape("bordercolor=0 0 0"),
pl.NoEscape("width=2.5in"), "height=0.3in"
],
arguments=''))
# OUTPUT PDF FILE
pdf_file = os.path.join(work_dir,
'{}_{}_{}'.format(report_id, genus, date))
try:
doc.generate_pdf(pdf_file, clean_tex=False)
except:
pass
pdf_file += '.pdf'
return pdf_file, idiot_proofing_list
