def _save_to_fs(self, full_key_path: str, plot: plt.figure):
bytes_buffer = io.BytesIO()
plot.savefig(bytes_buffer, **self._save_args)
with self._fs.open(full_key_path,
**self._fs_open_args_save) as fs_file:
fs_file.write(bytes_buffer.getvalue())
def writeplots(fg: figure, plotprefix: str, params: dict):
""" Save Matplotlib plots to disk.
inputs:
------
fg: Matplotlib figure handle
odir: directory to write image into e.g. for this particular simulation.
plotprefix: stem of filename
E0: beam energy (eV)
method: format of image
Some observations on image formats:
* TIF was not faster and was 100 times the file size!
* PGF is slow and big file,
* RAW crashes
* JPG no faster than PNG
"""
if 'odir' not in params or not params['odir']:
return
odir = Path(params['odir']).expanduser()
draw(
) # Must have this here or plot doesn't update in animation multiplot mode!
cn = odir / (plotprefix + f'beam{params["E0"]:.0f}.{params["plotformat"]}')
print('write', cn)
fg.savefig(cn, bbox_inches='tight', dpi=dpi) # this is slow and async
def plot_image_grid(images: List[np.array],
titles: List[str] = None,
figure: plt.figure = None,
grayscale: bool = False,
transpose: bool = False) -> plt.figure:
"""
Plot a grid of n x m images
Input
-----
images: List[np.array]
Images in a n x m array
titles: List[str] (opt.)
List of m titles for each image column
figure: plt.figure (opt.)
Pyplot figure (if None, will be created)
grayscale: bool (opt.)
If True return grayscaled images
transpose: bool (opt.)
If True, transpose the grid
Output
------
Pyplot figure filled with the images
"""
num_cols, num_rows = len(images), len(images[0])
img_ratio = images[0][0].shape[1] / images[0][0].shape[0]
if transpose:
vert_grid_shape, hori_grid_shape = (1, num_rows), (num_cols, 1)
figsize = (int(num_rows * 5 * img_ratio), num_cols * 5)
wspace, hspace = 0.2, 0.
else:
vert_grid_shape, hori_grid_shape = (num_rows, 1), (1, num_cols)
figsize = (int(num_cols * 5 * img_ratio), num_rows * 5)
hspace, wspace = 0.2, 0.
if figure is None:
figure = plt.figure(figsize=figsize)
imshow_params = {'cmap': plt.get_cmap('gray')} if grayscale else {}
grid_spec = gridspec.GridSpec(*hori_grid_shape, wspace=0, hspace=0)
for j in range(num_cols):
grid_spec_j = gridspec.GridSpecFromSubplotSpec(
*vert_grid_shape,
subplot_spec=grid_spec[j],
wspace=wspace,
hspace=hspace)
for i in range(num_rows):
ax_img = figure.add_subplot(grid_spec_j[i])
# ax_img.axis('off')
ax_img.set_yticks([])
ax_img.set_xticks([])
if titles is not None:
if transpose:
ax_img.set_ylabel(titles[j], fontsize=25)
else:
ax_img.set_title(titles[j], fontsize=15)
ax_img.imshow(images[j][i], **imshow_params)
figure.tight_layout()
return figure
def plot_contours(roi_array: np.ndarray,
template: np.ndarray,
fig: plt.figure = None,
ax: plt.axis = None):
"""Plot contours of roi.
Args:
roi_array (numpy.ndarray): Array of roi masks.
template (numpy.ndarray): Template image.
fig (matplotlib.pyplot.figure, optional): Defaults to None. Figure to plot to.
ax (matplotlib.pyplot.axis, optional): Defaults to None. Axis to plot to.
"""
if fig is None and ax is None:
fig = plt.figure()
ax = fig.add_subplot(111)
elif ax is None and fig is not None:
ax = fig.add_subplot(111)
z, _, _ = roi_array.shape
np.random.seed(128)
colors = np.random.rand(3, z)
for idx, img_slice in enumerate(roi_array):
clean_border = segmentation.clear_border(img_slice).astype(np.int)
template = segmentation.mark_boundaries(template,
clean_border,
color=colors[:, idx],
mode='thick')
ax.imshow(template)
def savePlot(path: str, filename: str, fig: plt.figure) -> str:
"""Save plot whilst checking for overwriting
Parameters
----------
path : str
The path to save the file
filename : str
Filename to save to
fig : plt.figure
Matplotlig figure to save
Returns
-------
str
The filename written out to
"""
import os
filenameForWriting = filename
fileExists = True
counter = 1
while fileExists:
filepath = os.path.join(path, "{}.png".format(filenameForWriting))
if os.path.exists(filepath):
filenameForWriting = "{}.{}".format(filename, counter)
counter += 1
else:
fileExists = False
break
filepath = os.path.join(path, "{}.png".format(filenameForWriting))
fig.savefig(filepath)
return filenameForWriting
def plot_state_space_projection(values, conditions, figure_size = (5, 5), alpha = 0.5):
"""
Plots things in 2D for a single attentional condition
:param values: [state dimension][time]
:param conditions: conditions points by actual condition? vect of ints, one per 4 conds
:param colormap: conditions to use per actual condition
:param figure_size: matplotlib figure size, useful when embedding into ipython notebook
:param alpha: alpha of the individual dots
:return: reference to figure object
"""
figure = Figure(figsize = figure_size)
axes = figure.add_subplot(111, aspect = 'equal')
values_by_condition = sort_by_condition(values, conditions, 4)
centroids = numpy.zeros([4, values.shape[0]])
for i in range(4):
centroids[i, :] = numpy.mean(values_by_condition[i], 1)
# QR decomposition can flip signs on axes
# check and flip back if needed
x_sign = 1
y_sign = 1
if centroids[3, 0] < centroids[0, 0]:
x_sign = -1
if centroids[3, 1] < centroids[0, 1]:
y_sign = -1
axes.scatter(x_sign * values_by_condition[0][0, :], y_sign * values_by_condition[0][1, :], alpha = alpha,
marker = 'o', c = '#ff5100', edgecolor = 'none', lw = 0)
axes.scatter(x_sign * values_by_condition[1][0, :], y_sign * values_by_condition[1][1, :], alpha = alpha,
marker = 'o', c = '#91d90d', edgecolor = 'none', lw = 0)
axes.scatter(x_sign * values_by_condition[2][0, :], y_sign * values_by_condition[2][1, :], alpha = alpha,
marker = 'o', c = '#0084ff', edgecolor = 'none', lw = 0)
axes.scatter(x_sign * values_by_condition[3][0, :], y_sign * values_by_condition[3][1, :], alpha = alpha,
marker = 'o', c = '#a200ff', edgecolor = 'none', lw = 0)
for i in range(4):
colors = {0: '#ff5100', 1: '#91d90d', 2: '#0084ff', 3: '#a200ff'}
axes.scatter(x_sign * centroids[i, 0], y_sign * centroids[i, 1], alpha = 1,
c = "#000000", marker = '+', s = 700, linewidth = 8)
axes.scatter(x_sign * centroids[i, 0], y_sign * centroids[i, 1], alpha = 1,
c = colors[i], marker = '+', s = 500, linewidth = 4)
axes.set_xlabel('Human presence')
axes.set_ylabel('Vehicle presence')
axes.title.set_fontsize(16)
axes.xaxis.label.set_fontsize(20)
axes.yaxis.label.set_fontsize(20)
axes.set_xlim([-4, 4])
axes.set_ylim([-4, 4])
for item in (axes.get_xticklabels() + axes.get_yticklabels()):
item.set_fontsize(12)
return figure
def save_plot(folder_to_save: str,
fig: plt.figure,
name_prefix: str,
data_path: Optional[str] = None,
extension: consts.EXTENSION = consts.EXTENSION.PNG) -> None:
name = get_result_file_name(name_prefix, data_path, extension)
log.info('Saving ' + name)
# 'tight' is used to alter the size of the bounding box (whitespace) around the output image
fig.savefig(os.path.join(folder_to_save, name), bbox_inches='tight')
def calculate_new_figsize(self,
real_fig: plt.figure) -> typing.List[float]:
""" Calculate figure size to allow for labels, etc
Args:
real_fig (plt.figure): Figure before calculation
Returns:
typing.List[float]: The dimensions [left, bottom, width, height] of the new axes. All quantities are in fractions of figure width and height.
"""
import io
# df_values = self.prepare_data()
fig = plt.Figure(figsize=self.figsize)
# if self.title:
# fig.tight_layout(rect=[0, 0, 1, 0.9]) # To include title
ax = fig.add_subplot()
fake_cols = [chr(i + 70) for i in range(self.df.shape[1])]
max_val = self.df.max().max()
if self.orientation == "h":
ax.barh(fake_cols, [1] * self.df.shape[1])
self.extracted_from_calculate_new_figsize_15(ax, 0, "y", fig, io)
orig_pos = ax.get_position()
ax.set_yticklabels(self.df.columns)
ax.set_xticklabels([max_val] * len(ax.get_xticks()))
else:
ax.bar(fake_cols, [1] * self.df.shape[1])
self.extracted_from_calculate_new_figsize_15(ax, 30, "x", fig, io)
orig_pos = ax.get_position()
ax.set_xticklabels(self.df.columns, ha="right")
ax.set_yticklabels([max_val] * len(ax.get_yticks()))
fig.canvas.print_figure(io.BytesIO(), format="png")
new_pos = ax.get_position()
coordx, prev_coordx = new_pos.x0, orig_pos.x0
coordy, prev_coordy = new_pos.y0, orig_pos.y0
old_w, old_h = self.figsize
# if coordx > prev_coordx or coordy > prev_coordy:
prev_w_inches = prev_coordx * old_w
total_w_inches = coordx * old_w
extra_w_inches = total_w_inches - prev_w_inches
new_w_inches = extra_w_inches + old_w
prev_h_inches = prev_coordy * old_h
total_h_inches = coordy * old_h
extra_h_inches = total_h_inches - prev_h_inches
new_h_inches = extra_h_inches + old_h
real_fig.set_size_inches(new_w_inches, new_h_inches)
left = total_w_inches / new_w_inches
bottom = total_h_inches / new_h_inches
width = orig_pos.x1 - left
height = orig_pos.y1 - bottom
return [left, bottom, width, height]
def save_and_show_if_needed(folder_to_save: str,
to_show: bool,
fig: plt.figure,
data_path: Optional[str] = None,
name_prefix: str = '') -> None:
if folder_to_save:
save_plot(folder_to_save, fig, name_prefix, data_path)
if to_show:
log.info('Showing ' + data_path)
fig.show()
def plot_new_canvas(figure: plt.figure, main: tk.Tk) -> None:
"""
Plot the new figure on the Tk canvas
:param figure: fig object that is the plot figure
:param main: tk.Tk() instance
:return: None
"""
figure.set_size_inches((9, 6))
canvas = FigureCanvasTkAgg(figure, master=main)
canvas.draw()
canvas.get_tk_widget().grid(column=0, row=0, columnspan=2, rowspan=8)
def showPlot3D(self, x, y, z):
fig = Figure(dpi=100, frameon=False)
a = fig.add_subplot(111, projection='3d')
a.plot_surface(x, y, z, rstride=4, cstride=4, linewidth=0, color='b')
canvas = FigureCanvas(fig)
#Makes possible to rotate the plot
a.mouse_init()
self.update_canvas(canvas)
def showPlot3D(self,x,y,z):
fig = Figure(dpi=100, frameon=False)
a = fig.add_subplot(111, projection='3d')
a.plot_surface(x,y,z, rstride=4, cstride=4, linewidth=0, color='b')
canvas = FigureCanvas(fig)
#Makes possible to rotate the plot
a.mouse_init()
self.update_canvas(canvas)
def _save_chart(self, fig: plt.figure) -> BytesIO:
"""Private method to save chart as a bytes stream
Args:
fig (plt.figure): matplotlib figure to save
Returns:
BytesIO: bytes stream containing the chart's data
"""
temp = BytesIO()
fig.savefig(temp, bbox_inches=self.bbox_inches, pad_inches=self.pad_inches, dpi=fig.dpi)
plt.close()
return temp
def showPlot2D(self, x, y, squared):
if squared:
asp = 'equal'
else:
asp = 'auto'
fig = Figure(dpi=100, frameon=False)
a = fig.add_subplot(111, aspect=asp)
a.plot(x, y)
canvas = FigureCanvas(fig)
self.update_canvas(canvas)
def showPlot2D(self, x, y, squared):
if squared:
asp = 'equal'
else:
asp = 'auto'
fig = Figure(dpi=100, frameon=False)
a = fig.add_subplot(111, aspect=asp)
a.plot(x,y)
canvas = FigureCanvas(fig)
self.update_canvas(canvas)
def calculate_new_figsize(self,
real_fig: plt.figure) -> typing.List[float]:
""" Calculate figure size to allow for labels, etc
Args:
real_fig (plt.figure): Figure before calculation
Returns:
typing.List[float]: The dimensions [left, bottom, width, height] of the new axes. All quantities are in fractions of figure width and height.
"""
import io
fig = plt.Figure(figsize=self.figsize)
ax = fig.add_subplot()
max_val = self.df.values.max().max()
ax.tick_params(labelrotation=0,
axis="y",
labelsize=self.tick_label_size)
fig.canvas.print_figure(io.BytesIO())
orig_pos = ax.get_position()
ax.set_yticklabels(self.df.columns)
ax.set_xticklabels([max_val] * len(ax.get_xticks()))
fig.canvas.print_figure(io.BytesIO(), format="png")
new_pos = ax.get_position()
coordx, prev_coordx = new_pos.x0, orig_pos.x0
coordy, prev_coordy = new_pos.y0, orig_pos.y0
old_w, old_h = self.figsize
# if coordx > prev_coordx or coordy > prev_coordy:
prev_w_inches = prev_coordx * old_w
total_w_inches = coordx * old_w
extra_w_inches = total_w_inches - prev_w_inches
new_w_inches = extra_w_inches + old_w
prev_h_inches = prev_coordy * old_h
total_h_inches = coordy * old_h
extra_h_inches = total_h_inches - prev_h_inches
new_h_inches = extra_h_inches + old_h
real_fig.set_size_inches(new_w_inches, new_h_inches)
left = total_w_inches / new_w_inches
bottom = total_h_inches / new_h_inches
width = orig_pos.x1 - left
height = orig_pos.y1 - bottom
return [left, bottom, width, height]
def histplot_2d(
var_x: pd.Series, var_y: pd.Series,
xbins: Union[tuple, list], ybins: Union[tuple, list],
weights: pd.Series,
ax: plt.axes,
fig: plt.figure,
n_threads: int = config.n_threads,
is_z_log: bool = True,
is_square: bool = True,
) -> bh.Histogram:
"""
Plots and prints out 2d histogram. Does not support axis transforms (yet!)
:param var_x: pandas series of var to plot on x-axis
:param var_y: pandas series of var to plot on y-axis
:param xbins: tuple of bins in x (n_bins, start, stop) or list of bin edges
:param ybins: tuple of bins in y (n_bins, start, stop) or list of bin edges
:param weights: series of weights to apply to axes
:param ax: axis to plot on
:param fig: figure to plot on (for colourbar)
:param n_threads: number of threads for filling
:param is_z_log: whether z-axis should be scaled logarithmically
:param is_square: whether to set square aspect ratio
:return: histogram
"""
# setup and fill histogram
hist_2d = bh.Histogram(get_axis(xbins), get_axis(ybins))
hist_2d.fill(var_x, var_y, weight=weights, threads=n_threads)
if is_z_log:
norm = LogNorm()
else:
norm = None
# setup mesh differently depending on bh storage
if hasattr(hist_2d.view(), 'value'):
mesh = ax.pcolormesh(*hist_2d.axes.edges.T, hist_2d.view().value.T, norm=norm)
else:
mesh = ax.pcolormesh(*hist_2d.axes.edges.T, hist_2d.view().T, norm=norm)
fig.colorbar(mesh, ax=ax, fraction=0.046, pad=0.04)
if is_square: # square aspect ratio
ax.set_aspect(1 / ax.get_data_ratio())
return hist_2d
def show_graph(figure: plt.figure, filename: Optional[str]):
"""
Show or save plot
Args:
figure: plt.figure
filename: Файл для записи
Returns:
"""
if filename:
try:
figure.savefig(filename)
except (FileNotFoundError, IsADirectoryError) as err:
print("Невозможно сохранить файл. Проверьте путь. "
"Error '{0}' occured. Arguments {1}.".format(err, err.args))
else:
plt.show()
def add_label(
self,
fig: plt.figure,
):
"""Add a text label to a figure in a funny position.
Parameters
----------
fig : plt.figure
A matplotlib figure
"""
label = "Gamestonk Terminal"
fig.text(
0.69,
0.0420,
label,
fontsize=12,
color="gray",
alpha=0.5,
)
def draw_model_prediction(fig: plt.figure,
ax: plt.axis,
model: nn.Module,
gw: int = 0,
min_point: (float, float) = (0, 0),
max_point: (float, float) = (500, 500),
resolution: float = 1,
apply_ploss: bool = True,
colorbar: bool = True,
detection_flag: bool = False,
detection_threshold: float = -140,
device='cpu',
*args,
**kwargs):
x_mesh, y_mesh = np.mgrid[min_point[0]:max_point[0]:resolution,
min_point[1]:max_point[1]:resolution]
x = x_mesh.ravel()
y = y_mesh.ravel()
data_x = np.stack([x, y], axis=1).astype(np.float32)
if detection_flag is False:
z = model(torch.from_numpy(data_x).to(device),
apply_ploss).detach()[:, gw, 0].cpu().numpy()
else:
#z = model(torch.from_numpy(data_x).to(device), apply_ploss).detach()[:, gw, 0].cpu().numpy()
z = model(torch.from_numpy(data_x).to(device),
apply_ploss).detach()[:, gw, :].cpu().numpy()
sub_z1 = z[:, 1]
sub_z = z[:, 0]
sub_z[sub_z1 < 0.9] = detection_threshold
z = z[:, 0]
z = z.reshape((int(max_point[0] - min_point[0]),
int(max_point[1] - min_point[1]))).transpose()
z[z < detection_threshold] = detection_threshold
im = ax.imshow(z, norm=colors.PowerNorm(gamma=2))
if colorbar:
divider = make_axes_locatable(ax)
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im, cax=cax, orientation='vertical', label='dBm')
return im
def add_cmd_source(
self,
fig: plt.figure,
):
"""Add a text label to a figure in a funny position.
Parameters
----------
fig : plt.figure
A matplotlib figure
"""
from gamestonk_terminal.helper_funcs import command_location
if command_location:
fig.text(
0.01,
0.5,
command_location,
rotation=90,
fontsize=12,
color="gray",
alpha=0.5,
verticalalignment="center",
)
def plot_z_fitpar(fig: plt.figure, fit_par: str, img_id: int, channel: int,
led_arrays: Union[Tuple[int, ...], int]) -> None:
"""plots the height of a LED array against one fit parameter"""
# make led_arrays a tuple
if type(led_arrays) == int:
led_arrays = (led_arrays,)
fit_parameters = calc.read_hdf(channel)
fit_parameters = calc.include_column_if_nonexistent(fit_parameters, fit_par, channel)
fit_parameters = fit_parameters.loc[img_id, :]
ax = fig.gca(xlabel=fit_par, ylabel='height/m')
for line in led_arrays:
plot, = ax.plot(np.array(fit_parameters[fit_parameters['line'] == line][fit_par]),
np.array(fit_parameters[fit_parameters['line'] == line]['height']))
plot.set_label(f'LED_Array{line}, C{channel}')
ax.legend()
plt.title(f'Plot of fit parameter {fit_par} against the height.\n'
f'Image: {img_id}')
def __init__(self, title=None, rows=0, cols=0,
hspace=None, vspace=None,
hpadding=None, vpadding=None):
if hspace is None:
hspace = self.default_hspace
if vspace is None:
vspace = self.default_vspace
if hpadding is None:
hpadding = self.default_hpadding
if vpadding is None:
vpadding = self.default_vpadding
self.figure = Figure()
self.figure.plotter = self
self.title = self.figure.suptitle("" if title is None else title)
self.rows = rows
self.cols = cols
self.hspace = hspace
self.vspace = vspace
self.hpadding = hpadding
self.vpadding = vpadding
self.active_axes = None
def process_abba_band(recipe, utdate, refdate, band,
groupname,
obsids, frametypes,
config,
do_interactive_figure=False,
threshold_a0v=0.1,
objname="",
multiply_model_a0v=False,
html_output=False,
a0v_obsid=None,
basename_postfix=None):
target_type, nodding_type = recipe.split("_")
if target_type in ["A0V"]:
FIX_TELLURIC=False
elif target_type in ["STELLAR", "EXTENDED"]:
FIX_TELLURIC=True
else:
raise ValueError("Unknown recipe : %s" % recipe)
from recipe_extract_base import RecipeExtractPR
extractor = RecipeExtractPR(utdate, band,
obsids, config)
master_obsid = extractor.pr.master_obsid
igr_path = extractor.pr.igr_path
mastername = igr_path.get_basename(band, groupname)
tgt = extractor.get_oned_spec_helper(mastername,
basename_postfix=basename_postfix)
orders_w_solutions = extractor.orders_w_solutions
# if wavelengths list are sorted in increasing order of
# wavelength, recerse the order list
if tgt.um[0][0] < tgt.um[-1][0]:
orders_w_solutions = orders_w_solutions[::-1]
if FIX_TELLURIC:
if (a0v_obsid is None) or (a0v_obsid == "1"):
A0V_basename = extractor.basenames["a0v"]
else:
A0V_basename = "SDC%s_%s_%04d" % (band, utdate, int(a0v_obsid))
print A0V_basename
a0v = extractor.get_oned_spec_helper(A0V_basename,
basename_postfix=basename_postfix)
tgt_spec_cor = get_tgt_spec_cor(tgt, a0v,
threshold_a0v,
multiply_model_a0v,
config)
# prepare i1i2_list
i1i2_list = get_i1i2_list(extractor,
orders_w_solutions)
fig_list = []
if 1:
if do_interactive_figure:
from matplotlib.pyplot import figure as Figure
else:
from matplotlib.figure import Figure
fig1 = Figure(figsize=(12,6))
fig_list.append(fig1)
ax1a = fig1.add_subplot(211)
ax1b = fig1.add_subplot(212, sharex=ax1a)
for wvl, s, sn in zip(tgt.um,
tgt.spec, tgt.sn):
#s[s<0] = np.nan
#sn[sn<0] = np.nan
ax1a.plot(wvl, s)
ax1b.plot(wvl, sn)
ax1a.set_ylabel("Counts [DN]")
ax1b.set_ylabel("S/N per Res. Element")
ax1b.set_xlabel("Wavelength [um]")
ax1a.set_title(objname)
if FIX_TELLURIC:
fig2 = Figure(figsize=(12,6))
fig_list.append(fig2)
ax2a = fig2.add_subplot(211)
ax2b = fig2.add_subplot(212, sharex=ax2a)
#from igrins.libs.stddev_filter import window_stdev
for wvl, s, t in zip(tgt.um,
tgt_spec_cor,
a0v.flattened):
ax2a.plot(wvl, t, "0.8", zorder=0.5)
ax2b.plot(wvl, s, zorder=0.5)
s_max_list = []
s_min_list = []
for s in tgt_spec_cor[3:-3]:
s_max_list.append(np.nanmax(s))
s_min_list.append(np.nanmin(s))
s_max = np.max(s_max_list)
s_min = np.min(s_min_list)
ds_pad = 0.05 * (s_max - s_min)
ax2a.set_ylabel("A0V flattened")
ax2a.set_ylim(-0.05, 1.1)
ax2b.set_ylabel("Target / A0V")
ax2b.set_xlabel("Wavelength [um]")
ax2b.set_ylim(s_min-ds_pad, s_max+ds_pad)
ax2a.set_title(objname)
# save figures
if fig_list:
for fig in fig_list:
fig.tight_layout()
# tgt_basename = extractor.pr.tgt_basename
tgt_basename = mastername
dirname = "spec_"+tgt_basename
basename_postfix_s = basename_postfix if basename_postfix is not None else ""
filename_prefix = "spec_" + tgt_basename + basename_postfix_s
figout = igr_path.get_section_filename_base("QA_PATH",
filename_prefix,
dirname)
#figout = obj_path.get_secondary_path("spec", "spec_dir")
from igrins.libs.qa_helper import figlist_to_pngs
figlist_to_pngs(figout, fig_list)
# save html
if html_output:
if basename_postfix is not None:
igr_log.warn("For now, no html output is generated if basename-postfix option is used")
else:
dirname = config.get_value('HTML_PATH', utdate)
from igrins.libs.path_info import get_zeropadded_groupname
objroot = get_zeropadded_groupname(groupname)
html_save(utdate, dirname, objroot, band,
orders_w_solutions, tgt.um,
tgt.spec, tgt.sn, i1i2_list)
if FIX_TELLURIC:
objroot = get_zeropadded_groupname(groupname)+"A0V"
html_save(utdate, dirname, objroot, band,
orders_w_solutions, tgt.um,
a0v.flattened, tgt_spec_cor, i1i2_list,
spec_js_name="jj_a0v.js")
if do_interactive_figure:
import matplotlib.pyplot as plt
plt.show()
def process_abba_band(recipe,
utdate,
refdate,
band,
groupname,
obsids,
frametypes,
config,
do_interactive_figure=False,
threshold_a0v=0.1,
objname="",
multiply_model_a0v=False,
html_output=False,
a0v_obsid=None,
basename_postfix=None):
target_type, nodding_type = recipe.split("_")
if target_type in ["A0V"]:
FIX_TELLURIC = False
elif target_type in ["STELLAR", "EXTENDED"]:
FIX_TELLURIC = True
else:
raise ValueError("Unknown recipe : %s" % recipe)
from recipe_extract_base import RecipeExtractPR
extractor = RecipeExtractPR(utdate, band, obsids, config)
master_obsid = extractor.pr.master_obsid
igr_path = extractor.pr.igr_path
mastername = igr_path.get_basename(band, groupname)
tgt = extractor.get_oned_spec_helper(mastername,
basename_postfix=basename_postfix)
orders_w_solutions = extractor.orders_w_solutions
# if wavelengths list are sorted in increasing order of
# wavelength, recerse the order list
if tgt.um[0][0] < tgt.um[-1][0]:
orders_w_solutions = orders_w_solutions[::-1]
if FIX_TELLURIC:
if (a0v_obsid is None) or (a0v_obsid == "1"):
A0V_basename = extractor.basenames["a0v"]
else:
A0V_basename = "SDC%s_%s_%04d" % (band, utdate, int(a0v_obsid))
print A0V_basename
a0v = extractor.get_oned_spec_helper(A0V_basename,
basename_postfix=basename_postfix)
tgt_spec_cor = get_tgt_spec_cor(tgt, a0v, threshold_a0v,
multiply_model_a0v, config)
# prepare i1i2_list
i1i2_list = get_i1i2_list(extractor, orders_w_solutions)
fig_list = []
if 1:
if do_interactive_figure:
from matplotlib.pyplot import figure as Figure
else:
from matplotlib.figure import Figure
fig1 = Figure(figsize=(12, 6))
fig_list.append(fig1)
ax1a = fig1.add_subplot(211)
ax1b = fig1.add_subplot(212, sharex=ax1a)
for wvl, s, sn in zip(tgt.um, tgt.spec, tgt.sn):
#s[s<0] = np.nan
#sn[sn<0] = np.nan
ax1a.plot(wvl, s)
ax1b.plot(wvl, sn)
ax1a.set_ylabel("Counts [DN]")
ax1b.set_ylabel("S/N per Res. Element")
ax1b.set_xlabel("Wavelength [um]")
ax1a.set_title(objname)
if FIX_TELLURIC:
fig2 = Figure(figsize=(12, 6))
fig_list.append(fig2)
ax2a = fig2.add_subplot(211)
ax2b = fig2.add_subplot(212, sharex=ax2a)
#from igrins.libs.stddev_filter import window_stdev
for wvl, s, t in zip(tgt.um, tgt_spec_cor, a0v.flattened):
ax2a.plot(wvl, t, "0.8", zorder=0.5)
ax2b.plot(wvl, s, zorder=0.5)
s_max_list = []
s_min_list = []
for s in tgt_spec_cor[3:-3]:
s_max_list.append(np.nanmax(s))
s_min_list.append(np.nanmin(s))
s_max = np.max(s_max_list)
s_min = np.min(s_min_list)
ds_pad = 0.05 * (s_max - s_min)
ax2a.set_ylabel("A0V flattened")
ax2a.set_ylim(-0.05, 1.1)
ax2b.set_ylabel("Target / A0V")
ax2b.set_xlabel("Wavelength [um]")
ax2b.set_ylim(s_min - ds_pad, s_max + ds_pad)
ax2a.set_title(objname)
# save figures
if fig_list:
for fig in fig_list:
fig.tight_layout()
# tgt_basename = extractor.pr.tgt_basename
tgt_basename = mastername
dirname = "spec_" + tgt_basename
basename_postfix_s = basename_postfix if basename_postfix is not None else ""
filename_prefix = "spec_" + tgt_basename + basename_postfix_s
figout = igr_path.get_section_filename_base("QA_PATH", filename_prefix,
dirname)
#figout = obj_path.get_secondary_path("spec", "spec_dir")
from igrins.libs.qa_helper import figlist_to_pngs
figlist_to_pngs(figout, fig_list)
# save html
if html_output:
if basename_postfix is not None:
igr_log.warn(
"For now, no html output is generated if basename-postfix option is used"
)
else:
dirname = config.get_value('HTML_PATH', utdate)
from igrins.libs.path_info import get_zeropadded_groupname
objroot = get_zeropadded_groupname(groupname)
html_save(utdate, dirname, objroot, band, orders_w_solutions,
tgt.um, tgt.spec, tgt.sn, i1i2_list)
if FIX_TELLURIC:
objroot = get_zeropadded_groupname(groupname) + "A0V"
html_save(utdate,
dirname,
objroot,
band,
orders_w_solutions,
tgt.um,
a0v.flattened,
tgt_spec_cor,
i1i2_list,
spec_js_name="jj_a0v.js")
if do_interactive_figure:
import matplotlib.pyplot as plt
plt.show()
def make_plots_mpl(data, timeDeltaDict=None):
times = [d[0] for d in data]
nTplot = 1 + ((len(supply.temperatures) - 1) // 2)
nax = len(aNames) + len(rNames)
plotWidth = 3.6
plotHeight = 0.8 * (len(aNames) + len(rNames) + nTplot)
minPlotHeight = len(supply.outPins) * 0.42 # inch
plotHeight = max(plotHeight, minPlotHeight)
dpi = 100
fig = Figure(figsize=(plotWidth * 1.6, plotHeight * 1.6), dpi=dpi)
fig.set_facecolor(facecolor)
if not isTest:
canvas = FigureCanvasAgg(fig) # noqa
kw = {}
if versiontuple(mpl.__version__) >= versiontuple("2.0.0"):
kw['facecolor'] = facecolor
else:
kw['axisbg'] = facecolor
gs = gridspec.GridSpec(nax + 2,
1,
height_ratios=[2 * nTplot] +
[2 for i in range(nax)] + [1])
ax0 = fig.add_subplot(gs[0], **kw)
axi = [fig.add_subplot(gs[i], sharex=ax0, **kw) for i in range(1, nax + 2)]
for ax in [ax0] + axi:
for spine in ['bottom', 'top', 'left', 'right']:
ax.spines[spine].set_color(spinecolor)
ax.xaxis.label.set_color(spinecolor)
ax.yaxis.label.set_color(spinecolor)
ax.tick_params(axis='x', colors=spinecolor)
ax.tick_params(axis='y', colors=spinecolor)
tp = dict(bottom=True, top=True, labelbottom=False)
for ax in axi:
ax.tick_params(axis="x", labeltop=False, **tp)
ax0.tick_params(axis="x", labeltop=True, **tp)
axi[-1].set_xlabel(u'date time', fontsize=13)
axt = ax0 # temperature axes
axs = axi[-1] # state axes
axd = axi[:-1] # other sensor axes
axNames = ['temperature'] + aNames + rNames + ['ios']
axUnits = [supply.temperatureUnit] + aUnits + rUnits + ['']
for it, color in enumerate(tColors):
datat = [d[it + 2] for d in data]
lo = supply.temperatureOutlierLimits
times0 = [t for t, d in zip(times, datat) if lo[0] < d < lo[1]]
datat0 = [d for t, d in zip(times, datat) if lo[0] < d < lo[1]]
# td = [(t, d) for t, d in zip(times, datat) if lo[0] < d < lo[1]]
# times0, datat0 = zip(*td)
axt.plot(times0,
datat0,
'o-',
lw=lw,
color=color,
alpha=alpha,
ms=ms,
markeredgecolor=color)
for it, (ax, color) in enumerate(zip(axd, aColors + rColors)):
datao = [d[it + 2 + len(tNames)] for d in data]
ax.plot(times,
datao,
'o-',
lw=lw,
color=color,
alpha=alpha,
ms=ms,
markeredgecolor=color)
ioNames = supply.plotPins
onVals = [1.15 - i * 0.1 for i in range(len(ioNames))]
ioColors = []
ioDisplayed = []
ioState = []
for name, onVal in zip(ioNames, onVals):
if name in ioKeys:
ioDisplayed.append(name)
iio = ioKeys.index(name)
dataIO = [onVal if d[1] & 2**iio else 1 - onVal for d in data]
ioState.append(1 if dataIO[-1] > 0.5 else 0)
color = pColors[iio]
ioColors.append(color)
axs.plot(times,
dataIO,
'o-',
lw=lw,
color=color,
alpha=alpha,
ms=ms,
markeredgecolor=color)
if timeDeltaDict is not None:
now = datetime.datetime.now()
ax0.set_xlim([now - datetime.timedelta(**timeDeltaDict), now])
if yrange == "user":
axt.set_ylim(supply.temperatureDisplayLimits)
elif yrange == "auto":
axt.set_ylim([None, None])
axs.set_ylim(-0.2, 1.2)
axs.set_yticks([0, 1])
axs.set_yticklabels(['off', 'on'])
for ax, lims in zip(axd, aLimits + rLimits):
if yrange == "user":
ax.set_ylim(*lims)
elif yrange == "auto":
ax.set_ylim([None, None])
# fig.autofmt_xdate()
fig.canvas.draw()
if versiontuple(mpl.__version__) < versiontuple("2.0.0"):
tlabels = [item.get_text() for item in axi[-1].get_xticklabels()]
posdot = tlabels[0].find('.0')
if posdot > 0:
tlabels = [item[:posdot] for item in tlabels]
axi[-1].set_xticklabels(tlabels)
fig.subplots_adjust(left=0.09,
bottom=0.04,
right=0.98,
top=0.94,
hspace=0.04)
for ax, name, unit in zip([ax0] + axi, axNames, axUnits):
label = u'{0} ({1})'.format(name, unit) if unit else name
if name == 'ios':
color_text(0.01,
0.65,
ioDisplayed,
ioState,
ioColors,
ax,
va='top',
fontsize=13,
alpha=0.7)
else:
ax.text(0.01,
0.95,
label,
transform=ax.transAxes,
va='top',
fontsize=13,
color=spinecolor,
alpha=0.7)
if timeDeltaDict is not None:
ax0.annotate('',
xy=(1, 1),
xycoords='axes fraction',
xytext=(1, 1.25),
textcoords='axes fraction',
arrowprops=dict(color=spinecolor,
width=1,
headwidth=7,
headlength=10))
fig.text(0.01,
0.01,
'{0} time points'.format(len(times)),
color=spinecolor,
alpha=0.25)
xticklabels = ax0.get_xticklabels()
ax0.set_xticklabels(xticklabels[:-1], rotation=20, ha="left")
if isTest:
fig.show()
else:
buf = BytesIO()
fig.savefig(buf, format="png", facecolor=facecolor)
return (base64.b64encode(buf.getvalue()).decode("ascii"),
plotHeight * dpi)
def run(filename,
outfilename,
plot_dir=None,
tell_file='data/TelluricModel.dat',
blaze_corrected=False):
# prepare figures and axes
if plot_dir is None: #interactive mode
from matplotlib.pyplot import figure as Figure
else:
from matplotlib.figure import Figure
fig1 = Figure(figsize=(12, 6))
ax1 = fig1.add_subplot(111)
orders, order_numbers = read_data(filename, debug=False)
tell_model = get_tell_model(tell_file)
kwargs = {}
if blaze_corrected:
kwargs["N"] = None
else:
print 'Roughly removing blaze function for {}'.format(filename)
filtered_orders, original_pixels = remove_blaze(orders, tell_model,
**kwargs)
corrected_orders = []
print 'Optimizing wavelength for order ',
for o_n, order in zip(order_numbers, filtered_orders):
l_orig = plot_spec(ax1, order[0], order[1], 'k-', alpha=0.4)
l_model = plot_spec(ax1,
order[0],
tell_model(order[0]),
'r-',
alpha=0.6)
# Use the wavelength fit function to fit the wavelength.
print ' {}'.format(o_n),
sys.stdout.flush()
new_order = optimize_wavelength(order, tell_model, fitorder=2)
l_modified = plot_spec(ax1,
new_order[0],
new_order[1],
'g-',
alpha=0.4)
corrected_orders.append(new_order)
# do not trt to plot if number valid points is less than 2
if len(order[0]) < 2:
continue
ax1.legend([l_model, l_orig, l_modified],
["Telluric Model", "Original Spec.", "Modified Spec."])
if plot_dir is not None:
ax1.set_title('Individual order fit : {}'.format(o_n))
ax1.set_xlim(order[0][0], order[0][-1])
ax1.set_ylim(-0.05, 1.15)
figout = os.path.join(plot_dir, 'individual_order')
postfix = "_%03d" % (o_n, )
from igrins.libs.qa_helper import fig_to_png
fig_to_png(figout, fig1, postfix=postfix)
# fig1.savefig('{}{}-individual_order{}.png'.format(plot_dir, filename.split('/')[-1], i+1))
ax1.cla()
print
original_orders = [o.copy() for o in orders]
# Now, fit the entire chip to a surface
fig3d = Figure()
import warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore")
ax3d = fig3d.add_subplot(111, projection='3d')
final_orders = fit_chip(original_orders,
corrected_orders,
original_pixels,
order_numbers,
tell_model,
ax3d=ax3d)
if plot_dir is not None:
figout = os.path.join(plot_dir, 'fullchip_fit')
from igrins.libs.qa_helper import fig_to_png
fig_to_png(figout, fig3d)
fig3 = Figure(figsize=(12, 6))
ax3 = fig3.add_subplot(111)
# Filter again just for plotting
final_filtered, _ = remove_blaze(final_orders, tell_model, **kwargs)
for o_n, order in zip(order_numbers, final_filtered):
l_final = plot_spec(ax3, order[0], order[1], 'k-', alpha=0.4)
l_model = plot_spec(ax3,
order[0],
tell_model(order[0]),
'r-',
alpha=0.6)
if len(order[0]) < 2:
continue
ax3.legend([l_model, l_final], ["Telluric Model", "Final Spec."])
if plot_dir is not None:
ax3.set_title('Final wavelength solution : {}'.format(o_n))
ax3.set_xlim(order[0][0], order[0][-1])
ax3.set_ylim(-0.05, 1.15)
figout = os.path.join(plot_dir, 'final_order')
postfix = "_%03d" % (o_n, )
from igrins.libs.qa_helper import fig_to_png
fig_to_png(figout, fig3, postfix=postfix)
# fig1.savefig('{}{}-individual_order{}.png'.format(plot_dir, filename.split('/')[-1], i+1))
ax3.cla()
if plot_dir is None:
ax3.set_title('Final wavelength solution')
import matplotlib.pyplot as plt
with warnings.catch_warnings():
warnings.simplefilter("ignore")
plt.show()
# Output
wave_arr = np.array([o[0] for o in final_orders])
hdulist = fits.PrimaryHDU(wave_arr)
hdulist.writeto(outfilename, clobber=True)
def _save_to_s3(self, full_key_path: str, plot: figure):
bytes_buffer = io.BytesIO()
plot.savefig(bytes_buffer, **self._save_args)
with self._s3.open(full_key_path, mode="wb") as s3_file:
s3_file.write(bytes_buffer.getvalue())
def _create_ax(self, fig: plt.figure):
"""
Create the Axes onto which the plotted data will be drawn.
"""
return fig.add_subplot(1, 1, 1)
class Plotter(object):
"""This class is responsible for managing the layout of a figure, and also implementing
the plotting commands for simple graphs, hopefully making it even easier than using
matplotlib directly."""
ACTIVE_AXES = "active axes" # A constant used as default target of the plotting methods
default_hspace = 1.0 / 8.0
default_vspace = 1.0 / 8.0
default_hpadding = 0.05
default_vpadding = 0.05
def __init__(self, title=None, rows=0, cols=0,
hspace=None, vspace=None,
hpadding=None, vpadding=None):
if hspace is None:
hspace = self.default_hspace
if vspace is None:
vspace = self.default_vspace
if hpadding is None:
hpadding = self.default_hpadding
if vpadding is None:
vpadding = self.default_vpadding
self.figure = Figure()
self.figure.plotter = self
self.title = self.figure.suptitle("" if title is None else title)
self.rows = rows
self.cols = cols
self.hspace = hspace
self.vspace = vspace
self.hpadding = hpadding
self.vpadding = vpadding
self.active_axes = None
def update(self):
"""Show changes in the figure."""
self.figure.show()
def save(self, *args, **kwargs):
"""Save the figure. Please refer to matplotlib's documentation."""
self.figure.savefig(*args, **kwargs)
def close(self):
close_figure(self.figure)
def layout(self, rows=None, cols=None, update=True):
"""Change the layout of the figure, i.e. the number of rows and columns."""
n = len(self.figure.axes)
if rows is None and cols is None:
rows = int(round(sqrt(n)))
cols = rows + (1 if n > rows**2 else 0)
elif rows is None:
rows = int(ceil(float(n) / cols))
elif cols is None:
cols = int(ceil(float(n) / rows))
elif rows * cols < n:
raise ValueError("insufficient cells")
self.rows = rows
self.cols = cols
self.redraw(update)
def spacing(self, hspace=None, vspace=None, update=True):
"""Change the spacing between axes in the figure."""
if hspace is None and vspace is None:
return
if hspace is not None:
if not 0.0 <= hspace <= 1.0:
raise ValueError("illegal horizontal spacing (must be in [0, 1])")
self.hspace = hspace
if vspace is not None:
if not 0.0 <= vspace <= 1.0:
raise ValueError("illegal vertical spacing (must be in [0, 1])")
self.vspace = vspace
self.redraw(update)
def padding(self, hpadding=None, vpadding=None, update=True):
if hpadding is None and vpadding is None:
return
if hpadding is not None:
if not 0.0 <= hpadding < 0.5:
raise ValueError("illegal horizontal spacing (must be in [0, 0.5))")
self.hpadding = hpadding
if vpadding is not None:
if not 0.0 <= vpadding < 0.5:
raise ValueError("illegal vertical spacing (must be in [0, 0.5))")
self.vpadding = vpadding
self.redraw(update)
def redraw(self, update=True):
"""Redraw the axes in the figure. This is usually used only after changes to layout
or spacing in the figure."""
total_width = self.cols * (1.0 + 2.0*self.hspace)
total_height = self.rows * (1.0 + 2.0*self.vspace)
plot_width = (1.0 - 2.0*self.hpadding) / total_width
plot_height = (1.0 - 2.0*self.vpadding) / total_height
space_width = self.hspace * plot_width
space_height = self.vspace * plot_height
# Reposition the axes according to the new dimensions
n = len(self.figure.axes)
index = 0
y_pos = 1.0 - plot_height - space_height - self.vpadding
for _ in xrange(self.rows):
x_pos = space_width + self.hpadding
for x in xrange(self.cols):
axes = self.figure.axes[index]
axes.set_position([x_pos, y_pos, plot_width, plot_height])
index += 1
x_pos += plot_width + 2 * space_width
if index == n:
break
y_pos -= plot_height + 2 * space_height
if index == n:
break
if update:
self.update()
def set_title(self, title, update=True):
"""Set the title of the figure (not axes!)."""
self.title.set_text("" if title is None else title)
if update: self.update()
def set_size(self, width, height, inches=False):
"""Sets the size of the image in pixels or inches (if 'inches' is true)."""
dpi = self.figure.get_dpi()
if not inches:
width /= dpi
height /= dpi
self.figure.set_size_inches(width, height, forward=True)
def add_axes(self, make_active=True):
"""Add a new axes to the figure and place it in the right position. If the current
grid of graphs cannot accommodate the new axes, the figure layout is recalculated."""
axes = self.figure.add_subplot(1, 1, 1, label=str(len(self.figure.axes)))
if make_active:
self.active_axes = axes
rows, cols = None, None
if self.rows * self.cols >= len(self.figure.axes):
rows, cols = self.rows, self.cols
self.layout(rows, cols, update=False)
return axes
def config_axes(self, axes=ACTIVE_AXES, title=None, xlabel=None, ylabel=None,
xlimit=None, ylimit=None, legend=None, grid=None, update=True):
"""A many-in-one configuration method. Saves a few boring lines of matplotlib code."""
axes = self.get_axes(axes)
if title is not None: axes.set_title(title)
if xlabel is not None: axes.set_xlabel(xlabel)
if ylabel is not None: axes.set_ylabel(ylabel)
if xlimit is not None: axes.set_xlim(xlimit)
if ylimit is not None: axes.set_ylim(ylimit)
if legend is not None: axes.legend(loc=legend)
if grid is not None: axes.grid(bool(grid))
if update:
self.update()
def get_axes(self, axes=ACTIVE_AXES):
"""This method can be used to check if a given axes belongs to the figure, retrieve
the currently active axes, or add new axes to the figure."""
if axes is Plotter.ACTIVE_AXES:
if self.active_axes is None:
self.add_axes(make_active=True)
return self.active_axes
if axes is None:
return self.add_axes(make_active=False)
if axes in self.figure.axes:
return axes
raise Exception("axes does not belong to this Plotter")
def set_active(self, axes):
"""Set the plotter's active axes, i.e. the default target of plotting commands."""
if axes not in self.figure.axes:
raise Exception("axes does not belong to this Plotter")
self.active_axes = axes
# -------------------------------------------------
# Plotting methods
@contextmanager
def plotting_on(self, axes, update=True):
yield self.get_axes(axes)
if update:
self.update()
def legend(self, axes=ACTIVE_AXES, update=True, **kwargs):
"""Add a legend to the given axes."""
with self.plotting_on(axes, update) as axes:
axes.legend(**kwargs)
return axes
def pie_chart(self, values, freqs, axes=ACTIVE_AXES, update=True, **kwargs):
with self.plotting_on(axes, update) as axes:
axes.pie(freqs, labels=values, **kwargs)
return axes
def bar_chart(self, values, freqs, axes=ACTIVE_AXES, update=True, **kwargs):
with self.plotting_on(axes, update) as axes:
data = self.__prepare_bar_chart(values, freqs)
axes.xaxis.set_ticks(data.xtick_locs)
axes.xaxis.set_ticklabels(data.xtick_labels)
axes.bar(data.left, data.height, width=data.bar_width, **kwargs)
return axes
def pareto_chart(self, values, freqs, axes=ACTIVE_AXES, update=True, **kwargs):
with self.plotting_on(axes, update) as axes:
data = self.__prepare_pareto_chart(values, freqs)
axes.xaxis.set_ticks(data.xtick_locs)
axes.xaxis.set_ticklabels(data.xtick_labels)
axes.bar(data.left, data.height, width=data.bar_width, **kwargs)
axes.plot(data.xs, data.ys, "r-", label="Cumulative frequency")
return axes
def histogram(self, values, freqs=None, bins=10, axes=ACTIVE_AXES, update=True, **kwargs):
with self.plotting_on(axes, update) as axes:
data = self.__prepare_histogram(values, freqs, bins)
axes.bar(data.left, data.height, width=data.bar_width, **kwargs)
return axes
def box_plot(self, values, axes=ACTIVE_AXES, update=True, **kwargs):
with self.plotting_on(axes, update) as axes:
axes.boxplot(values, **kwargs)
return axes
def run_chart(self, times, values, numeric=True, axes=ACTIVE_AXES, update=True, **kwargs):
"""A run chart of a time series."""
with self.plotting_on(axes, update) as axes:
data = self.__prepare_run_chart(list(times), list(values), numeric)
if not numeric:
axes.yaxis.set_ticks(data.ytick_locs)
axes.yaxis.set_ticklabels(data.ytick_labels)
axes.plot(data.xs, data.ys, **kwargs)
return axes
def line_plot(self, xs, ys, axes=ACTIVE_AXES, update=True, **kwargs):
"""A simple 2D line plot."""
with self.plotting_on(axes, update) as axes:
axes.plot(list(xs), list(ys), **kwargs)
return axes
def function_plot(self, function, start=0, stop=1.0, observations=100,
axes=ACTIVE_AXES, update=True, **kwargs):
"""Make a quick plot of a function on a given interval."""
xs, ys = [], []
dx = float(stop - start) / (observations - 1)
x = start
for i in xrange(observations):
xs.append(x)
ys.append(function(x))
x += dx
return self.line_plot(xs, ys, axes=axes, update=True, **kwargs)
# -------------------------------------------------
# Preparation of data for plotting
def __prepare_bar_chart(self, values, freqs, bar_width=1.0, bar_space=0.5):
left = [(bar_width + bar_space) * x for x in xrange(len(freqs))]
xtick_locs = [l + bar_width / 2.0 for l in left]
return Namespace(left=left, height=freqs,
bar_width=bar_width,
xtick_locs=xtick_locs,
xtick_labels=values)
def __prepare_pareto_chart(self, values, freqs, bar_width=1.0):
total = float(sum(freqs))
items = sorted([(f / total, v) for f, v in zip(freqs, values)], reverse=True)
height = []
left = [bar_width * x for x in xrange(len(items))]
xtick_locs = [l + bar_width / 2.0 for l in left]
xtick_labels = []
xs = [bar_width * x for x in xrange(len(items) + 1)]
ys = [0.0]
for f, v in items:
height.append(f)
xtick_labels.append(v)
ys.append(ys[-1] + f)
return Namespace(left=left, height=height,
xs=xs, ys=ys,
bar_width=bar_width,
xtick_locs=xtick_locs,
xtick_labels=xtick_labels)
def __prepare_histogram(self, values, freqs, bins):
if freqs is None:
freqs = [1.0] * len(values)
items = sorted(zip(values, freqs))
minimum = items[ 0][0]
maximum = items[-1][0]
total_freq = sum(freqs)
bin_span = float(maximum - minimum) / bins
bin_end = [minimum + bin_span * (x + 1) for x in xrange(bins)]
bin_end[-1] = maximum
bin_freq = [0.0] * bins
cur_bin = 0
for v, f in items:
while v > bin_end[cur_bin]:
cur_bin += 1
bin_freq[cur_bin] += f / (bin_span * total_freq)
return Namespace(left=[end - bin_span for end in bin_end],
height=bin_freq,
bar_width=bin_span)
def __prepare_run_chart(self, times, values, numeric):
xs = []
ys = []
prev_y = values[0]
for y, t in zip(values, times):
xs.extend((t, t))
ys.extend((prev_y, y))
prev_y = y
ytick_locs = None
ytick_labels = None
if not numeric:
# map objects to integer y values if the tseries is not numeric
y_set = sorted(set(ys))
y_mapping = dict(zip(y_set, xrange(len(y_set))))
ys = [y_mapping[y] for y in ys]
# prepare y ticks explaining the translation from objects to integers
yticks = sorted((i, v) for v, i in y_mapping.iteritems())
ytick_locs = [i for i, _ in yticks]
ytick_labels = [v for _, v in yticks]
return Namespace(xs=xs, ys=ys,
ytick_locs=ytick_locs,
ytick_labels=ytick_labels)
def Plot_Solution(
fig : plt.figure,
Axes : numpy.ndarray,
Sol_NN : Neural_Network,
PDE_NN : Neural_Network,
Time_Derivative_Order : int,
Spatial_Derivative_Order : int,
Data : Data_Container) -> None:
""" This function makes four plots. One for the approximate solution, one
for the true solution, one for their difference, and one for the PDE
residual. x_points and t_points specify the domain of all four plots.
Note: this function only works if u is a function of 1 spatial variable.
----------------------------------------------------------------------------
Arguments:
fig: The figure object to which the Axes belong. We need this to set up
the color bars.
Axes: The array of Axes object that we will plot on. Note that this
function overwrites these axes.
Sol_NN: The network that approximates the PDE solution.
PDE_NN: The network that approximates the PDE.
Time_Derivative_Order: The order of the time derivative on the left-hand
side of the PDE.
Spatial_Derivative_Order: The highest order spatial derivatives of Sol_NN we
need to evaluate.
Data: This is a Data_Container object. It should contain four members (all
of which are numpy arrays): x_points, t_points, Data_Set, and Noisy_Data_Set.
x_points, t_points contain the set of possible x and t values, respectively.
Data_Set and Noisy_Data_Set should contain the true solution with and
without noise at each grid point (t, x coordinate). If t_points and x_points
have n_t and n_x elements, respectively, then Data_Set and Noisy_Data_Set
should be an n_x by n_t array whose i,j element holds the value of the true
solution at t_points[j], x_points[i].
----------------------------------------------------------------------------
Returns:
Nothing! """
# First, construct the set of possible coordinates. grid_t_coords and
# grid_x_coords are 2d NumPy arrays. Each row of these arrays corresponds to
# a specific position. Each column corresponds to a specific time.
grid_t_coords, grid_x_coords = numpy.meshgrid(Data.t_points, Data.x_points);
# Flatten t_coords, x_coords. use them to generate grid point coodinates.
flattened_grid_x_coords = grid_x_coords.reshape(-1, 1);
flattened_grid_t_coords = grid_t_coords.reshape(-1, 1);
Grid_Point_Coords = torch.from_numpy(numpy.hstack((flattened_grid_t_coords, flattened_grid_x_coords)));
# Get number of x and t values, respectively.
n_x = len(Data.x_points);
n_t = len(Data.t_points);
# Put networks into evaluation mode.
Sol_NN.eval();
PDE_NN.eval();
# Evaluate the network's approximate solution, the absolute error, and the
# PDE residual at each coordinate. We need to reshape these into n_x by n_t
# grids because that's what matplotlib's contour function wants.
Approx_Sol_on_grid = Evaluate_Approx_Sol(Sol_NN, Grid_Point_Coords).reshape(n_x, n_t);
Error_On_Grid = numpy.abs(Approx_Sol_on_grid - Data.Data_Set);
Residual_on_Grid = Evaluate_Residual(
Sol_NN = Sol_NN,
PDE_NN = PDE_NN,
Time_Derivative_Order = Time_Derivative_Order,
Spatial_Derivative_Order = Spatial_Derivative_Order,
Coords = Grid_Point_Coords).reshape(n_x, n_t);
# Plot the true solution + color bar.
data_min : float = numpy.min(Data.Noisy_Data_Set);
data_max : float = numpy.max(Data.Noisy_Data_Set);
ColorMap0 = Axes[0].contourf( grid_t_coords,
grid_x_coords,
Data.Noisy_Data_Set,
levels = numpy.linspace(data_min, data_max, 500),
cmap = plt.cm.jet);
fig.colorbar(ColorMap0, ax = Axes[0], fraction=0.046, pad=0.04, orientation='vertical');
# Plot the learned solution + color bar
sol_min : float = numpy.min(Approx_Sol_on_grid);
sol_max : float = numpy.max(Approx_Sol_on_grid);
ColorMap1 = Axes[1].contourf( grid_t_coords,
grid_x_coords,
Approx_Sol_on_grid,
levels = numpy.linspace(sol_min, sol_max, 500),
cmap = plt.cm.jet);
fig.colorbar(ColorMap1, ax = Axes[1], fraction=0.046, pad=0.04, orientation='vertical');
# Plot the Error between the approx solution and noise-free data set. + color bar.
error_min : float = numpy.min(Error_On_Grid);
error_max : float = numpy.max(Error_On_Grid);
ColorMap2 = Axes[2].contourf( grid_t_coords,
grid_x_coords,
Error_On_Grid,
levels = numpy.linspace(error_min, error_max, 500),
cmap = plt.cm.jet);
fig.colorbar(ColorMap2, ax = Axes[2], fraction=0.046, pad=0.04, orientation='vertical');
# Plot the residual + color bar
resid_min : float = numpy.min(Residual_on_Grid);
resid_max : float = numpy.max(Residual_on_Grid);
ColorMap3 = Axes[3].contourf( grid_t_coords,
grid_x_coords,
Residual_on_Grid,
levels = numpy.linspace(resid_min, resid_max, 500),
cmap = plt.cm.jet);
fig.colorbar(ColorMap3, ax = Axes[3], fraction=0.046, pad=0.04, orientation='vertical');
# Set tight layout (to prevent overlapping... I have no idea why this isn't
# a default setting. Matplotlib, you are special kind of awful).
fig.tight_layout();
def process_abba_band(recipe, utdate, refdate, band, obsids, frametypes,
config,
do_interactive_figure=False,
threshold_a0v=0.1,
objname="",
multiply_model_a0v=False,
html_output=False):
from libs.products import ProductDB, PipelineStorage
if recipe == "A0V_AB":
FIX_TELLURIC=False
elif recipe == "STELLAR_AB":
FIX_TELLURIC=True
elif recipe == "EXTENDED_AB":
FIX_TELLURIC=True
elif recipe == "EXTENDED_ONOFF":
FIX_TELLURIC=True
else:
raise ValueError("Unsupported Recipe : %s" % recipe)
if 1:
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
obj_filenames = igr_path.get_filenames(band, obsids)
master_obsid = obsids[0]
tgt_basename = os.path.splitext(os.path.basename(obj_filenames[0]))[0]
db = {}
basenames = {}
db_types = ["flat_off", "flat_on", "thar", "sky"]
for db_type in db_types:
db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"%s.db" % db_type,
)
db[db_type] = ProductDB(db_name)
# db on output path
db_types = ["a0v"]
for db_type in db_types:
db_name = igr_path.get_section_filename_base("OUTDATA_PATH",
"%s.db" % db_type,
)
db[db_type] = ProductDB(db_name)
# to get basenames
db_types = ["flat_off", "flat_on", "thar", "sky"]
if FIX_TELLURIC:
db_types.append("a0v")
for db_type in db_types:
basenames[db_type] = db[db_type].query(band, master_obsid)
if 1: # make aperture
from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC
sky_basename = db["sky"].query(band, master_obsid)
wvlsol_products = igr_storage.load([SKY_WVLSOL_JSON_DESC],
sky_basename)[SKY_WVLSOL_JSON_DESC]
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = map(np.array, wvlsol_products["wvl_sol"])
# prepare i1i2_list
from libs.storage_descriptions import ORDER_FLAT_JSON_DESC
prod = igr_storage.load([ORDER_FLAT_JSON_DESC],
basenames["flat_on"])[ORDER_FLAT_JSON_DESC]
new_orders = prod["orders"]
i1i2_list_ = prod["i1i2_list"]
order_indices = []
for o in orders_w_solutions:
o_new_ind = np.searchsorted(new_orders, o)
order_indices.append(o_new_ind)
i1i2_list = get_fixed_i1i2_list(order_indices, i1i2_list_)
from libs.storage_descriptions import (SPEC_FITS_DESC,
SN_FITS_DESC)
if 1: # load target spectrum
tgt_spec_ = igr_storage.load([SPEC_FITS_DESC],
tgt_basename)[SPEC_FITS_DESC]
tgt_spec = list(tgt_spec_.data)
tgt_sn_ = igr_storage.load([SN_FITS_DESC],
tgt_basename)[SN_FITS_DESC]
tgt_sn = list(tgt_sn_.data)
fig_list = []
# telluric
if 1: #FIX_TELLURIC:
A0V_basename = db["a0v"].query(band, master_obsid)
from libs.storage_descriptions import SPEC_FITS_FLATTENED_DESC
telluric_cor_ = igr_storage.load([SPEC_FITS_FLATTENED_DESC],
A0V_basename)[SPEC_FITS_FLATTENED_DESC]
#A0V_path = ProductPath(igr_path, A0V_basename)
#fn = A0V_path.get_secondary_path("spec_flattened.fits")
telluric_cor = list(telluric_cor_.data)
a0v_spec_ = igr_storage.load([SPEC_FITS_DESC],
A0V_basename)[SPEC_FITS_DESC]
a0v_spec = list(a0v_spec_.data)
if 1:
if do_interactive_figure:
from matplotlib.pyplot import figure as Figure
else:
from matplotlib.figure import Figure
fig1 = Figure(figsize=(12,6))
fig_list.append(fig1)
ax1a = fig1.add_subplot(211)
ax1b = fig1.add_subplot(212, sharex=ax1a)
for wvl, s, sn in zip(wvl_solutions, tgt_spec, tgt_sn):
#s[s<0] = np.nan
#sn[sn<0] = np.nan
ax1a.plot(wvl, s)
ax1b.plot(wvl, sn)
ax1a.set_ylabel("Counts [DN]")
ax1b.set_ylabel("S/N per Res. Element")
ax1b.set_xlabel("Wavelength [um]")
ax1a.set_title(objname)
if FIX_TELLURIC:
fig2 = Figure(figsize=(12,6))
fig_list.append(fig2)
ax2a = fig2.add_subplot(211)
ax2b = fig2.add_subplot(212, sharex=ax2a)
#from libs.stddev_filter import window_stdev
tgt_spec_cor = []
#for s, t in zip(s_list, telluric_cor):
for s, t, t2 in zip(tgt_spec, a0v_spec, telluric_cor):
st = s/t
#print np.percentile(t[np.isfinite(t)], 95), threshold_a0v
t0 = np.percentile(t[np.isfinite(t)], 95)*threshold_a0v
st[t<t0] = np.nan
st[t2 < threshold_a0v] = np.nan
tgt_spec_cor.append(st)
if multiply_model_a0v:
# multiply by A0V model
from libs.a0v_spec import A0VSpec
a0v_model = A0VSpec()
a0v_interp1d = a0v_model.get_flux_interp1d(1.3, 2.5,
flatten=True,
smooth_pixel=32)
for wvl, s in zip(wvl_solutions,
tgt_spec_cor):
aa = a0v_interp1d(wvl)
s *= aa
for wvl, s, t in zip(wvl_solutions,
tgt_spec_cor,
telluric_cor):
ax2a.plot(wvl, t, "0.8", zorder=0.5)
ax2b.plot(wvl, s, zorder=0.5)
s_max_list = []
s_min_list = []
for s in tgt_spec_cor[3:-3]:
s_max_list.append(np.nanmax(s))
s_min_list.append(np.nanmin(s))
s_max = np.max(s_max_list)
s_min = np.min(s_min_list)
ds_pad = 0.05 * (s_max - s_min)
ax2a.set_ylabel("A0V flattened")
ax2a.set_ylim(-0.05, 1.1)
ax2b.set_ylabel("Target / A0V")
ax2b.set_xlabel("Wavelength [um]")
ax2b.set_ylim(s_min-ds_pad, s_max+ds_pad)
ax2a.set_title(objname)
# save figures
if fig_list:
for fig in fig_list:
fig.tight_layout()
figout = igr_path.get_section_filename_base("QA_PATH",
"spec_"+tgt_basename,
"spec_"+tgt_basename)
#figout = obj_path.get_secondary_path("spec", "spec_dir")
from libs.qa_helper import figlist_to_pngs
figlist_to_pngs(figout, fig_list)
# save html
if html_output:
dirname = config.get_value('HTML_PATH', utdate)
objroot = "%04d" % (master_obsid,)
html_save(utdate, dirname, objroot, band,
orders_w_solutions, wvl_solutions,
tgt_spec, tgt_sn, i1i2_list)
if FIX_TELLURIC:
objroot = "%04dA0V" % (master_obsid,)
html_save(utdate, dirname, objroot, band,
orders_w_solutions, wvl_solutions,
telluric_cor, tgt_spec_cor, i1i2_list,
spec_js_name="jj_a0v.js")
if do_interactive_figure:
import matplotlib.pyplot as plt
plt.show()
def plot_subimage(fig: plt.figure, hdu: Union[str, fits.HDUList], ra: float, dec: float,
frame: Union[int, float], world_frame: bool = False, title: str = None,
n: int = 1, n_x: int = 1, n_y: int = 1,
cmap: str = 'viridis', show_cbar: bool = False, stretch: str = 'sqrt', vmin: float = None,
vmax: float = None,
show_grid: bool = False,
ticks: int = None, interval: str = 'minmax',
show_coords: bool = True, ylabel: str = None,
font_size: int = 12,
reverse_y=False):
"""
:param fig:
:param hdu:
:param ra:
:param dec:
:param frame: in pixels, or in arcsecs (?) if world_frame is True.
:param world_frame:
:param title:
:param n:
:param n_x:
:param n_y:
:param cmap:
:param show_cbar:
:param stretch:
:param vmin:
:param vmax:
:param show_grid:
:param ticks:
:param interval:
:param show_coords:
:param ylabel:
:param font_size:
:param reverse_y:
:return:
"""
print(hdu)
hdu, path = ff.path_or_hdu(hdu=hdu)
print(hdu[0].data.shape)
hdu_cut = ff.trim_frame_point(hdu=hdu, ra=ra, dec=dec, frame=frame, world_frame=world_frame)
wcs_cut = wcs.WCS(header=hdu_cut[0].header)
print(n_y, n_x, n)
if show_coords:
plot = fig.add_subplot(n_y, n_x, n, projection=wcs_cut)
if ticks is not None:
lat = plot.coords[0]
lat.set_ticks(number=ticks)
else:
plot = fig.add_subplot(n_y, n_x, n)
frame1 = plt.gca()
frame1.axes.get_xaxis().set_visible(False)
frame1.axes.set_yticks([])
# frame1.axes.get_yaxis().set_visible(False)
if show_grid:
plt.grid(color='black', ls='dashed')
if type(vmin) is str:
if vmin == 'median_full':
vmin = np.nanmedian(hdu[0].data)
elif vmin == 'median_cut':
vmin = np.nanmedian(hdu_cut[0].data)
else:
raise ValueError('Unrecognised vmin string argument.')
if interval == 'minmax':
interval = MinMaxInterval()
elif interval == 'zscale':
interval = ZScaleInterval()
else:
raise ValueError('Interval not recognised.')
print(hdu_cut[0].data.shape)
if stretch == 'log':
norm = ImageNormalize(hdu_cut[0].data, interval=interval, stretch=LogStretch(), vmin=vmin, vmax=vmax)
elif stretch == 'sqrt':
norm = ImageNormalize(hdu_cut[0].data, interval=interval, stretch=SqrtStretch(), vmin=vmin, vmax=vmax)
else:
raise ValueError('Stretch not recognised.')
plot.title.set_text(title)
plot.title.set_size(font_size)
print(ylabel)
if ylabel is not None:
plot.set_ylabel(ylabel, size=12)
im = plt.imshow(hdu_cut[0].data, norm=norm, cmap=cmap)
if reverse_y:
plot.invert_yaxis()
c_ticks = np.linspace(norm.vmin, norm.vmax, 5, endpoint=True)
if show_cbar:
cbar = plt.colorbar(im) # ticks=c_ticks)
return plot, hdu_cut
def process_abba_band(recipe,
utdate,
refdate,
band,
obsids,
frametypes,
config,
do_interactive_figure=False,
threshold_a0v=0.1,
objname="",
multiply_model_a0v=False,
html_output=False):
from libs.products import ProductDB, PipelineStorage
if recipe == "A0V_AB":
FIX_TELLURIC = False
elif recipe == "STELLAR_AB":
FIX_TELLURIC = True
elif recipe == "EXTENDED_AB":
FIX_TELLURIC = True
elif recipe == "EXTENDED_ONOFF":
FIX_TELLURIC = True
else:
raise ValueError("Unsupported Recipe : %s" % recipe)
if 1:
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
obj_filenames = igr_path.get_filenames(band, obsids)
master_obsid = obsids[0]
tgt_basename = os.path.splitext(os.path.basename(obj_filenames[0]))[0]
db = {}
basenames = {}
db_types = ["flat_off", "flat_on", "thar", "sky"]
for db_type in db_types:
db_name = igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"%s.db" % db_type,
)
db[db_type] = ProductDB(db_name)
# db on output path
db_types = ["a0v"]
for db_type in db_types:
db_name = igr_path.get_section_filename_base(
"OUTDATA_PATH",
"%s.db" % db_type,
)
db[db_type] = ProductDB(db_name)
# to get basenames
db_types = ["flat_off", "flat_on", "thar", "sky"]
if FIX_TELLURIC:
db_types.append("a0v")
for db_type in db_types:
basenames[db_type] = db[db_type].query(band, master_obsid)
if 1: # make aperture
from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC
sky_basename = db["sky"].query(band, master_obsid)
wvlsol_products = igr_storage.load([SKY_WVLSOL_JSON_DESC],
sky_basename)[SKY_WVLSOL_JSON_DESC]
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = map(np.array, wvlsol_products["wvl_sol"])
# prepare i1i2_list
from libs.storage_descriptions import ORDER_FLAT_JSON_DESC
prod = igr_storage.load([ORDER_FLAT_JSON_DESC],
basenames["flat_on"])[ORDER_FLAT_JSON_DESC]
new_orders = prod["orders"]
i1i2_list_ = prod["i1i2_list"]
order_indices = []
for o in orders_w_solutions:
o_new_ind = np.searchsorted(new_orders, o)
order_indices.append(o_new_ind)
i1i2_list = get_fixed_i1i2_list(order_indices, i1i2_list_)
from libs.storage_descriptions import (SPEC_FITS_DESC, SN_FITS_DESC)
if 1: # load target spectrum
tgt_spec_ = igr_storage.load([SPEC_FITS_DESC],
tgt_basename)[SPEC_FITS_DESC]
tgt_spec = list(tgt_spec_.data)
tgt_sn_ = igr_storage.load([SN_FITS_DESC], tgt_basename)[SN_FITS_DESC]
tgt_sn = list(tgt_sn_.data)
fig_list = []
# telluric
if 1: #FIX_TELLURIC:
A0V_basename = db["a0v"].query(band, master_obsid)
from libs.storage_descriptions import SPEC_FITS_FLATTENED_DESC
telluric_cor_ = igr_storage.load(
[SPEC_FITS_FLATTENED_DESC], A0V_basename)[SPEC_FITS_FLATTENED_DESC]
#A0V_path = ProductPath(igr_path, A0V_basename)
#fn = A0V_path.get_secondary_path("spec_flattened.fits")
telluric_cor = list(telluric_cor_.data)
a0v_spec_ = igr_storage.load([SPEC_FITS_DESC],
A0V_basename)[SPEC_FITS_DESC]
a0v_spec = list(a0v_spec_.data)
if 1:
if do_interactive_figure:
from matplotlib.pyplot import figure as Figure
else:
from matplotlib.figure import Figure
fig1 = Figure(figsize=(12, 6))
fig_list.append(fig1)
ax1a = fig1.add_subplot(211)
ax1b = fig1.add_subplot(212, sharex=ax1a)
for wvl, s, sn in zip(wvl_solutions, tgt_spec, tgt_sn):
#s[s<0] = np.nan
#sn[sn<0] = np.nan
ax1a.plot(wvl, s)
ax1b.plot(wvl, sn)
ax1a.set_ylabel("Counts [DN]")
ax1b.set_ylabel("S/N per Res. Element")
ax1b.set_xlabel("Wavelength [um]")
ax1a.set_title(objname)
if FIX_TELLURIC:
fig2 = Figure(figsize=(12, 6))
fig_list.append(fig2)
ax2a = fig2.add_subplot(211)
ax2b = fig2.add_subplot(212, sharex=ax2a)
#from libs.stddev_filter import window_stdev
tgt_spec_cor = []
#for s, t in zip(s_list, telluric_cor):
for s, t, t2 in zip(tgt_spec, a0v_spec, telluric_cor):
st = s / t
#print np.percentile(t[np.isfinite(t)], 95), threshold_a0v
t0 = np.percentile(t[np.isfinite(t)], 95) * threshold_a0v
st[t < t0] = np.nan
st[t2 < threshold_a0v] = np.nan
tgt_spec_cor.append(st)
if multiply_model_a0v:
# multiply by A0V model
from libs.a0v_spec import A0VSpec
a0v_model = A0VSpec()
a0v_interp1d = a0v_model.get_flux_interp1d(1.3,
2.5,
flatten=True,
smooth_pixel=32)
for wvl, s in zip(wvl_solutions, tgt_spec_cor):
aa = a0v_interp1d(wvl)
s *= aa
for wvl, s, t in zip(wvl_solutions, tgt_spec_cor, telluric_cor):
ax2a.plot(wvl, t, "0.8", zorder=0.5)
ax2b.plot(wvl, s, zorder=0.5)
s_max_list = []
s_min_list = []
for s in tgt_spec_cor[3:-3]:
s_max_list.append(np.nanmax(s))
s_min_list.append(np.nanmin(s))
s_max = np.max(s_max_list)
s_min = np.min(s_min_list)
ds_pad = 0.05 * (s_max - s_min)
ax2a.set_ylabel("A0V flattened")
ax2a.set_ylim(-0.05, 1.1)
ax2b.set_ylabel("Target / A0V")
ax2b.set_xlabel("Wavelength [um]")
ax2b.set_ylim(s_min - ds_pad, s_max + ds_pad)
ax2a.set_title(objname)
# save figures
if fig_list:
for fig in fig_list:
fig.tight_layout()
figout = igr_path.get_section_filename_base("QA_PATH",
"spec_" + tgt_basename,
"spec_" + tgt_basename)
#figout = obj_path.get_secondary_path("spec", "spec_dir")
from libs.qa_helper import figlist_to_pngs
figlist_to_pngs(figout, fig_list)
# save html
if html_output:
dirname = config.get_value('HTML_PATH', utdate)
objroot = "%04d" % (master_obsid, )
html_save(utdate, dirname, objroot, band, orders_w_solutions,
wvl_solutions, tgt_spec, tgt_sn, i1i2_list)
if FIX_TELLURIC:
objroot = "%04dA0V" % (master_obsid, )
html_save(utdate,
dirname,
objroot,
band,
orders_w_solutions,
wvl_solutions,
telluric_cor,
tgt_spec_cor,
i1i2_list,
spec_js_name="jj_a0v.js")
if do_interactive_figure:
import matplotlib.pyplot as plt
plt.show()
def _save_figure(fig: plt.figure, save_file_name: str):
if save_file_name != None:
try:
fig.savefig(save_file_name)
except NotADirectoryError:
logger.warning("Unable to save file " + save_file_name)
def run(filename, outfilename,
plot_dir=None, tell_file='data/TelluricModel.dat',
blaze_corrected=False):
# prepare figures and axes
if plot_dir is None: #interactive mode
from matplotlib.pyplot import figure as Figure
else:
from matplotlib.figure import Figure
fig1 = Figure(figsize=(12,6))
ax1 = fig1.add_subplot(111)
orders, order_numbers = read_data(filename, debug=False)
tell_model = get_tell_model(tell_file)
kwargs = {}
if blaze_corrected:
kwargs["N"] = None
else:
print 'Roughly removing blaze function for {}'.format(filename)
filtered_orders, original_pixels = remove_blaze(orders, tell_model,
**kwargs)
corrected_orders = []
print 'Optimizing wavelength for order ',
for o_n, order in zip(order_numbers, filtered_orders):
l_orig = plot_spec(ax1, order[0], order[1],
'k-', alpha=0.4)
l_model = plot_spec(ax1, order[0], tell_model(order[0]),
'r-', alpha=0.6)
# Use the wavelength fit function to fit the wavelength.
print ' {}'.format(o_n),
sys.stdout.flush()
new_order = optimize_wavelength(order, tell_model,
fitorder=2)
l_modified = plot_spec(ax1, new_order[0], new_order[1],
'g-', alpha=0.4)
corrected_orders.append(new_order)
# do not trt to plot if number valid points is less than 2
if len(order[0]) < 2:
continue
ax1.legend([l_model, l_orig, l_modified],
["Telluric Model",
"Original Spec.",
"Modified Spec."])
if plot_dir is not None:
ax1.set_title('Individual order fit : {}'.format(o_n))
ax1.set_xlim(order[0][0], order[0][-1])
ax1.set_ylim(-0.05, 1.15)
figout = os.path.join(plot_dir, 'individual_order')
postfix="_%03d" % (o_n,)
from libs.qa_helper import fig_to_png
fig_to_png(figout, fig1, postfix=postfix)
# fig1.savefig('{}{}-individual_order{}.png'.format(plot_dir, filename.split('/')[-1], i+1))
ax1.cla()
print
original_orders = [o.copy() for o in orders]
# Now, fit the entire chip to a surface
fig3d = Figure()
import warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore")
ax3d = fig3d.add_subplot(111, projection='3d')
final_orders = fit_chip(original_orders,
corrected_orders,
original_pixels,
order_numbers,
tell_model,
ax3d=ax3d)
if plot_dir is not None:
figout = os.path.join(plot_dir, 'fullchip_fit')
from libs.qa_helper import fig_to_png
fig_to_png(figout, fig3d)
fig3 = Figure(figsize=(12,6))
ax3 = fig3.add_subplot(111)
# Filter again just for plotting
final_filtered, _ = remove_blaze(final_orders, tell_model, **kwargs)
for o_n, order in zip(order_numbers, final_filtered):
l_final = plot_spec(ax3, order[0], order[1], 'k-', alpha=0.4)
l_model = plot_spec(ax3, order[0], tell_model(order[0]),
'r-', alpha=0.6)
if len(order[0]) < 2:
continue
ax3.legend([l_model, l_final],
["Telluric Model",
"Final Spec."])
if plot_dir is not None:
ax3.set_title('Final wavelength solution : {}'.format(o_n))
ax3.set_xlim(order[0][0], order[0][-1])
ax3.set_ylim(-0.05, 1.15)
figout = os.path.join(plot_dir, 'final_order')
postfix="_%03d" % (o_n,)
from libs.qa_helper import fig_to_png
fig_to_png(figout, fig3, postfix=postfix)
# fig1.savefig('{}{}-individual_order{}.png'.format(plot_dir, filename.split('/')[-1], i+1))
ax3.cla()
if plot_dir is None:
ax3.set_title('Final wavelength solution')
import matplotlib.pyplot as plt
with warnings.catch_warnings():
warnings.simplefilter("ignore")
plt.show()
# Output
wave_arr = np.array([o[0] for o in final_orders])
hdulist = fits.PrimaryHDU(wave_arr)
hdulist.writeto(outfilename, clobber=True)
def Update_Axes(fig: plt.figure, Axes: np.ndarray, u_NN: Neural_Network,
Points: torch.Tensor, n: int) -> None:
""" This function plots the approximate solution and residual at the
specified points.
----------------------------------------------------------------------------
Arguments:
fig : The figure object to which the Axes belong. We need this to set up
the color bars.
Axes : The array of Axes object that we will plot on. Note that this
function will overwrite these axes.
u_NN : The neural network that gives the approximate solution to Poisson's
equation.
Points : The set of points we want to evaluate the approximate and true
solutions, as well as the PDE Residual. This should be an (n*n)x2 tensor,
whose ith row holds the x,y coordinates of the ith point we want to plot.
Each element of Points should be an element of [0,1]x[0,1].
n : the number of gridpoints along each axis. Points should be an n*n x 2
tensor.
----------------------------------------------------------------------------
Returns:
Nothing! """
# First, evaluate the network's approximate solution, the true solution, and
# the PDE residual at the specified Points. We need to reshape these into
# nxn grids, because that's what matplotlib's contour function wants. It's
# annoying, but it is what it is.
u_NN_at_Points = Evaluate_NN(u_NN, Points).reshape(n, n)
True_Sol_at_Points = Evaluate_True_Solution(Points).reshape(n, n)
Residual_at_Points = PDE_Residual(u_NN, Points).reshape(n, n)
# Extract the x and y coordinates of points, as np arrays. We also need to
# reshape these as nxn grids (same reason as above.
x = Points[:, 0].numpy().reshape(n, n)
y = Points[:, 1].numpy().reshape(n, n)
# Plot the approximate solution + colorbar.
ColorMap0 = Axes[0].contourf(x,
y,
u_NN_at_Points.reshape(n, n),
levels=50,
cmap=plt.cm.jet)
fig.colorbar(ColorMap0,
ax=Axes[0],
fraction=0.046,
pad=0.04,
orientation='vertical')
# Plot the true solution + colorbar
ColorMap1 = Axes[1].contourf(x,
y,
True_Sol_at_Points.reshape(n, n),
levels=50,
cmap=plt.cm.jet)
fig.colorbar(ColorMap1,
ax=Axes[1],
fraction=0.046,
pad=0.04,
orientation='vertical')
# Plot the residual + colorbar
ColorMap2 = Axes[2].contourf(x,
y,
Residual_at_Points.reshape(n, n),
levels=50,
cmap=plt.cm.jet)
fig.colorbar(ColorMap2,
ax=Axes[2],
fraction=0.046,
pad=0.04,
orientation='vertical')
# Set tight layout (to prevent overlapping... I have no idea why this isn't
# a default setting. Matplotlib, you are special kind of awful).
fig.tight_layout()