class get_steps_schema:
def __init__(self, db="Prod", eTserver="Prod"):
self.eTserver = eTserver
self.db = db
self.runData = None
if self.eTserver == 'Prod':
pS = True
else:
pS = False
self.connect = Connection(
operator='richard',
db=self.db,
exp='LSST-CAMERA',
prodServer=pS)
def get(self, run=None):
if run is None:
print('Error: missing run number')
raise ValueError
self.runData = self.connect.getRunResults(run=run)
return self.runData
def get_test_info(self, runData=None):
if runData is None:
r = self.runData
else:
r = runData
self.test_dict = OrderedDict()
for steps in r["steps"]:
if "acq" in steps:
continue
schema_list = r["steps"][steps]
for schema in schema_list:
if "info" in schema or "versions" in schema:
continue
test_list = schema_list[schema][0]
for item in test_list:
if 'amp' in item or "slot" in item or "raft" in item or "sensor" in item \
or "schema" in item or "job" in item or \
"subset" in item or "file" in item or "host" in item or "wavelength" in item:
continue
self.test_dict[item] = [steps, schema]
return self.test_dict
def __init__(self, run, user='******', prodServer=True):
"""
Parameters
----------
run: str
Run number. If it ends in 'D', the Dev database will be
queried.
user: str ['ccs']
User id to pass to the etravelerAPI.connection.Connnection
initializer.
prodServer: bool [True]
Flag to use the prod or dev eT server.
"""
super(ETResults, self).__init__()
db_name = 'Dev' if run.endswith('D') else 'Prod'
conn = Connection(user, db_name, prodServer=prodServer)
self.results = conn.getRunResults(run=run)
self._extract_schema_values()
class plotGoodRaftRuns():
def __init__(self, db='Prod', server='Prod', base_dir=None):
self.traveler_name = {}
self.test_type = "fe55_raft_analysis"
self.traveler_name['I&T-Raft'] = 'INT-SR-EOT-01'
self.traveler_name['BNL-Raft'] = 'SR-RTM-EOT-03'
self.base_dir = base_dir
self.db = db
self.server = server
if server == 'Prod':
pS = True
else:
pS = False
self.connect = Connection(operator='richard',
db=db,
exp='LSST-CAMERA',
prodServer=pS)
def find_runs(self, site_type=None, runs=None):
#data = self.connect.getResultsJH(htype="LCA-11021_RTM",
# stepName=self.test_type,
# travelerName=self.traveler_name)
if runs is not None:
runs = runs
return runs
def make_run_pages(self, site_type=None, runs=None):
run_list = self.find_runs(site_type=site_type, runs=runs)
raft_list = []
for run in run_list:
data = self.connect.getRunResults(run=run)
raft = data["experimentSN"]
raft_list.append(raft)
self.write_run_plot(run=run, site_type=site_type, raft=raft)
return run_list, raft_list
def write_run_plot(self, run=None, site_type=None, raft=None):
print('Operating on run ', run)
g = get_EO_analysis_results(db=self.db, server=self.server)
raft_list, data = g.get_tests(site_type=site_type,
test_type="gain",
run=run)
res = g.get_results(test_type="gain", data=data, device=raft)
raft_list_ptc, data_ptc = g.get_tests(site_type=site_type,
test_type="ptc_gain",
run=run)
res_ptc_gain = g.get_results(test_type='ptc_gain',
data=data_ptc,
device=raft)
raft_list_psf, data_psf = g.get_tests(site_type=site_type,
test_type="psf_sigma",
run=run)
res_psf = g.get_results(test_type='psf_sigma',
data=data_psf,
device=raft)
raft_list_rn, data_rn = g.get_tests(site_type=site_type,
test_type="read_noise",
run=run)
res_rn = g.get_results(test_type='read_noise',
data=data_rn,
device=raft)
raft_list_cti_low_serial, data_cls = g.get_tests(
site_type=site_type, test_type="cti_low_serial", run=run)
res_cls = g.get_results(test_type='cti_low_serial',
data=data_cls,
device=raft)
raft_list_cti_high_serial, data_chs = g.get_tests(
site_type=site_type, test_type="cti_high_serial", run=run)
res_chs = g.get_results(test_type='cti_high_serial',
data=data_chs,
device=raft)
raft_list_cti_low_parallel, data_clp = g.get_tests(
site_type=site_type, test_type="cti_low_parallel", run=run)
res_clp = g.get_results(test_type='cti_low_parallel',
data=data_clp,
device=raft)
raft_list_cti_high_parallel, data_chp = g.get_tests(
site_type=site_type, test_type="cti_high_parallel", run=run)
res_chp = g.get_results(test_type='cti_high_parallel',
data=data_chp,
device=raft)
raft_list_dark_pixels, data_dp = g.get_tests(site_type=site_type,
test_type="dark_pixels",
run=run)
res_dp = g.get_results(test_type='dark_pixels',
data=data_dp,
device=raft)
raft_list_dark_columns, data_dc = g.get_tests(site_type=site_type,
test_type="dark_columns",
run=run)
res_dc = g.get_results(test_type='dark_columns',
data=data_dc,
device=raft)
raft_list_bright_pixels, data_bp = g.get_tests(
site_type=site_type, test_type="bright_pixels", run=run)
res_bp = g.get_results(test_type='bright_pixels',
data=data_bp,
device=raft)
raft_list_bright_columns, data_bc = g.get_tests(
site_type=site_type, test_type="bright_columns", run=run)
res_bc = g.get_results(test_type='bright_columns',
data=data_bc,
device=raft)
raft_list_traps, data_tp = g.get_tests(site_type=site_type,
test_type="num_traps",
run=run)
res_tp = g.get_results(test_type='num_traps',
data=data_tp,
device=raft)
raft_list_qe, data_qe = g.get_tests(site_type=site_type,
test_type="QE",
run=run)
res_qe = g.get_results(test_type='QE', data=data_qe, device=raft)
raft_list_drkC, data_drkC = g.get_tests(site_type=site_type,
test_type="dark_current_95CL",
run=run)
res_drkC = g.get_results(test_type='dark_current_95CL',
data=data_drkC,
device=raft)
raft_list_fw, data_fw = g.get_tests(site_type=site_type,
test_type="full_well",
run=run)
res_fw = g.get_results(test_type='full_well',
data=data_fw,
device=raft)
raft_list_nonl, data_nonl = g.get_tests(site_type=site_type,
test_type="max_frac_dev",
run=run)
res_nonl = g.get_results(test_type='max_frac_dev',
data=data_nonl,
device=raft)
test_list = []
test_list_ptc = []
test_list_psf = []
test_list_rn = []
test_list_cls = []
test_list_chs = []
test_list_clp = []
test_list_chp = []
test_list_dp = []
test_list_dc = []
test_list_bp = []
test_list_bc = []
test_list_tp = []
test_list_drkC = []
test_list_fw = []
test_list_nonl = []
# treat qe specially since all filters are in the same list
test_list_qe_u = []
test_list_qe_g = []
test_list_qe_r = []
test_list_qe_i = []
test_list_qe_z = []
test_list_qe_y = []
for ccd in res:
test_list.extend(res[ccd])
test_list_ptc.extend(res_ptc_gain[ccd])
test_list_psf.extend(res_psf[ccd])
test_list_rn.extend(res_rn[ccd])
test_list_cls.extend(res_cls[ccd])
test_list_chs.extend(res_chs[ccd])
test_list_clp.extend(res_clp[ccd])
test_list_chp.extend(res_chp[ccd])
test_list_bp.extend(res_bp[ccd])
test_list_bc.extend(res_bc[ccd])
test_list_dp.extend(res_dp[ccd])
test_list_dc.extend(res_dc[ccd])
test_list_tp.extend(res_tp[ccd])
test_list_drkC.extend(res_drkC[ccd])
test_list_fw.extend(res_fw[ccd])
test_list_nonl.extend(res_nonl[ccd])
test_list_qe_u.append(res_qe[ccd][0])
test_list_qe_g.append(res_qe[ccd][1])
test_list_qe_r.append(res_qe[ccd][2])
test_list_qe_i.append(res_qe[ccd][3])
test_list_qe_z.append(res_qe[ccd][4])
test_list_qe_y.append(res_qe[ccd][5])
# NEW: create a column data source for the plots to share
source = ColumnDataSource(data=dict(x=range(0, len(test_list)),
gain=test_list,
ptc=test_list_ptc,
psf=test_list_psf,
rn=test_list_rn,
cls=test_list_cls,
chs=test_list_chs,
clp=test_list_clp,
chp=test_list_chp,
bp=test_list_bp,
bc=test_list_bc,
dp=test_list_dp,
dc=test_list_dc,
tp=test_list_tp,
drkC=test_list_drkC,
fw=test_list_fw,
nonl=test_list_nonl))
source_qe = ColumnDataSource(data=dict(x=range(0, len(test_list_qe_u)),
u=test_list_qe_u,
g=test_list_qe_g,
r=test_list_qe_r,
i=test_list_qe_i,
z=test_list_qe_z,
y=test_list_qe_y))
TOOLS = "pan,wheel_zoom,box_zoom,reset,save,box_select,lasso_select"
# create a new plot with a title and axis labels
p = figure(tools=TOOLS,
title="gains",
x_axis_label='amp',
y_axis_label='gain')
ptc = figure(tools=TOOLS,
title="ptc gains",
x_axis_label='amp',
y_axis_label='gain')
psf = figure(tools=TOOLS,
title="psf",
x_axis_label='amp',
y_axis_label='psf')
rn = figure(tools=TOOLS,
title="Read noise",
x_axis_label='amp',
y_axis_label='Noise')
cls = figure(tools=TOOLS,
title="CTI low serial",
x_axis_label='amp',
y_axis_label='CTI')
chs = figure(tools=TOOLS,
title="CTI high serial",
x_axis_label='amp',
y_axis_label='CTI')
clp = figure(tools=TOOLS,
title="CTI low parallel",
x_axis_label='amp',
y_axis_label='CTI')
chp = figure(tools=TOOLS,
title="CTI high parallel",
x_axis_label='amp',
y_axis_label='CTI')
bp = figure(tools=TOOLS,
title="Bright Pixels",
x_axis_label='amp',
y_axis_label='Bright Pixels')
bc = figure(tools=TOOLS,
title="Bright Columns",
x_axis_label='amp',
y_axis_label='Bright Columns')
dp = figure(tools=TOOLS,
title="Dark Pixels",
x_axis_label='amp',
y_axis_label='Dark Pixels')
dc = figure(tools=TOOLS,
title="Dark Columns",
x_axis_label='amp',
y_axis_label='Dark Columns')
tp = figure(tools=TOOLS,
title="Traps",
x_axis_label='amp',
y_axis_label='Traps')
drkC = figure(tools=TOOLS,
title="Dark Current",
x_axis_label='amp',
y_axis_label='Current')
fw = figure(tools=TOOLS,
title="Full Well",
x_axis_label='amp',
y_axis_label='Full Well')
nonl = figure(tools=TOOLS,
title="Non-linearity",
x_axis_label='amp',
y_axis_label='Max dev')
qe_u = figure(tools=TOOLS,
title="QE: u band",
x_axis_label='sensor',
y_axis_label='QE')
qe_g = figure(tools=TOOLS,
title="QE: g band",
x_axis_label='sensor',
y_axis_label='QE')
qe_r = figure(tools=TOOLS,
title="QE: r band",
x_axis_label='sensor',
y_axis_label='QE')
qe_i = figure(tools=TOOLS,
title="QE: i band",
x_axis_label='sensor',
y_axis_label='QE')
qe_z = figure(tools=TOOLS,
title="QE: z band",
x_axis_label='sensor',
y_axis_label='QE')
qe_y = figure(tools=TOOLS,
title="QE: y band",
x_axis_label='sensor',
y_axis_label='QE')
# add a line renderer with legend and line thickness
#sensor_lines = [sensor_start, sensor_end, sensor_third]
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
p.circle('x',
'gain',
source=source,
legend="Gain: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
p.add_layout(sensor_line)
my_label = Label(x=0, y=10, text='S00')
p.add_layout(my_label)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
ptc.circle('x',
'ptc',
source=source,
legend="ptc Gain: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
ptc.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
psf.circle('x',
'psf',
source=source,
legend="PSF: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
psf.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
rn.circle('x',
'rn',
source=source,
legend="Read Noise: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
rn.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
cls.circle('x',
'cls',
source=source,
legend="CTI low serial: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
cls.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
chs.circle('x',
'chs',
source=source,
legend="CTI high serial: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
chs.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
clp.circle('x',
'clp',
source=source,
legend="CTI low parallel: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
clp.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
chp.circle('x',
'chp',
source=source,
legend="CTI high parallel: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
chp.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
bp.circle('x',
'bp',
source=source,
legend="Bright Pixels: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
bp.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
bc.circle('x',
'bc',
source=source,
legend="Bright Columns: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
bc.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
dp.circle('x',
'dp',
source=source,
legend="Dark Pixels: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
dp.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
dc.circle('x',
'dc',
source=source,
legend="Dark Columns: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
dc.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
tp.circle('x',
'tp',
source=source,
legend="Traps: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
tp.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
drkC.circle('x',
'drkC',
source=source,
legend="Dark Current: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
drkC.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
fw.circle('x',
'fw',
source=source,
legend="Full Well: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
fw.add_layout(sensor_line)
sensor_lines = []
for i in range(0, 160, 16):
sensor_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
nonl.circle('x',
'nonl',
source=source,
legend="Non-linearity: Run " + str(run),
line_width=2)
for sensor_line in sensor_lines:
nonl.add_layout(sensor_line)
raft_lines = []
for i in range(0, 9, 1):
raft_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
qe_u.circle('x',
'u',
source=source_qe,
legend="QE u band: Run " + str(run),
line_width=2)
for raft_line in raft_lines:
qe_u.add_layout(raft_line)
raft_lines = []
for i in range(0, 9, 1):
raft_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
qe_g.circle('x',
'g',
source=source_qe,
legend="QE g band: Run " + str(run),
line_width=2)
for raft_line in raft_lines:
qe_g.add_layout(raft_line)
raft_lines = []
for i in range(0, 9, 1):
raft_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
qe_r.circle('x',
'r',
source=source_qe,
legend="QE r band: Run " + str(run),
line_width=2)
for raft_line in raft_lines:
qe_r.add_layout(raft_line)
raft_lines = []
for i in range(0, 9, 1):
raft_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
qe_i.circle('x',
'i',
source=source_qe,
legend="QE i band: Run " + str(run),
line_width=2)
for raft_line in raft_lines:
qe_i.add_layout(raft_line)
raft_lines = []
for i in range(0, 9, 1):
raft_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
qe_z.circle('x',
'z',
source=source_qe,
legend="QE z band: Run " + str(run),
line_width=2)
for raft_line in raft_lines:
qe_z.add_layout(raft_line)
raft_lines = []
for i in range(0, 9, 1):
raft_lines.append(
Span(location=i,
dimension='height',
line_color='grey',
line_dash='dashed',
line_width=3))
qe_y.circle('x',
'y',
source=source_qe,
legend="QE y band: Run " + str(run),
line_width=2)
for raft_line in raft_lines:
qe_y.add_layout(raft_line)
# NEW: put the subplots in a gridplot
grid = gridplot([[p, ptc], [psf, rn]])
grid_cti = gridplot([[cls, chs], [clp, chp]])
grid_def = gridplot([[bp, bc, None], [dp, dc, tp]])
grid_qe = gridplot([[qe_u, qe_g, qe_r], [qe_i, qe_z, qe_y]])
grid_misc = gridplot([[fw, drkC], [nonl, None]])
raft_text = "Plots for I&T Raft run " + str(run) + " " + raft
pre = PreText(text=raft_text, width=1000, height=100)
tab1 = Panel(child=grid, title="Gains, PSF, Noise")
tab2 = Panel(child=grid_cti, title="CTI")
tab3 = Panel(child=grid_def, title="Defects")
tab4 = Panel(child=grid_qe, title="QE 6-band")
tab5 = Panel(child=grid_misc, title="Dark C, Full well, Non-lin")
tabs = Tabs(tabs=[tab1, tab2, tab3, tab4, tab5])
l = layout([widgetbox(pre), tabs], sizing_mode='scale_both')
o_file = self.base_dir + raft + "-" + str(run) + ".html"
output_file(o_file)
save(l)
def write_table(self, run_list=None, raft_list=None, type_list=None):
data = dict(rafts=raft_list, runs=run_list, types=type_list)
dashboard = ColumnDataSource(data)
columns = [
TableColumn(field="rafts", title="Raft"),
TableColumn(field="types", title="Type"),
TableColumn(field="runs",
title="Run",
formatter=HTMLTemplateFormatter(template="<a href= \
'http://slac.stanford.edu/~richard/LSST/<%= rafts %>-<%= runs %>.html' \
><%= runs %></a>"))
]
data_table = DataTable(source=dashboard,
columns=columns,
width=400,
height=280)
return data_table
help="separate app instance, dash should not be prepended ")
args = parser.parse_args()
print 'Discover step and schema names for run ', args.run
if args.run == None:
print 'Error: missing run number'
raise ValueError
if args.eTserver == 'Prod': pS = True
else: pS = False
if args.appSuffix != '':
appSuffix = '-' + args.appSuffix
print 'args. appSuffix, appSuffix = ', args.appSuffix, appSuffix
connect = Connection(operator='richard',
db=args.db,
exp='LSST-CAMERA',
prodServer=pS,
appSuffix=appSuffix)
returnData = connect.getRunResults(run=args.run)
for step in returnData['steps']:
stepDict = returnData['steps'][step]
print '\n Step ', step, '\n'
for schemaList in stepDict:
print schemaList