def __init__(self, xaxis, yaxis, start, end, camera):
self.yaxis = yaxis
self.xaxis = xaxis
self.start = datetime.strptime(start, '%Y%m%d').strftime('%Y-%m-%d')
self.end = datetime.strptime(end, '%Y%m%d').strftime('%Y-%m-%d')
self.camera = camera
self.models = QLFModels()
def __init__(self, process_id, cam, exp, night):
self.cam = cam
self.exp = exp
self.night = night
self.process_id = process_id
# This is True if the pipeline didn't generate some yaml file
self.error = dict(zip(self.qa_name, ['False'] * len(self.qa_name)))
logger.info(
'check *rms_over *bias *SUMCOUNT_RMS shouldbe SUMCOUNT_MED_SKY' +
'Resigf skyresid- residrms')
# QA tests and keys to respective values
self.metric_qa_list = [
'getbias', 'getrms', 'skycont', 'countbins', 'countpix', 'snr',
'skyresid', 'skypeak', 'integ', 'xsigma', 'wsigma'
] # THIS LINE : TB CHECKED
self.metric_key_list = [
'BIAS', 'RMS_OVER', 'SKYCONT', 'NGOODFIBERS', 'NPIX_LOW',
'ELG_FIDMAG_SNR', 'RESID_RMS', 'SUMCOUNT_MED_SKY', 'MAGDIFF_AVG',
'XSHIFT', 'WSHIFT'
]
self.metric_dict = dict(zip(self.metric_qa_list, self.metric_key_list))
self.models = QLFModels()
try: # ff
self.metrics, self.tests = self.Load_metrics_n_tests()
except Exception as e: # ff
print(e)
sys.exit("Could not load metrics and tests")
class QLFProcess(object):
""" Class responsible for managing Quick Look pipeline process. """
def __init__(self, data):
self.pipeline_name = 'Quick Look'
self.data = data
self.models = QLFModels()
def start_process(self):
""" Start pipeline. """
logger.info('Started %s ...' % self.pipeline_name)
logger.info('Night: %s' % self.data.get('night'))
logger.info('Exposure: %s' % str(self.data.get('expid')))
self.data['start'] = datetime.datetime.now().replace(microsecond=0)
# create process in database and obtain the process id
process = self.models.insert_process(self.data, self.pipeline_name)
self.data['process_id'] = process.id
self.data['status'] = process.status
# TODO: ingest configuration file used, this should be done by process
# self.models.insert_config(process.id)
logger.info('Process ID: %i' % process.id)
logger.info('Starting...')
output_dir = os.path.join('exposures', self.data.get('night'),
self.data.get('zfill'))
output_full_dir = os.path.join(self.data.get('desi_spectro_redux'),
output_dir)
# Make sure output dir is created
if not os.path.isdir(output_full_dir):
os.makedirs(output_full_dir)
logger.info('Output dir: %s' % output_dir)
self.data['output_dir'] = output_dir
def finish_process(self):
""" Finish pipeline. """
self.data['end'] = datetime.datetime.now().replace(microsecond=0)
self.data['duration'] = str(
self.data.get('end') - self.data.get('start'))
logger.info("Process with expID {} completed in {}.".format(
self.data.get('expid'), self.data.get('duration')))
self.models.update_process(process_id=self.data.get('process_id'),
end=self.data.get('end'),
process_dir=self.data.get('output_dir'),
status=self.data.get('status'))
def load_qa(self):
cam = self.selected_arm + str(self.selected_spectrograph)
mergedqa = QLFModels().get_output(self.selected_process_id, cam)
check_ccds = mergedqa['TASKS']['CHECK_CCDs']
getrms = check_ccds['METRICS']
nrg = check_ccds['PARAMS']['NOISE_AMP_NORMAL_RANGE']
wrg = check_ccds['PARAMS']['NOISE_AMP_WARN_RANGE']
if mergedqa['FLAVOR'].upper() == 'SCIENCE':
program = mergedqa['GENERAL_INFO']['PROGRAM'].upper()
program_prefix = '_' + program
else:
program_prefix = ''
refexp = mergedqa['TASKS']['CHECK_CCDs']['PARAMS']['NOISE_AMP' +
program_prefix +
'_REF']
# amp 1
p = Patch().plot_amp(dz=getrms["NOISE_AMP"],
refexp=refexp,
name="NOISE_AMP",
description="NOISE per Amp (photon counts)",
wrg=wrg)
# amp 2
p2 = Patch().plot_amp(
dz=getrms["NOISE_OVERSCAN_AMP"],
refexp=refexp,
name="NOISE_OVERSCAN_AMP",
description="NOISE Overscan per Amp (photon counts)",
wrg=wrg)
info_col = Title().write_description('getrms')
# Prepare tables
current_exposures = check_ccds['METRICS']['NOISE_AMP']
gen_info = mergedqa['GENERAL_INFO']
flavor = mergedqa["FLAVOR"]
if flavor == 'science':
program = gen_info['PROGRAM'].upper()
reference_exposures = check_ccds['PARAMS']['LITFRAC_AMP_' +
program + '_REF']
else:
reference_exposures = check_ccds['PARAMS']['LITFRAC_AMP_REF']
keynames = ["NOISE_AMP" for i in range(len(current_exposures))]
metric = Table().reference_table(keynames, current_exposures,
reference_exposures)
alert = Table().alert_table(nrg, wrg)
layout = column(info_col,
Div(),
metric,
alert,
column(p, sizing_mode='scale_both'),
column(p2, sizing_mode='scale_both'),
css_classes=["display-grid"])
return file_html(layout, CDN, "GETRMS")
def load_qa(self):
cam = self.selected_arm+str(self.selected_spectrograph)
mergedqa = QLFModels().get_output(self.selected_process_id, cam)
check_ccds = mergedqa['TASKS']['CHECK_CCDs']
getbias = check_ccds['METRICS']
nrg = check_ccds['PARAMS']['BIAS_AMP_NORMAL_RANGE']
wrg = check_ccds['PARAMS']['BIAS_AMP_WARN_RANGE']
if mergedqa['FLAVOR'].upper() == 'SCIENCE':
program = mergedqa['GENERAL_INFO']['PROGRAM'].upper()
program_prefix = '_'+program
else:
program_prefix = ''
refexp = mergedqa['TASKS']['CHECK_CCDs']['PARAMS']['BIAS_AMP' +
program_prefix+'_REF']
# PATCH
p = Patch().plot_amp(
dz=getbias["BIAS_AMP"],
refexp=refexp,
name="BIAS_AMP",
description="Average bias value (photon counts)",
wrg=wrg
)
p_status = Patch().plot_amp(
dz=getbias["BIAS_AMP"],
refexp=refexp,
name="BIAS_AMP (STATUS)",
description="Average bias value (photon counts)",
wrg=wrg,
nrg=nrg,
status_plot=True,
)
# INFO TABLES:
info_col = Title().write_description('getbias')
# Prepare tables
current_exposures = check_ccds['METRICS']['BIAS_AMP']
gen_info = mergedqa['GENERAL_INFO']
flavor = mergedqa["FLAVOR"]
if flavor == 'science':
program = gen_info['PROGRAM'].upper()
reference_exposures = check_ccds['PARAMS']['BIAS_AMP_' +
program + '_REF']
else:
reference_exposures = check_ccds['PARAMS']['BIAS_AMP_REF']
keynames = ["BIAS_AMP" for i in range(len(current_exposures))]
table = Table().single_table(keynames, current_exposures, reference_exposures, nrg, wrg)
layout = column(info_col, Div(),
table, Div(),
column(p, sizing_mode='scale_both'), #css_classes=["main-one"]),
column(p_status, sizing_mode='scale_both'),
css_classes=["display-grid"])
return file_html(layout, CDN, "GETBIAS")
def load_qa(self):
cam = self.selected_arm + str(self.selected_spectrograph)
mergedqa = QLFModels().get_output(self.selected_process_id, cam)
check_flat = mergedqa['TASKS']['CHECK_FIBERFLAT']
flat = check_flat['METRICS']
nrg = check_flat['PARAMS']['CHECKFLAT_NORMAL_RANGE']
wrg = check_flat['PARAMS']['CHECKFLAT_WARN_RANGE']
info_col = Title().write_description('fiberflat')
# Prepare tables
current_exposures = check_flat['METRICS']['CHECKFLAT']
reference_exposures = check_flat['PARAMS']['CHECKFLAT_REF']
keynames = ["CHECKFLAT"]
table = Table().single_table(keynames, [current_exposures],
reference_exposures, nrg, wrg)
layout = column(info_col,
Div(),
table,
Div(),
css_classes=["display-grid"])
return file_html(layout, CDN, "FIBERFLAT")
def load_qa(self):
cam = self.selected_arm + str(self.selected_spectrograph)
mergedqa = QLFModels().get_output(self.selected_process_id, cam)
check_arc = mergedqa['TASKS']['CHECK_ARC']
nrg = check_arc['PARAMS']['CHECKARC_NORMAL_RANGE']
wrg = check_arc['PARAMS']['CHECKARC_WARN_RANGE']
info_col = Title().write_description('arc')
# Prepare tables
current_exposures = check_arc['METRICS']['CHECKARC']
reference_exposures = check_arc['PARAMS']['CHECKARC_REF']
keynames = ["CHECKARC"]
table = Table().single_table(keynames, [current_exposures],
reference_exposures, list(map(int, nrg)),
list(map(int, wrg)))
layout = column(info_col,
Div(),
table,
Div(),
css_classes=["display-grid"])
return file_html(layout, CDN, "ARC")
def load_qa(self):
cam = self.selected_arm+str(self.selected_spectrograph)
mergedqa = QLFModels().get_output(self.selected_process_id, cam)
check_fibers = mergedqa['TASKS']['CHECK_FIBERS']
gen_info = mergedqa['GENERAL_INFO']
flavor = mergedqa['FLAVOR']
nrg = check_fibers['PARAMS']['XYSHIFTS_NORMAL_RANGE']
wrg = check_fibers['PARAMS']['XYSHIFTS_WARN_RANGE']
info_col = Title().write_description('xyshifts')
# Prepare tables
current_exposures = check_fibers['METRICS']['XYSHIFTS']
program = gen_info['PROGRAM'].upper()
if flavor == 'science':
reference_exposures = check_fibers['PARAMS']['XYSHIFTS_' +
program + '_REF']
else:
reference_exposures = []
keynames = ["X", "Y"]
metric = Table().reference_table(keynames, current_exposures, reference_exposures)
alert = Table().alert_table(nrg, wrg)
layout = column(info_col, Div(),
metric, alert,
css_classes=["display-grid"])
return file_html(layout, CDN, "XYSHIFTS")
def load_qa(self):
cam = self.selected_arm+str(self.selected_spectrograph)
mergedqa = QLFModels().get_output(self.selected_process_id, cam)
gen_info = mergedqa['GENERAL_INFO']
check_spectra = mergedqa['TASKS']['CHECK_SPECTRA']
nrg = check_spectra['PARAMS']['SKYRBAND_NORMAL_RANGE']
wrg = check_spectra['PARAMS']['SKYRBAND_WARN_RANGE']
info_col = Title().write_description('skyR')
# Prepare Tables
current_exposures = [check_spectra['METRICS']['SKYRBAND']]
program = gen_info['PROGRAM'].upper()
reference_exposures = check_spectra['PARAMS']['SKYRBAND_' +
program + '_REF']
keynames = ["SKYRBAND" for i in range(len(current_exposures))]
table = Table().single_table(keynames, current_exposures, reference_exposures, nrg, wrg)
layout = column(info_col, Div(),
table, Div(),
css_classes=["display-grid"])
return file_html(layout, CDN, "SKYR")
def data_source(self, fmap):
""" Creating data source for plots
"""
data_model = {
'goodfiber': [],
'status': [],
'color': [],
'cam': [],
'OBJ_TYPE': [],
'ra': [],
'dec': [],
}
process_id = self.selected_process_id
joblist = [
entry.camera.camera
for entry in Job.objects.filter(process_id=process_id)
]
ra_tile = fmap.fiber_ra
dec_tile = fmap.fiber_dec
otype_tile = fmap.objtype
y = []
color = []
status = []
cam_inst = []
for spec in list(range(10)):
cam = self.selected_arm + str(spec)
if cam in joblist:
mergedqa = QLFModels().get_output(self.selected_process_id,
cam)
countbins = mergedqa['TASKS']['CHECK_FIBERS']['METRICS'][
'GOOD_FIBERS']
y = y + countbins
color = color + [
'green' if idx == 1 else 'red' for idx in countbins
]
status = status + [
'GOOD' if idx == 1 else 'BAD' for idx in countbins
]
else:
y = y + 500 * ['']
color = color + ['lightgray'] * 500
status = status + [''] * 500
cam_inst = cam_inst + [cam] * 500
data_model['goodfiber'] = y
data_model['color'] = color
data_model['status'] = status
data_model['cam'] = cam_inst
data_model['OBJ_TYPE'] = otype_tile
data_model['ra'] = ra_tile
data_model['dec'] = dec_tile
source = ColumnDataSource(data=data_model)
return source
def __init__(self, data):
self.pipeline_name = 'Quick Look'
self.data = data
self.models = QLFModels()
output_dir = os.path.join('exposures', self.data.get('night'),
self.data.get('zfill'))
output_full_dir = os.path.join(desi_spectro_redux, output_dir)
# Remove old dir
if os.path.isdir(output_full_dir):
shutil.rmtree(output_full_dir)
# Make output dir
os.makedirs(output_full_dir)
self.data['output_dir'] = output_dir
def qa_tests(self, process_id):
qa_tests = list()
for job in QLFModels().get_jobs_by_process_id(process_id):
try:
process = job.process
exposure = process.exposure
lm = LoadMetrics(process_id, job.camera_id,
process.exposure_id, exposure.night)
qa_tests.append({job.camera_id: lm.load_qa_tests()})
except:
mainlogger.error('qa_tests error camera %s' % job.camera)
return qa_tests
def load_scalar_metrics(self, process_id, cam):
scalar_metrics = dict()
try:
process = QLFModels().get_process_by_process_id(process_id)
exposure = process.exposure
lm = LoadMetrics(process_id, cam, process.exposure_id,
exposure.night)
scalar_metrics['metrics'] = lm.metrics
scalar_metrics['tests'] = lm.tests
except:
print('load_scalar_metrics error')
return scalar_metrics
class QLFProcess(object):
""" Class responsible for managing Quick Look pipeline process. """
def __init__(self, data):
self.pipeline_name = 'Quick Look'
self.data = data
self.models = QLFModels()
output_dir = os.path.join('exposures', self.data.get('night'),
self.data.get('zfill'))
output_full_dir = os.path.join(desi_spectro_redux, output_dir)
# Remove old dir
if os.path.isdir(output_full_dir):
shutil.rmtree(output_full_dir)
# Make output dir
os.makedirs(output_full_dir)
self.data['output_dir'] = output_dir
def start_process(self):
""" Start pipeline. """
self.data['start'] = datetime.datetime.now().replace(microsecond=0)
# create process in database and obtain the process id
process = self.models.insert_process(self.data, self.pipeline_name)
self.data['process_id'] = process.id
self.data['status'] = process.status
# TODO: ingest configuration file used, this should be done by process
# self.models.insert_config(process.id)
pipe_logger.info('...{}'.format('\n' * 20))
pipe_logger.info('Process ID {}'.format(process.id))
pipe_logger.info('ExpID {} started.'.format(self.data.get('expid')))
return process.id
def finish_process(self):
""" Finish pipeline. """
self.data['end'] = datetime.datetime.now().replace(microsecond=0)
self.data['duration'] = self.data.get('end') - self.data.get('start')
pipe_logger.info("ExpID {} ended (runtime: {}).".format(
self.data.get('expid'), str(self.data.get('duration'))))
proc = Thread(target=self.ingest_parallel_qas)
proc.start()
def __init__(self):
super().__init__()
self.running = Event()
self.exit = Event()
self.dos_monitor = DOSmonitor()
self.last_night = str()
self.current_exposure = None
exposure = QLFModels().get_last_exposure()
if exposure:
self.last_night = exposure.night
def __init__(self, process_id, cam, exp=None, night=None, flavor=None):
self.cam = cam
self.exp = exp
self.night = night
self.process_id = process_id
# This is True if the pipeline didn't generate some json file
self.error = dict(zip(self.qa_name, ['False']*len(self.qa_name)))
self.models = QLFModels()
stages = self.load_flavors()
self.steps_dic = dict()
self.qas_status = dict()
self.alert_keys = dict()
for step in self.flavor_stages[flavor]['step_list']:
self.steps_dic[step['name']] = list()
self.qas_status[step['name']] = list()
for qa in step['qa_list']:
self.steps_dic[step['name']].append(qa['name'])
self.qas_status[step['name']].append('None')
self.alert_keys[qa['name']] = qa['status_key']
def stop(self):
if self.is_running():
QLFModels().abort_current_process()
if self.monitor and self.monitor.is_alive():
logger.debug("Stop pid %i" % self.monitor.pid)
pid = self.monitor.pid
self.monitor.shutdown()
kill_proc_tree(pid, include_parent=False)
del self.monitor
gc.collect
self.monitor = None
else:
logger.debug("Monitor is not initialized.")
def data_source(self, fmap):
""" Creating data source for plots
"""
data_model = {
'x': [],
'w': [],
'cam': [],
'OBJ_TYPE': [],
'ra': [],
'dec': [],
}
process_id = self.selected_process_id
joblist = [
entry.camera.camera
for entry in Job.objects.filter(process_id=process_id)
]
ra_tile = fmap.fiber_ra
dec_tile = fmap.fiber_dec
otype_tile = fmap.objtype
y = []
w = []
cam_inst = []
for spec in list(range(10)):
cam = self.selected_arm + str(spec)
if cam in joblist:
mergedqa = QLFModels().get_output(self.selected_process_id,
cam)
xwsig = mergedqa['TASKS']['CHECK_CCDs']['METRICS'][
'XWSIGMA_FIB']
y = y + xwsig[0]
w = w + xwsig[1]
else:
y = y + 500 * [np.nan]
w = w + 500 * [np.nan]
cam_inst = cam_inst + [cam] * 500
data_model['x'] = y
data_model['w'] = w
data_model['cam'] = cam_inst
data_model['OBJ_TYPE'] = otype_tile
data_model['ra'] = ra_tile
data_model['dec'] = dec_tile
source = ColumnDataSource(data=data_model)
return source
def load_qa(self):
cam = self.selected_arm + str(self.selected_spectrograph)
mergedqa = QLFModels().get_output(self.selected_process_id, cam)
gen_info = mergedqa['GENERAL_INFO']
check_spectra = mergedqa['TASKS']['CHECK_SPECTRA']
std_fiberid = mergedqa['GENERAL_INFO']['STAR_FIBERID']
nrg = check_spectra['PARAMS']['DELTAMAG_TGT_NORMAL_RANGE']
wrg = check_spectra['PARAMS']['DELTAMAG_TGT_WARN_RANGE']
fiber_mag = np.array(mergedqa['GENERAL_INFO']['FIBER_MAGS'])
if 'b' in cam:
arm_id = 0
elif 'r' in cam:
arm_id = 1
else:
arm_id = 2
fiber_mag = fiber_mag[arm_id] #.flatten()
tooltip = """
<div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">INTEG: </span>
<span style="font-size: 1vw; color: #515151;">@integ</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">FIBER ID: </span>
<span style="font-size: 1vw; color: #515151;">@x</span>
</div>
</div>
"""
source = ColumnDataSource(
data={
'integ': [i if i > -999 else np.nan for i in fiber_mag],
'x': np.arange(len(fiber_mag)),
})
print(len(fiber_mag))
yrange = [0, 1.1 * max(fiber_mag)]
fiber_hist = Plot2d(
yrange,
x_label="Fibers",
y_label="Integral (counts)",
tooltip=tooltip,
title="",
width=600,
height=400,
yscale="auto",
hover_mode="vline",
).vbar(
source,
y="integ",
)
info_col = Title().write_description('integ')
# Reading obj_type
objlist = mergedqa["TASKS"]["CHECK_SPECTRA"]["METRICS"]["OBJLIST"]
if 'SKY' in objlist:
objlist.remove('SKY')
# Prepare tables
current_exposures = check_spectra['METRICS']['DELTAMAG_TGT']
program = gen_info['PROGRAM'].upper()
reference_exposures = check_spectra['PARAMS']['DELTAMAG_TGT_' +
program + '_REF']
keynames = ["DELTAMAG_TGT" + " ({})".format(i) for i in objlist]
table = Table().single_table(keynames, current_exposures,
reference_exposures, nrg, wrg)
layout = column(info_col,
Div(),
table,
Div(),
column(fiber_hist,
sizing_mode='scale_both',
css_classes=["main-one"]),
css_classes=["display-grid"])
return file_html(layout, CDN, "INTEG")
def load_qa(self):
cam = self.selected_arm+str(self.selected_spectrograph)
mergedqa = QLFModels().get_output(self.selected_process_id, cam)
check_spectra = mergedqa['TASKS']['CHECK_SPECTRA']
gen_info = mergedqa['GENERAL_INFO']
ra = gen_info['RA']
dec = gen_info['DEC']
nrg = check_spectra['PARAMS']['PEAKCOUNT_NORMAL_RANGE']
wrg = check_spectra['PARAMS']['PEAKCOUNT_WARN_RANGE']
current_exposures = [check_spectra['METRICS']['PEAKCOUNT']]
program = gen_info['PROGRAM'].upper()
reference_exposures = check_spectra['PARAMS']['PEAKCOUNT_' +
program + '_REF']
obj_type = sort_obj(gen_info)
my_palette = get_palette("RdYlBu_r")
peak_tooltip = """
<div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">PEAKCOUNT: </span>
<span style="font-size: 1vw; color: #515151">@peakcount_fib</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">RA: </span>
<span style="font-size: 1vw; color: #515151;">@x1</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">DEC: </span>
<span style="font-size: 1vw; color: #515151;">@y1</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">Obj Type: </span>
<span style="font-size: 1vw; color: #515151;">@OBJ_TYPE</span>
</div>
</div>
"""
url = "http://legacysurvey.org/viewer?ra=@ra&dec=@dec&zoom=16&layer=decals-dr5"
qlf_fiberid = np.arange(0, 500)
peak_hover = HoverTool(tooltips=peak_tooltip)
peakcount_fib = check_spectra['METRICS']['PEAKCOUNT_FIB']
source = ColumnDataSource(data={
'x1': ra,
'y1': dec,
'peakcount_fib': peakcount_fib,
'delta_peakcount_fib': np.array(peakcount_fib)-reference_exposures,
'QLF_FIBERID': qlf_fiberid,
'OBJ_TYPE': obj_type,
})
low, high = wrg
mapper = LinearColorMapper(palette=my_palette,
low=low, #0.98*np.min(peakcount_fib),
high=high, #1.02*np.max(peakcount_fib))
nan_color='darkgrey')
radius = 0.0165
radius_hover = 0.02
# centralize wedges in plots:
ra_center=0.5*(max(ra)+min(ra))
dec_center=0.5*(max(dec)+min(dec))
xrange_wedge = Range1d(start=ra_center + .95, end=ra_center-.95)
yrange_wedge = Range1d(start=dec_center+.82, end=dec_center-.82)
# axes limit
xmin, xmax = [min(gen_info['RA'][:]), max(gen_info['RA'][:])]
ymin, ymax = [min(gen_info['DEC'][:]), max(gen_info['DEC'][:])]
xfac, yfac = [(xmax-xmin)*0.06, (ymax-ymin)*0.06]
left, right = xmin - xfac, xmax+xfac
bottom, top = ymin-yfac, ymax+yfac
wedge_plot = Plot2d(
x_range=xrange_wedge,
y_range=yrange_wedge,
x_label="RA",
y_label="DEC",
tooltip=peak_tooltip,
title="PEAKCOUNT",
width=500,
height=380,
).wedge(
source,
x='x1',
y='y1',
field='delta_peakcount_fib',
mapper=mapper,
).plot
info_col = Title().write_description('skypeak')
# ================================
# histogram
hist_tooltip = """
<div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">Frequency: </span>
<span style="font-size: 1vw; color: #515151">@hist</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">Peakcount: </span>
<span style="font-size: 1vw; color: #515151;">[@left, @right]</span>
</div>
</div>
"""
hist, edges = np.histogram(peakcount_fib, bins="sqrt")
source_hist = ColumnDataSource(data={
'hist': hist,
'histplusone': hist+1,
'bottom': [0] * len(hist),
'bottomplusone': [1]*len(hist),
'left': edges[:-1],
'right': edges[1:]
})
p_hist = Plot2d(
y_range=(1, 11**(int(np.log10(max(hist)))+1)),
x_label='PEAKCOUNT',
y_label='Frequency + 1',
tooltip=hist_tooltip,
title="",
width=550,
height=300,
yscale="log",
hover_mode="vline",
).quad(
source_hist,
top='histplusone',
bottom='bottomplusone',
line_width=1,
)
# Prepare tables
keynames = ["PEAKCOUNT" for i in range(len(current_exposures))]
metric = Table().reference_table(keynames, current_exposures, reference_exposures)
alert = Table().alert_table(nrg, wrg)
layout = column(info_col, Div(),
metric, alert,
column(wedge_plot, sizing_mode='scale_both'),
column(p_hist, sizing_mode='scale_both'),
css_classes=["display-grid"])
return file_html(layout, CDN, "SKYPEAK")
def wedge_plot(self, wedge_arm, fmap, common_source=None, sigma_kind='x'):
ra_center = fmap.exposure.telra
dec_center = fmap.exposure.teldec
fiber_tooltip = """
<div>
<div>
<span style="font-size: 12px; font-weight: bold; color: #303030;">SIGMA: </span>
<span style="font-size: 13px; color: #515151">@y</span>
</div>
<div>
<span style="font-size: 12px; font-weight: bold; color: #303030;">RA: </span>
<span style="font-size: 13px; color: #515151;">@ra</span>
</div>
<div>
<span style="font-size: 12px; font-weight: bold; color: #303030;">DEC: </span>
<span style="font-size: 13px; color: #515151;">@dec</span>
</div>
<div>
<span style="font-size: 12px; font-weight: bold; color: #303030;">Obj Type: </span>
<span style="font-size: 13px; color: #515151;">@OBJ_TYPE</span>
</div>
<div>
<span style="font-size: 12px; font-weight: bold; color: #303030;">CAM: </span>
<span style="font-size: 13px; color: #515151;">@cam</span>
</div>
"""
fiber_tooltip = fiber_tooltip.replace('SIGMA:',
'%sSIGMA:' % sigma_kind.upper())
hover = HoverTool(tooltips=fiber_tooltip)
my_palette = get_palette("bwr")
source = common_source
process_id = self.selected_process_id
joblist = [
entry.camera.camera
for entry in Job.objects.filter(process_id=process_id)
]
if len(joblist) > 0:
cam = joblist[0]
mergedqa = QLFModels().get_output(self.selected_process_id, cam)
warn_range = mergedqa['TASKS']['CHECK_CCDs']['PARAMS'][
'XWSIGMA_WARN_RANGE']
arg_kind = {'x': 0, 'w': 1}
refvalue = mergedqa['TASKS']['CHECK_CCDs']['PARAMS'][
'XWSIGMA_REF'][arg_kind[sigma_kind]]
rng_warn_min, rng_warn_max = warn_range[0] + \
refvalue, warn_range[1] + refvalue
sigma = source.data['{}'.format(sigma_kind)]
rng_min, rng_max = np.nanmin(sigma), np.nanmax(sigma)
rng = rng_max - rng_min
if np.isnan(rng_min) or np.isnan(rng_max):
fill_color = 'lightgray'
else:
mapper = LinearColorMapper(palette=my_palette,
nan_color='lightgray',
low=rng_warn_min,
high=rng_warn_max)
fill_color = {'field': '%s' % (sigma_kind), 'transform': mapper}
radius = 0.017
radius_hover = 0.018
xrange = Range1d(start=ra_center + 2, end=ra_center - 2)
yrange = Range1d(start=dec_center + 1.8, end=dec_center - 1.8)
p = Figure(title='FOCUS %s (ARM %s)' % (sigma_kind.upper(), wedge_arm),
x_axis_label='RA',
y_axis_label='DEC',
plot_width=600,
plot_height=600,
tools=[
hover,
"box_zoom,pan,wheel_zoom,reset,lasso_select,crosshair"
],
active_drag="box_zoom",
x_range=xrange,
y_range=yrange)
p.title.align = 'center'
p.circle('ra',
'dec',
source=source,
name="data",
radius=radius,
fill_color=fill_color,
line_color='black',
line_width=0.4,
hover_line_color='red')
p.circle('ra',
'dec',
source=source,
name="data",
radius=radius_hover,
hover_fill_color=fill_color,
fill_color=None,
line_color=None,
line_width=3,
hover_line_color='orange')
if 'mapper' in locals():
cbar = Figure(
height=p.plot_height,
width=120,
toolbar_location=None,
min_border=0,
outline_line_color=None,
)
color_bar = ColorBar(color_mapper=mapper,
label_standoff=14,
major_label_text_font_style="bold",
padding=26,
major_label_text_align='right',
major_label_text_font_size="10pt",
location=(0, 0))
cbar.title.align = 'center'
cbar.title.text_font_size = '10pt'
cbar.add_layout(color_bar, 'left')
p_list = [cbar, p]
else:
p_list = [p]
return p_list
def data_source_arm(self, fmap, arm):
""" Creating data source for plots
"""
data_model = {
'resids': [],
'hover': [],
'OBJ_TYPE': [],
'ra': [],
'dec': [],
'cam': [],
}
process_id = self.selected_process_id
process = Process.objects.get(pk=process_id)
joblist = [entry.camera.camera for entry in Job.objects.filter(
process_id=process_id)]
ra_tile = fmap.fiber_ra
dec_tile = fmap.fiber_dec
otype_tile = fmap.objtype
objlist = sorted(set(otype_tile))
if 'SKY' in objlist:
objlist.remove('SKY')
ra_snr = []
dec_snr = []
resids_snr = []
ot_snr = []
cam_snr = []
for spect in list(range(10)):
cam = arm+str(spect)
ra_petal = []
dec_petal = []
resids_petal = []
ot_petal = []
cam_petal = []
if cam in joblist:
mergedqa = QLFModels().get_output(
self.selected_process_id, cam)
med_snr = np.array(
mergedqa['TASKS']['CHECK_SPECTRA']['METRICS']["MEDIAN_SNR"])
# color values:
resids_petal = (
mergedqa['TASKS']['CHECK_SPECTRA']['METRICS']['SNR_RESID'])
ra_petal = []
dec_petal = []
fibers_snr = []
obj = np.arange(len(objlist))
for t in range(len(obj)):
otype = list(objlist)[t]
oid = np.where(np.array(list(objlist)) == otype)[0][0]
if otype == 'STD':
fibers = mergedqa['GENERAL_INFO']['STD_FIBERID']
elif otype == 'TGT':
fibers = mergedqa['GENERAL_INFO']['STAR_FIBERID']
# fibers = mergedqa['GENERAL_INFO']['%s_FIBERID'%otype]
fibers_snr = fibers_snr + fibers
for j in range(len(fibers)):
ras = mergedqa['GENERAL_INFO']['RA'][fibers[j]]
decs = mergedqa['GENERAL_INFO']['DEC'][fibers[j]]
ra_petal.append(ras)
dec_petal.append(decs)
ot_petal.append(otype)
ra_snr = ra_snr+ra_petal
dec_snr = dec_snr+dec_petal
resids_snr = resids_snr + \
[i if i > -999 else np.nan for i in resids_petal]
ot_snr = ot_snr + ot_petal
cam_snr = cam_snr + [cam]*len(ra_petal)
else:
ra_snr = ra_snr + list(ra_tile[500*spect: 500*(spect + 1)])
dec_snr = dec_snr + list(dec_tile[500*spect: 500*(spect + 1)])
resids_snr = resids_snr + 500*[np.nan]
ot_snr = ot_snr + list(otype_tile[500*spect: 500*(spect + 1)])
cam_snr = cam_snr + [str(cam)]*500
data_model['ra'] = ra_snr # ra_tile
data_model['dec'] = dec_snr # dec_tile
data_model['resids'] = resids_snr
data_model['hover'] = ['%4.3f' % (ires) for ires in resids_snr]
data_model['OBJ_TYPE'] = ot_snr
data_model['cam'] = cam_snr
source = ColumnDataSource(data=data_model)
return source
from qlf_models import QLFModels
from migrate_jobs_outputs import migrate_job_outputs
from copy import copy
import collections
import numpy
import datetime
import random
qlf_models = QLFModels()
FLAVORS = {'science': 1.0, 'flat': 0.1, 'arc': 0.1}
METRICS = {
'CHECK_SPECTRA': [
'FIBER_MAG', 'SNR_MAG_TGT', 'SKYCONT_FIBER', 'PEAKCOUNT',
'PEAKCOUNT_FIB', 'DELTAMAG_TGT', 'DELTAMAG'
],
'CHECK_CCDs': [
'BIAS_AMP', 'NOISE_AMP', 'XWSIGMA_FIB', 'NOISE_OVERSCAN_AMP',
'XWSIGMA_AMP', 'XWSIGMA'
]
}
def generate_processes_by_exposure(exposure):
""" Creates a simulated process based on an exposure.
Arguments:
exposure {object} -- Exposure model
Raises:
class LoadMetrics:
""" Read values from the json's files and return an alert (NORMAL, WARN or ALARM)
associated to a given metric. It also attributes a color for a wedge in the interface
CORRECTING WXSIGMA
Functions:
----------
find_qa_check(qa)
update_status(qa)
"""
silent = 'False' # Defining a silent mode
prfx = 'ql-'
qa_name = ['countpix', 'getbias', 'getrms', 'xwsigma',
'countbins', 'integ', 'skycont', 'skypeak', 'snr']
def __init__(self, process_id, cam, exp=None, night=None, flavor=None):
self.cam = cam
self.exp = exp
self.night = night
self.process_id = process_id
# This is True if the pipeline didn't generate some json file
self.error = dict(zip(self.qa_name, ['False']*len(self.qa_name)))
self.models = QLFModels()
stages = self.load_flavors()
self.steps_dic = dict()
self.qas_status = dict()
self.alert_keys = dict()
for step in self.flavor_stages[flavor]['step_list']:
self.steps_dic[step['name']] = list()
self.qas_status[step['name']] = list()
for qa in step['qa_list']:
self.steps_dic[step['name']].append(qa['name'])
self.qas_status[step['name']].append('None')
self.alert_keys[qa['name']] = qa['status_key']
def load_flavors(self):
flavors = ['science', 'arc', 'flat']
self.flavor_stages = dict()
for flavor in flavors:
flavor_path = os.path.join(qlf_root, "framework", "ql_mapping", "{}.json".format(flavor))
try:
stages_json = open(flavor_path).read()
self.flavor_stages[flavor] = json.loads(stages_json)
except Exception as err:
logger.error("flavor file not found {}".format(err))
def find_qa_check(self, qa):
for check in self.steps_dic:
if qa in self.steps_dic[check]:
return check
return None
def load_merged_qa(self):
data = None
if self.process_id is not None:
data = self.models.get_output(self.process_id, self.cam)
return data
def update_status(self, qa):
index = -1
current_step = None
qa_status = None
for step in list(self.steps_dic):
try:
index = self.steps_dic[step].index(qa)
current_step = step
break
except ValueError:
continue
data = self.load_merged_qa()
try:
alert_key = self.alert_keys[qa]
check = self.find_qa_check(qa)
qa_status = data["TASKS"][check]["METRICS"][alert_key]
if current_step:
self.qas_status[current_step][index] = qa_status
except Exception as err:
logger.warning('Failed to update camera status: {}'.format(err))
def get_merged_qa_status(self):
for qa in list(self.alert_keys):
self.update_status(qa)
def load_qa(self):
cam = self.selected_arm + str(self.selected_spectrograph)
mergedqa = QLFModels().get_output(self.selected_process_id, cam)
gen_info = mergedqa['GENERAL_INFO']
ra = gen_info['RA']
dec = gen_info['DEC']
check_spectra = mergedqa['TASKS']['CHECK_SPECTRA']
skycont = check_spectra['METRICS']
sky = skycont['SKYCONT_FIBER']
skyfibers = gen_info['SKY_FIBERID']
nrg = check_spectra['PARAMS']['SKYCONT_NORMAL_RANGE']
wrg = check_spectra['PARAMS']['SKYCONT_WARN_RANGE']
current_exposures = [check_spectra['METRICS']['SKYCONT']]
program = gen_info['PROGRAM'].upper()
reference_exposures = check_spectra['PARAMS']['SKYCONT_' + program +
'_REF']
ra_sky = [ra[i % 500] for i in skyfibers]
dec_sky = [dec[i % 500] for i in skyfibers]
my_palette = get_palette("RdYlBu_r")
skc_tooltips = """
<div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">SKY CONT: </span>
<span style="font-size: 1vw; color: #515151;">@skycont</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">RA: </span>
<span style="font-size: 1vw; color: #515151;">@ra</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">DEC: </span>
<span style="font-size: 1vw; color: #515151;">@dec</span>
</div>
</div>
"""
url = "http://legacysurvey.org/viewer?ra=@ra&dec=@dec&zoom=16&layer=decals-dr5"
qlf_fiberid = np.arange(0, 500)
# sky continuum per sky fiber averaged over two continuum regions,
# 'n' is number of sky fibers
ra_not, dec_not = [], []
fiber_mod = []
for fiber in skyfibers:
fiber_mod.append(fiber % 500)
for i in range(500):
if i not in fiber_mod:
ra_not.append(ra[i])
dec_not.append(dec[i])
source = ColumnDataSource(
data={
'skycont': sky,
'delta_skycont': np.array(sky) - reference_exposures,
'fiberid': skyfibers,
'ra': ra_sky,
'dec': dec_sky
})
source_not = ColumnDataSource(data={
'ra': ra_not,
'dec': dec_not,
'skycont': [''] * len(dec_not)
})
low, high = wrg
mapper = LinearColorMapper(
palette=my_palette,
low=low, #np.min(sky),
high=high, #np.max(sky),
nan_color='darkgrey')
radius = 0.0165 # 0.013
radius_hover = 0.02 # 0.015
# centralize wedges in plots:
ra_center = 0.5 * (max(ra) + min(ra))
dec_center = 0.5 * (max(dec) + min(dec))
xrange_wedge = Range1d(start=ra_center + .95, end=ra_center - .95)
yrange_wedge = Range1d(start=dec_center + .82, end=dec_center - .82)
wedge_plot = Plot2d(x_range=xrange_wedge,
y_range=yrange_wedge,
x_label="RA",
y_label="DEC",
tooltip=skc_tooltips,
title="SKY_CONT",
width=500,
height=380,
yscale="auto").wedge(
source,
x='ra',
y='dec',
field='delta_skycont',
mapper=mapper,
).wedge(
source_not,
x='ra',
y='dec',
).plot
taptool = wedge_plot.select(type=TapTool)
taptool.callback = OpenURL(url=url)
# infos
info_col = Title().write_description('skycont')
# Prepare tables
keynames = ["SKYCONT" for i in range(len(current_exposures))]
table = Table().single_table(keynames, current_exposures,
reference_exposures, nrg, wrg)
layout = column(info_col,
Div(),
table,
Div(),
column(wedge_plot,
sizing_mode='scale_both',
css_classes=["main-one"]),
css_classes=["display-grid"])
# End of Bokeh Block
return file_html(layout, CDN, "SKYCONT")
def load_qa(self):
cam = self.selected_arm + str(self.selected_spectrograph)
mergedqa = QLFModels().get_output(self.selected_process_id, cam)
# list of available object in petal
objlist = mergedqa["TASKS"]["CHECK_SPECTRA"]["METRICS"]["OBJLIST"]
if 'SKY' in objlist:
objlist.remove('SKY')
gen_info = mergedqa['GENERAL_INFO']
ra = gen_info['RA']
dec = gen_info['DEC']
check_spectra = mergedqa['TASKS']['CHECK_SPECTRA']
snr = check_spectra['METRICS']
nrg = check_spectra['PARAMS']['FIDSNR_TGT_NORMAL_RANGE']
wrg = check_spectra['PARAMS']['FIDSNR_TGT_WARN_RANGE']
# Object identification in fibers:
obj_fiber = sort_obj(gen_info)
qlf_obj = ['ELG', 'LRG', 'QSO', 'STAR']
avobj = ['STAR' if x == 'STD' else x for x in objlist]
obj_idx = {}
for o in qlf_obj:
try:
obj_idx.update({o: avobj.index(o)})
except:
obj_idx.update({o: None})
try:
exptime = gen_info['EXPTIME']
name_warn = ''
except:
exptime = 1000
name_warn = ' (exptime fixed)'
# Sort objects for QLF:
obj_idx = {}
for o in qlf_obj:
try:
obj_idx.update({o: avobj.index(o)})
except:
obj_idx.update({o: None})
def good_idx(mag, snr):
# Filtering measurements with good SNR & good MAG
# Treating None (inf and Nan already treated in db)
mag_2 = np.array([-9998 if x is None else x for x in mag])
snr_2 = np.array([-9998 if x is None else x for x in snr])
idx = np.arange(len(snr_2))
# Filtering values with good mag AND snr
return list(idx[(mag_2 > -999) & (snr_2 > 0)])
# Treating bad snr and mag
mag_snr = {}
for o in avobj:
snr_ql, mag_ql = snr['SNR_MAG_TGT'][obj_idx[o]]
idx = good_idx(mag_ql, snr_ql)
x = [mag_ql[i] for i in idx]
y = [snr_ql[i] for i in idx]
mag_snr.update({o: [y, x]})
# Preparing xy_plot data:
if obj_idx['ELG'] is not None:
elg_snr = mag_snr['ELG']
if obj_idx['LRG'] is not None:
lrg_snr = mag_snr['LRG']
if obj_idx['QSO'] is not None:
qso_snr = mag_snr['QSO']
if obj_idx['STAR'] is not None:
star_snr = mag_snr['STAR']
#lrg_snr = mag_snr['LRG']
def fit_func(xdata, coeff):
""" astro fit
"""
r1 = 0.0 # read noise
a, b = coeff
x = np.linspace(min(xdata), max(xdata), 1000)
Flux = 10**(-0.4 * (x - 22.5))
y = a * Flux * exptime / np.sqrt(a * Flux * exptime + b * exptime +
r1**2)
return x, y
data_model = {
'x': [],
'y': [],
'y2': [],
'fiber_id': [],
'ra': [],
'dec': [],
}
elg = ColumnDataSource(data=data_model.copy())
lrg = ColumnDataSource(data=data_model.copy())
qso = ColumnDataSource(data=data_model.copy())
star = ColumnDataSource(data=data_model.copy())
data_fit = {
'x': [],
'y': [],
'y2': [],
'fiber_id': [],
'ra': [],
'dec': []
}
elg_fit = ColumnDataSource(data=data_fit.copy())
lrg_fit = ColumnDataSource(data=data_fit.copy())
qso_fit = ColumnDataSource(data=data_fit.copy())
star_fit = ColumnDataSource(data=data_fit.copy())
if obj_idx['ELG'] is not None:
elg.data['x'] = elg_snr[1]
elg.data['y'] = np.array(elg_snr[0])
elg.data['y2'] = np.array(elg_snr[0])**2
elg.data['fiber_id'] = gen_info['ELG_FIBERID']
elg.data['ra'] = [ra[i % 500] for i in gen_info['ELG_FIBERID']]
elg.data['dec'] = [dec[i % 500] for i in gen_info['ELG_FIBERID']]
xfit, yfit = fit_func(elg_snr[1],
snr['FITCOEFF_TGT'][obj_idx['ELG']])
elg_fit.data['x'] = xfit
elg_fit.data['y'] = np.array(yfit)
elg_fit.data['y2'] = np.array(yfit)**2
for key in ['fiber_id', 'ra', 'dec']:
elg_fit.data[key] = [''] * len(yfit)
if obj_idx['LRG'] is not None:
lrg.data['x'] = lrg_snr[1]
lrg.data['y'] = np.array(lrg_snr[0])
lrg.data['y2'] = np.array(lrg_snr[0])**2
lrg.data['fiber_id'] = gen_info['LRG_FIBERID']
lrg.data['ra'] = [ra[i % 500] for i in gen_info['LRG_FIBERID']]
lrg.data['dec'] = [dec[i % 500] for i in gen_info['LRG_FIBERID']]
xfit, yfit = fit_func(lrg_snr[1],
snr['FITCOEFF_TGT'][obj_idx['LRG']])
lrg_fit.data['x'] = xfit
lrg_fit.data['y'] = np.array(yfit)
lrg_fit.data['y2'] = np.array(yfit)**2
for key in ['fiber_id', 'ra', 'dec']:
lrg_fit.data[key] = [''] * len(yfit)
if obj_idx['QSO'] is not None:
qso.data['x'] = qso_snr[1]
qso.data['y'] = np.array(qso_snr[0])
qso.data['y2'] = np.array(qso_snr[0])**2
qso.data['fiber_id'] = gen_info['QSO_FIBERID']
qso.data['ra'] = [ra[i % 500] for i in gen_info['QSO_FIBERID']]
qso.data['dec'] = [dec[i % 500] for i in gen_info['QSO_FIBERID']]
xfit, yfit = fit_func(qso_snr[1],
snr['FITCOEFF_TGT'][obj_idx['QSO']])
qso_fit.data['x'] = xfit
qso_fit.data['y'] = np.array(yfit)
qso_fit.data['y2'] = np.array(yfit)**2
for key in ['fiber_id', 'ra', 'dec']:
qso_fit.data[key] = [''] * len(yfit)
if obj_idx['STAR'] is not None:
star.data['x'] = star_snr[1]
star.data['y'] = np.array(star_snr[0])
star.data['y2'] = np.array(star_snr[0])**2
star.data['fiber_id'] = gen_info['STAR_FIBERID']
star.data['ra'] = [ra[i % 500] for i in gen_info['STAR_FIBERID']]
star.data['dec'] = [dec[i % 500] for i in gen_info['STAR_FIBERID']]
xfit, yfit = fit_func(star_snr[1],
snr['FITCOEFF_TGT'][obj_idx['STAR']])
star_fit.data['x'] = xfit
star_fit.data['y'] = np.array(yfit)
star_fit.data['y2'] = np.array(yfit)**2
for key in ['fiber_id', 'ra', 'dec']:
star_fit.data[key] = [''] * len(yfit)
html_tooltip = """
<div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">SNR: </span>
<span style="font-size: 1vw; color: #515151;">@y</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">DECAM_{}: </span>
<span style="font-size: 1vw; color: #515151;">@x</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">Fiber ID: </span>
<span style="font-size: 1vw; color: #515151;">@fiber_id</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">RA: </span>
<span style="font-size: 1vw; color: #515151;">@ra</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">Dec: </span>
<span style="font-size: 1vw; color: #515151">@dec</span>
</div>
</div>
""".format(str(self.selected_arm).upper())
url = "http://legacysurvey.org/viewer?ra=@ra&dec=@dec&zoom=16&layer=decals-dr5"
elg_plot = Plot2d(
x_label="DECAM_{}".format(str(self.selected_arm).upper()),
y_label="MEDIAN SNR^2",
tooltip=html_tooltip,
title="ELG",
width=500,
height=380,
yscale="log",
).line(
source=elg_fit,
y='y2',
).circle(
source=elg,
size=8,
y='y2',
fill_color="blue",
).plot
taptool = elg_plot.select(type=TapTool)
taptool.callback = OpenURL(url=url)
lrg_plot = Plot2d(x_label="DECAM_{}".format(
str(self.selected_arm).upper()),
y_label="MEDIAN SNR^2",
tooltip=html_tooltip,
title="LRG",
width=500,
height=380,
yscale="log").line(
source=lrg_fit,
y='y2',
).circle(
source=lrg,
size=8,
y='y2',
fill_color="red",
).plot
taptool = lrg_plot.select(type=TapTool)
taptool.callback = OpenURL(url=url)
qso_plot = Plot2d(x_label="DECAM_{}".format(
str(self.selected_arm).upper()),
y_label="MEDIAN SNR^2",
tooltip=html_tooltip,
title="QSO",
width=500,
height=380,
yscale="log").line(
source=qso_fit,
y='y2',
).circle(
source=qso,
size=8,
y='y2',
fill_color="green",
).plot
taptool = qso_plot.select(type=TapTool)
taptool.callback = OpenURL(url=url)
star_plot = Plot2d(
x_label="DECAM_{}".format(str(self.selected_arm).upper()),
y_label="MEDIAN SNR^2",
tooltip=html_tooltip,
title="STAR",
width=500,
height=380,
yscale="log",
).line(source=star_fit, y='y2').circle(
source=star,
size=8,
y='y2',
fill_color="yellow",
).plot
taptool = star_plot.select(type=TapTool)
taptool.callback = OpenURL(url=url)
# infos
info_col = Title().write_description('snr')
# -----------------
# WEDGES
snr_tooltip = """
<div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">Resid: </span>
<span style="font-size: 1vw; color: #515151">@resid_snr</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">Obj Type: </span>
<span style="font-size: 1vw; color: #515151;">@OBJ_TYPE</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">RA: </span>
<span style="font-size: 1vw; color: #515151;">@x1</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">DEC: </span>
<span style="font-size: 1vw; color: #515151;">@y1</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">FIBER ID: </span>
<span style="font-size: 1vw; color: #515151;">@QLF_FIBERID</span>
</div>
</div>
"""
median = snr['MEDIAN_SNR']
resid = snr['SNR_RESID']
qlf_fiberid = range(0, 500)
my_palette = get_palette('bwr')
fibersnr_tgt = []
for i in avobj:
fibersnr_tgt.append(gen_info[i + '_FIBERID'])
fibersnr = []
for i in list(range(len(fibersnr_tgt))):
fibersnr = fibersnr + fibersnr_tgt[i]
source = ColumnDataSource(
data={
'x1': [ra[i % 500] for i in fibersnr],
'y1': [dec[i % 500] for i in fibersnr],
'resid_snr': resid,
'QLF_FIBERID': fibersnr,
'OBJ_TYPE': [obj_fiber[i % 500] for i in fibersnr],
'median': median
})
ra_not = []
dec_not = []
obj_not = []
fiber_not = []
fiber_mod = []
for fiber in fibersnr:
fiber_mod.append(fiber % 500)
for i in range(500):
if i not in fiber_mod:
ra_not.append(ra[i])
dec_not.append(dec[i])
fiber_not.append(i)
obj_not.append(obj_fiber[i])
source_not = ColumnDataSource(
data={
'x1': ra_not,
'y1': dec_not,
'resid_snr': [''] * len(dec_not),
'QLF_FIBERID': fiber_not,
'OBJ_TYPE': obj_not
})
rmax, rmin = np.nanmax(resid), np.nanmin(resid)
if np.isnan(rmax) or np.isnan(rmin):
fill_color = 'lightgray'
else:
dy = (rmax - rmin) * 0.1
mapper = LinearColorMapper(palette=my_palette,
nan_color='darkgray',
low=rmin - dy,
high=rmax + dy)
fill_color = {'field': 'resid_snr', 'transform': mapper}
radius = 0.0165
radius_hover = 0.02
# centralize wedges in plots:
ra_center = 0.5 * (max(ra) + min(ra))
dec_center = 0.5 * (max(dec) + min(dec))
xrange_wedge = Range1d(start=ra_center + .95, end=ra_center - .95)
yrange_wedge = Range1d(start=dec_center + .82, end=dec_center - .82)
# axes limit
xmin, xmax = [min(ra[:]), max(ra[:])]
ymin, ymax = [min(dec[:]), max(dec[:])]
xfac, yfac = [(xmax - xmin) * 0.06, (ymax - ymin) * 0.06]
left, right = xmin - xfac, xmax + xfac
bottom, top = ymin - yfac, ymax + yfac
# WEDGE RESIDUAL plots
wedge_plot = Plot2d(x_range=xrange_wedge,
y_range=yrange_wedge,
x_label="RA",
y_label="DEC",
tooltip=snr_tooltip,
title='Residual SNR' + name_warn,
width=500,
height=380,
yscale="auto").wedge(
source,
x='x1',
y='y1',
field='resid_snr',
mapper=mapper,
).wedge(
source_not,
x='x1',
y='y1',
).plot
taptool = wedge_plot.select(type=TapTool)
taptool.callback = OpenURL(url=url)
# -------------------
# Median plot
median_tooltip = """
<div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">MEDIAN: </span>
<span style="font-size: 1vw; color: #515151">@median</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">Resid: </span>
<span style="font-size: 1vw; color: #515151">@resid_snr</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">Obj Type: </span>
<span style="font-size: 1vw; color: #515151;">@OBJ_TYPE</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">RA: </span>
<span style="font-size: 1vw; color: #515151;">@x1</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">DEC: </span>
<span style="font-size: 1vw; color: #515151;">@y1</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">FIBER ID: </span>
<span style="font-size: 1vw; color: #515151;">@QLF_FIBERID</span>
</div>
</div>
"""
mediam_plot = Plot2d(
x_label="Fiber",
y_label='Median SNR',
tooltip=median_tooltip,
title="",
width=600,
height=400,
yscale="auto",
hover_mode="vline",
).vbar(
source,
y="median",
x="QLF_FIBERID",
line_width=0.4,
)
# Prepare tables
current_exposures = check_spectra['METRICS']['FIDSNR_TGT']
program = gen_info['PROGRAM'].upper()
reference_exposures = check_spectra['PARAMS']['FIDSNR_TGT_' + program +
'_REF']
keynames = ["FIDSNR_TGT" + " ({})".format(i) for i in objlist]
metric = Table().reference_table(keynames, current_exposures,
reference_exposures)
alert = Table().alert_table(nrg, wrg)
layout = column(info_col,
Div(),
metric,
alert,
column(elg_plot, sizing_mode='scale_both'),
column(lrg_plot, sizing_mode='scale_both'),
column(qso_plot, sizing_mode='scale_both'),
column(star_plot, sizing_mode='scale_both'),
column(mediam_plot, sizing_mode='scale_both'),
column(wedge_plot, sizing_mode='scale_both'),
css_classes=["display-grid"])
return file_html(layout, CDN, "MEDIAN SNR")
def __init__(self, data):
self.pipeline_name = 'Quick Look'
self.data = data
self.models = QLFModels()
def data_source(self, fmap):
""" Creating data source for plots
"""
data_model = {
'x_b': [],
'x_r': [],
'x_z': [],
'OBJ_TYPE': [],
'ra': [],
'dec': [],
'cam_b': [],
'cam_r': [],
'cam_z': []
}
process_id = self.selected_process_id
joblist = [entry.camera.camera for entry in Job.objects.filter(
process_id=process_id)]
ra_tile = fmap.fiber_ra
dec_tile = fmap.fiber_dec
otype_tile = fmap.objtype
objlist = sorted(set(otype_tile))
if 'SKY' in objlist:
objlist.remove('SKY')
fibaux = []
y = []
cam_inst = []
ra_all = []
dec_all = []
for spect in list(range(1)):
cam = self.selected_arm+str(spect)
if cam in joblist:
mergedqa = QLFModels().get_output(
self.selected_process_id, cam)
# Assign available objects
objtype = otype_tile[500*spect: 500*(spect + 1)]
med_snr = np.array(
mergedqa['TASKS']['CHECK_SPECTRA']['METRICS']["MEDIAN_SNR"])
resids = mergedqa['TASKS']['CHECK_SPECTRA']['METRICS']['SNR_RESID']
obj = np.arange(len(objlist))
rayes = []
decyes = []
for t in range(len(obj)):
otype = list(objlist)[t]
oid = np.where(np.array(list(objlist)) == otype)[0][0]
if otype == 'STD':
fibers = mergedqa['GENERAL_INFO']['STAR_FIBERID']
else:
fibers = mergedqa['GENERAL_INFO']['%s_FIBERID' % otype]
fibaux = fibaux + [500*spect + i for i in fibers]
for i in range(len(fibers)):
ras = mergedqa['GENERAL_INFO']['RA'][fibers[i]]
decs = mergedqa['GENERAL_INFO']['DEC'][fibers[i]]
rayes.append(ras)
decyes.append(decs)
ra_all = ra_all + rayes
dec_all = dec_all + decyes
nanresids = [i if i > -9999. else np.nan for i in resids]
y = y + nanresids
else:
y = y + 500*[np.nan]
ra_all = ra_all + \
[i for i in ra_tile[500*spect:500*(spect + 1)]]
dec_all = dec_all + \
[i for i in dec_tile[500*spect:500*(spect + 1)]]
fibaux = fibaux + list(range(500*spect, 500*(spect+1)))
cam_inst = cam_inst + [cam]*500
data_model['x_' + cam[0]] = y
data_model['cam_'+cam[0]] = cam_inst
data_model['OBJ_TYPE'] = [otype_tile[ii] for ii in fibaux]
data_model['ra'] = ra_all
data_model['dec'] = dec_all
source = ColumnDataSource(data=data_model)
return source
def load_qa(self):
cam = self.selected_arm + str(self.selected_spectrograph)
mergedqa = QLFModels().get_output(self.selected_process_id, cam)
gen_info = mergedqa['GENERAL_INFO']
flavor = mergedqa['FLAVOR']
if flavor == "science":
ra = gen_info['RA']
dec = gen_info['DEC']
check_ccds = mergedqa['TASKS']['CHECK_CCDs']
xwsigma = check_ccds['METRICS']['XWSIGMA']
xw_amp = check_ccds['METRICS']['XWSIGMA_AMP']
xw_fib = check_ccds['METRICS']['XWSIGMA_FIB']
nrg = check_ccds['PARAMS']['XWSIGMA_NORMAL_RANGE']
wrg = check_ccds['PARAMS']['XWSIGMA_WARN_RANGE']
obj_type = sort_obj(gen_info) # [""]*500
if mergedqa['FLAVOR'].upper() == 'SCIENCE':
program = mergedqa['GENERAL_INFO']['PROGRAM'].upper()
program_prefix = '_' + program
else:
program_prefix = ''
xw_ref = check_ccds['PARAMS']['XWSIGMA' + program_prefix + '_REF']
xsigma = xw_fib[0]
wsigma = xw_fib[1]
delta_rg = wrg[1] - wrg[0]
hist_rg = (wrg[0] - 0.1 * delta_rg, wrg[1] + 0.1 * delta_rg)
my_palette = get_palette("RdYlBu_r")
xfiber = np.arange(len(xsigma))
wfiber = np.arange(len(wsigma))
if mergedqa['FLAVOR'].upper() != 'SCIENCE':
ra = np.full(500, np.nan)
dec = np.full(500, np.nan)
source = ColumnDataSource(
data={
'x1': ra,
'y1': dec,
'xsigma': xsigma,
'wsigma': wsigma,
'delta_xsigma': np.array(xsigma) - xw_ref[0],
'delta_wsigma': np.array(wsigma) - xw_ref[1],
'xref': [xw_ref[0]] * len(xsigma),
'wref': [xw_ref[1]] * len(xsigma),
'xfiber': xfiber,
'wfiber': wfiber,
'OBJ_TYPE': obj_type,
'left': np.arange(0, 500) - 0.4,
'right': np.arange(0, 500) + 0.4,
'bottom': [0] * 500
})
xsigma_tooltip = """
<div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">XSigma: </span>
<span style="font-size: 1vw; color: #515151">@xsigma</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">Reference: </span>
<span style="font-size: 1vw; color: #515151">@xref</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">Obj Type: </span>
<span style="font-size: 1vw; color: #515151;">@OBJ_TYPE</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">RA: </span>
<span style="font-size: 1vw; color: #515151;">@x1</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">DEC: </span>
<span style="font-size: 1vw; color: #515151;">@y1</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">FIBER ID: </span>
<span style="font-size: 1vw; color: #515151;">@xfiber</span>
</div>
</div>
"""
wsigma_tooltip = """
<div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">WSigma: </span>
<span style="font-size: 1vw; color: #515151">@wsigma</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">Reference: </span>
<span style="font-size: 1vw; color: #515151">@wref</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">Obj Type: </span>
<span style="font-size: 1vw; color: #515151;">@OBJ_TYPE</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">RA: </span>
<span style="font-size: 1vw; color: #515151;">@x1</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">DEC: </span>
<span style="font-size: 1vw; color: #515151;">@y1</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">FIBER ID: </span>
<span style="font-size: 1vw; color: #515151;">@wfiber</span>
</div>
</div>
"""
url = "http://legacysurvey.org/viewer?ra=@ra&dec=@dec&zoom=16&layer=decals-dr5"
# determining the position of selected cam fibers:
obj_type = sort_obj(gen_info)
# ---------------------------------
if flavor == "science":
# axes limit
xmin, xmax = [min(ra[:]), max(ra[:])]
ymin, ymax = [min(dec[:]), max(dec[:])]
xfac, yfac = [(xmax - xmin) * 0.06, (ymax - ymin) * 0.06]
left, right = xmin - xfac, xmax + xfac
bottom, top = ymin - yfac, ymax + yfac
low, high = wrg
xmapper = LinearColorMapper(palette=my_palette,
low=low,
high=high,
nan_color="darkgrey")
wmapper = LinearColorMapper(palette=my_palette,
low=low,
high=high,
nan_color="darkgrey")
# ============
# XSIGMA WEDGE
radius = 0.0165
radius_hover = 0.02
# centralize wedges in plots:
ra_center = 0.5 * (max(ra) + min(ra))
dec_center = 0.5 * (max(dec) + min(dec))
xrange_wedge = Range1d(start=ra_center + .95, end=ra_center - .95)
yrange_wedge = Range1d(start=dec_center + .82,
end=dec_center - .82)
wedge_plot_x = Plot2d(x_range=xrange_wedge,
y_range=yrange_wedge,
x_label="RA",
y_label="DEC",
tooltip=xsigma_tooltip,
title="XSIGMA",
width=500,
height=380,
yscale="auto").wedge(
source,
x='x1',
y='y1',
field='delta_xsigma',
mapper=xmapper,
colorbar_title='xsigma').plot
taptool = wedge_plot_x.select(type=TapTool)
taptool.callback = OpenURL(url=url)
# ============
# WSIGMA WEDGE
wedge_plot_w = Plot2d(x_range=xrange_wedge,
y_range=yrange_wedge,
x_label="RA",
y_label="DEC",
tooltip=wsigma_tooltip,
title="WSIGMA",
width=500,
height=380,
yscale="auto").wedge(
source,
x='x1',
y='y1',
field='delta_wsigma',
mapper=wmapper,
colorbar_title='wsigma').plot
taptool = wedge_plot_w.select(type=TapTool)
taptool.callback = OpenURL(url=url)
# ================================
# Stat histogram
# x_fiber_hist
d_yplt = (max(xsigma) - min(xsigma)) * 0.1
yrange = [0, max(xsigma) + d_yplt]
xhist = Plot2d(
yrange,
x_label="Fiber number",
y_label="X std dev (number of pixels)",
tooltip=xsigma_tooltip,
title="",
width=600,
height=300,
yscale="auto",
hover_mode="vline",
).quad(
source,
top='xsigma',
)
# w_fiber_hist
d_yplt = (max(wsigma) - min(wsigma)) * 0.1
yrange = [0, max(wsigma) + d_yplt]
whist = Plot2d(
yrange,
x_label="Fiber number",
y_label="W std dev (number of pixels)",
tooltip=xsigma_tooltip,
title="",
width=600,
height=300,
yscale="auto",
hover_mode="vline",
).quad(
source,
top='wsigma',
)
# ================================
# Stat histogram
def histpar(yscale, hist):
if yscale == 'log':
ylabel = "Frequency + 1"
yrange = (1, 11**(int(np.log10(max(hist))) + 1))
bottomval = 'bottomplusone'
histval = 'histplusone'
else:
ylabel = "Frequency"
yrange = (0.0 * max(hist), 1.1 * max(hist))
bottomval = 'bottom'
histval = 'hist'
return [ylabel, yrange, bottomval, histval]
xhistlabel = "XSIGMA"
yscale = "auto"
hist_tooltip_x = """
<div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">Frequency: </span>
<span style="font-size: 1vw; color: #515151">@hist</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">XSIGMA: </span>
<span style="font-size: 1vw; color: #515151;">[@left, @right]</span>
</div>
</div>
"""
hist, edges = np.histogram(xsigma, 'sqrt')
source_hist = ColumnDataSource(
data={
'hist': hist,
'histplusone': hist + 1,
'bottom': [0] * len(hist),
'bottomplusone': [1] * len(hist),
'left': edges[:-1],
'right': edges[1:]
})
ylabel, yrange, bottomval, histval = histpar(yscale, hist)
p_hist_x = Plot2d(
x_label=xhistlabel,
y_label=ylabel,
tooltip=hist_tooltip_x,
title="",
width=600,
height=300,
yscale="auto",
y_range=yrange,
x_range=(hist_rg[0] + xw_ref[0], hist_rg[1] + xw_ref[0]),
hover_mode="vline",
).quad(
source_hist,
top=histval,
bottom=bottomval,
line_width=0.4,
)
# Histogram 2
xhistlabel = "WSIGMA"
hist_tooltip_w = """
<div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">Frequency: </span>
<span style="font-size: 1vw; color: #515151">@hist</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">WSIGMA: </span>
<span style="font-size: 1vw; color: #515151;">[@left, @right]</span>
</div>
</div>
"""
hist, edges = np.histogram(wsigma, 'sqrt')
source_hist = ColumnDataSource(
data={
'hist': hist,
'histplusone': hist + 1,
'bottom': [0] * len(hist),
'bottomplusone': [1] * len(hist),
'left': edges[:-1],
'right': edges[1:]
})
ylabel, yrange, bottomval, histval = histpar(yscale, hist)
yrangew = yrange
p_hist_w = Plot2d(
x_label=xhistlabel,
y_label=ylabel,
tooltip=hist_tooltip_w,
title="",
width=600,
height=300,
yscale="auto",
y_range=yrange,
x_range=(hist_rg[0] + xw_ref[1], hist_rg[1] + xw_ref[1]),
hover_mode="vline",
).quad(
source_hist,
top=histval,
bottom=bottomval,
line_width=0.8,
)
# --------------------------------------------------------------
# vlines ranges:
bname = 'XWSIGMA'
for ialert in nrg: # par[bname+'_NORMAL_RANGE']:
spans = Span(location=ialert + xw_ref[0],
dimension='height',
line_color='green',
line_dash='dashed',
line_width=2)
p_hist_x.add_layout(spans)
my_label = Label(x=ialert + xw_ref[0],
y=yrange[-1] / 2.2,
y_units='data',
text='Normal Range',
text_color='green',
angle=np.pi / 2.)
p_hist_x.add_layout(my_label)
for ialert in wrg: # par[bname+'_WARN_RANGE']:
spans = Span(location=ialert + xw_ref[0],
dimension='height',
line_color='tomato',
line_dash='dotdash',
line_width=2)
p_hist_x.add_layout(spans)
my_label = Label(x=ialert + xw_ref[0],
y=yrange[-1] / 2.2,
y_units='data',
text='Warning Range',
text_color='tomato',
angle=np.pi / 2.)
p_hist_x.add_layout(my_label)
for ialert in nrg: # par[bname+'_NORMAL_RANGE']:
spans = Span(location=ialert + xw_ref[1],
dimension='height',
line_color='green',
line_dash='dashed',
line_width=2)
p_hist_w.add_layout(spans)
my_label = Label(x=ialert + xw_ref[1],
y=yrangew[-1] / 2.2,
y_units='data',
text='Normal Range',
text_color='green',
angle=np.pi / 2.)
p_hist_w.add_layout(my_label)
for ialert in wrg: # par[bname+'_WARN_RANGE']:
spans = Span(location=ialert + xw_ref[1],
dimension='height',
line_color='tomato',
line_dash='dotdash',
line_width=2)
p_hist_w.add_layout(spans)
my_label = Label(x=ialert + xw_ref[1],
y=yrangew[-1] / 2.2,
y_units='data',
text='Warning Range',
text_color='tomato',
angle=np.pi / 2.)
p_hist_w.add_layout(my_label)
# amp 1
xamp = Patch().plot_amp(
dz=xw_amp[0],
refexp=[xw_ref[0]] * 4,
name="XSIGMA AMP",
description="X standard deviation per Amp (number of pixels)",
wrg=wrg)
# amp 2
wamp = Patch().plot_amp(
dz=xw_amp[1],
refexp=[xw_ref[1]] * 4,
name="WSIGMA AMP",
description="W standard deviation per Amp (number of pixels)",
wrg=wrg)
# -------------------------------------------------------------------------
info_col = Title().write_description('xwsigma')
current_exposures = check_ccds['METRICS']['XWSIGMA']
alert_x = Table().alert_table(nrg, wrg)
alert_w = Table().alert_table(nrg, wrg)
if flavor == 'science':
program = gen_info['PROGRAM'].upper()
reference_exposures = check_ccds['PARAMS']['XWSIGMA_' + program +
'_REF']
keynames = ["XSIGMA"]
x_metric = Table().reference_table(keynames,
[current_exposures[0]],
[reference_exposures[0]])
keynames = ["WSIGMA"]
w_metric = Table().reference_table(keynames,
[current_exposures[1]],
[reference_exposures[1]])
layout = column(
info_col,
widgetbox(Div(), css_classes=["tableranges"]),
widgetbox(
Div(text=
'<h2 align=center style="text-align:center;"> XSIGMA </h2>'
)),
widgetbox(
Div(text=
'<h2 align=center style="text-align:center;"> WSIGMA </h2>'
)),
x_metric,
w_metric,
alert_x,
alert_w,
column(wedge_plot_x, sizing_mode='scale_both'),
column(wedge_plot_w, sizing_mode='scale_both'),
column(xhist, sizing_mode='scale_both'),
column(whist, sizing_mode='scale_both'),
column(p_hist_x, sizing_mode='scale_both'),
column(p_hist_w, sizing_mode='scale_both'),
column(xamp, sizing_mode='scale_both'),
column(wamp, sizing_mode='scale_both'),
css_classes=["display-grid"],
sizing_mode='scale_width')
else:
reference_exposures = check_ccds['PARAMS']['XWSIGMA_REF']
keynames = ["XSIGMA"]
x_metric = Table().reference_table(keynames,
[current_exposures[0]],
[reference_exposures[0]])
keynames = ["WSIGMA"]
w_metric = Table().reference_table(keynames,
[current_exposures[1]],
[reference_exposures[1]])
layout = column(
info_col,
widgetbox(Div(), css_classes=["tableranges"]),
widgetbox(
Div(text=
'<h2 align=center style="text-align:center;"> XSIGMA </h2>'
)),
widgetbox(
Div(text=
'<h2 align=center style="text-align:center;"> WSIGMA </h2>'
)),
x_metric,
w_metric,
alert_x,
alert_w,
column(xhist, sizing_mode='scale_both'),
column(whist, sizing_mode='scale_both'),
column(p_hist_x, sizing_mode='scale_both'),
column(p_hist_w, sizing_mode='scale_both'),
column(xamp, sizing_mode='scale_both'),
column(wamp, sizing_mode='scale_both'),
css_classes=["display-grid"],
sizing_mode='scale_width')
return file_html(layout, CDN, "XWSIGMA")
class Regression():
def __init__(self, xaxis, yaxis, start, end, camera):
self.yaxis = yaxis
self.xaxis = xaxis
self.start = datetime.strptime(start, '%Y%m%d').strftime('%Y-%m-%d')
self.end = datetime.strptime(end, '%Y%m%d').strftime('%Y-%m-%d')
self.camera = camera
self.models = QLFModels()
def render_plot(self, outputs_x, outputs_y):
metrics_path = os.path.join(qlf_root, "framework", "ql_mapping",
"metrics.json")
with open(metrics_path) as f:
metrics = json.load(f)
y_data = metrics[self.yaxis]
x_data = metrics[self.xaxis]
df_x = pd.DataFrame(list(outputs_x))
df_y = pd.DataFrame(list(outputs_y))
source = ColumnDataSource(
data=dict(x=df_x['value'].apply(lambda x: x[0]),
y=df_y['value'].apply(lambda x: x[0]),
exposure=df_x['exposure_id'],
camera=df_x['camera']))
TOOLTIPS = """
<div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">{}: </span>
<span style="font-size: 1vw; color: #515151">@y</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">{}: </span>
<span style="font-size: 1vw; color: #515151;">@x</span>
</div>
<div>
<span style="font-size: 1vw; font-weight: bold; color: #303030;">Exposure: </span>
<span style="font-size: 1vw; color: #515151;">@exposure</span>
</div>
</div>
""".format(y_data['display'], x_data['display'])
hover = HoverTool(tooltips=TOOLTIPS)
plot = figure(
title="Camera: {}".format(self.camera),
toolbar_location='above',
active_drag="box_zoom",
plot_height=300,
# plot_width=500,
x_axis_label=x_data['display'],
y_axis_label=y_data['display'],
tools=[hover, 'box_zoom,pan,wheel_zoom,box_select,reset'],
sizing_mode='scale_width')
q = plot.circle('x',
'y',
source=source,
size=8,
fill_color='dodgerblue',
hover_fill_color='blue',
line_color='black')
font_size = "1.2vw"
plot.xaxis.major_label_text_font_size = font_size
plot.yaxis.major_label_text_font_size = font_size
plot.xaxis.axis_label_text_font_size = font_size
plot.yaxis.axis_label_text_font_size = font_size
plot.legend.label_text_font_size = font_size
plot.title.text_font_size = font_size
self.layout = plot
def render(self):
outputs_y = self.models.get_product_metrics_by_camera(
self.yaxis, self.camera, begin_date=self.start, end_date=self.end)
outputs_x = self.models.get_product_metrics_by_camera(
self.xaxis, self.camera, begin_date=self.start, end_date=self.end)
self.render_plot(outputs_x, outputs_y)
return file_html(self.layout, CDN, "Regression")
