def __init__(self, xaxis, yaxis, start, end, camera): self.yaxis = yaxis self.xaxis = xaxis self.start = datetime.strptime(start, '%Y%m%d').strftime('%Y-%m-%d') self.end = datetime.strptime(end, '%Y%m%d').strftime('%Y-%m-%d') self.camera = camera self.models = QLFModels()
def __init__(self, process_id, cam, exp, night): self.cam = cam self.exp = exp self.night = night self.process_id = process_id # This is True if the pipeline didn't generate some yaml file self.error = dict(zip(self.qa_name, ['False'] * len(self.qa_name))) logger.info( 'check *rms_over *bias *SUMCOUNT_RMS shouldbe SUMCOUNT_MED_SKY' + 'Resigf skyresid- residrms') # QA tests and keys to respective values self.metric_qa_list = [ 'getbias', 'getrms', 'skycont', 'countbins', 'countpix', 'snr', 'skyresid', 'skypeak', 'integ', 'xsigma', 'wsigma' ] # THIS LINE : TB CHECKED self.metric_key_list = [ 'BIAS', 'RMS_OVER', 'SKYCONT', 'NGOODFIBERS', 'NPIX_LOW', 'ELG_FIDMAG_SNR', 'RESID_RMS', 'SUMCOUNT_MED_SKY', 'MAGDIFF_AVG', 'XSHIFT', 'WSHIFT' ] self.metric_dict = dict(zip(self.metric_qa_list, self.metric_key_list)) self.models = QLFModels() try: # ff self.metrics, self.tests = self.Load_metrics_n_tests() except Exception as e: # ff print(e) sys.exit("Could not load metrics and tests")
class QLFProcess(object): """ Class responsible for managing Quick Look pipeline process. """ def __init__(self, data): self.pipeline_name = 'Quick Look' self.data = data self.models = QLFModels() def start_process(self): """ Start pipeline. """ logger.info('Started %s ...' % self.pipeline_name) logger.info('Night: %s' % self.data.get('night')) logger.info('Exposure: %s' % str(self.data.get('expid'))) self.data['start'] = datetime.datetime.now().replace(microsecond=0) # create process in database and obtain the process id process = self.models.insert_process(self.data, self.pipeline_name) self.data['process_id'] = process.id self.data['status'] = process.status # TODO: ingest configuration file used, this should be done by process # self.models.insert_config(process.id) logger.info('Process ID: %i' % process.id) logger.info('Starting...') output_dir = os.path.join('exposures', self.data.get('night'), self.data.get('zfill')) output_full_dir = os.path.join(self.data.get('desi_spectro_redux'), output_dir) # Make sure output dir is created if not os.path.isdir(output_full_dir): os.makedirs(output_full_dir) logger.info('Output dir: %s' % output_dir) self.data['output_dir'] = output_dir def finish_process(self): """ Finish pipeline. """ self.data['end'] = datetime.datetime.now().replace(microsecond=0) self.data['duration'] = str( self.data.get('end') - self.data.get('start')) logger.info("Process with expID {} completed in {}.".format( self.data.get('expid'), self.data.get('duration'))) self.models.update_process(process_id=self.data.get('process_id'), end=self.data.get('end'), process_dir=self.data.get('output_dir'), status=self.data.get('status'))
def load_qa(self): cam = self.selected_arm + str(self.selected_spectrograph) mergedqa = QLFModels().get_output(self.selected_process_id, cam) check_ccds = mergedqa['TASKS']['CHECK_CCDs'] getrms = check_ccds['METRICS'] nrg = check_ccds['PARAMS']['NOISE_AMP_NORMAL_RANGE'] wrg = check_ccds['PARAMS']['NOISE_AMP_WARN_RANGE'] if mergedqa['FLAVOR'].upper() == 'SCIENCE': program = mergedqa['GENERAL_INFO']['PROGRAM'].upper() program_prefix = '_' + program else: program_prefix = '' refexp = mergedqa['TASKS']['CHECK_CCDs']['PARAMS']['NOISE_AMP' + program_prefix + '_REF'] # amp 1 p = Patch().plot_amp(dz=getrms["NOISE_AMP"], refexp=refexp, name="NOISE_AMP", description="NOISE per Amp (photon counts)", wrg=wrg) # amp 2 p2 = Patch().plot_amp( dz=getrms["NOISE_OVERSCAN_AMP"], refexp=refexp, name="NOISE_OVERSCAN_AMP", description="NOISE Overscan per Amp (photon counts)", wrg=wrg) info_col = Title().write_description('getrms') # Prepare tables current_exposures = check_ccds['METRICS']['NOISE_AMP'] gen_info = mergedqa['GENERAL_INFO'] flavor = mergedqa["FLAVOR"] if flavor == 'science': program = gen_info['PROGRAM'].upper() reference_exposures = check_ccds['PARAMS']['LITFRAC_AMP_' + program + '_REF'] else: reference_exposures = check_ccds['PARAMS']['LITFRAC_AMP_REF'] keynames = ["NOISE_AMP" for i in range(len(current_exposures))] metric = Table().reference_table(keynames, current_exposures, reference_exposures) alert = Table().alert_table(nrg, wrg) layout = column(info_col, Div(), metric, alert, column(p, sizing_mode='scale_both'), column(p2, sizing_mode='scale_both'), css_classes=["display-grid"]) return file_html(layout, CDN, "GETRMS")
def load_qa(self): cam = self.selected_arm+str(self.selected_spectrograph) mergedqa = QLFModels().get_output(self.selected_process_id, cam) check_ccds = mergedqa['TASKS']['CHECK_CCDs'] getbias = check_ccds['METRICS'] nrg = check_ccds['PARAMS']['BIAS_AMP_NORMAL_RANGE'] wrg = check_ccds['PARAMS']['BIAS_AMP_WARN_RANGE'] if mergedqa['FLAVOR'].upper() == 'SCIENCE': program = mergedqa['GENERAL_INFO']['PROGRAM'].upper() program_prefix = '_'+program else: program_prefix = '' refexp = mergedqa['TASKS']['CHECK_CCDs']['PARAMS']['BIAS_AMP' + program_prefix+'_REF'] # PATCH p = Patch().plot_amp( dz=getbias["BIAS_AMP"], refexp=refexp, name="BIAS_AMP", description="Average bias value (photon counts)", wrg=wrg ) p_status = Patch().plot_amp( dz=getbias["BIAS_AMP"], refexp=refexp, name="BIAS_AMP (STATUS)", description="Average bias value (photon counts)", wrg=wrg, nrg=nrg, status_plot=True, ) # INFO TABLES: info_col = Title().write_description('getbias') # Prepare tables current_exposures = check_ccds['METRICS']['BIAS_AMP'] gen_info = mergedqa['GENERAL_INFO'] flavor = mergedqa["FLAVOR"] if flavor == 'science': program = gen_info['PROGRAM'].upper() reference_exposures = check_ccds['PARAMS']['BIAS_AMP_' + program + '_REF'] else: reference_exposures = check_ccds['PARAMS']['BIAS_AMP_REF'] keynames = ["BIAS_AMP" for i in range(len(current_exposures))] table = Table().single_table(keynames, current_exposures, reference_exposures, nrg, wrg) layout = column(info_col, Div(), table, Div(), column(p, sizing_mode='scale_both'), #css_classes=["main-one"]), column(p_status, sizing_mode='scale_both'), css_classes=["display-grid"]) return file_html(layout, CDN, "GETBIAS")
def load_qa(self): cam = self.selected_arm + str(self.selected_spectrograph) mergedqa = QLFModels().get_output(self.selected_process_id, cam) check_flat = mergedqa['TASKS']['CHECK_FIBERFLAT'] flat = check_flat['METRICS'] nrg = check_flat['PARAMS']['CHECKFLAT_NORMAL_RANGE'] wrg = check_flat['PARAMS']['CHECKFLAT_WARN_RANGE'] info_col = Title().write_description('fiberflat') # Prepare tables current_exposures = check_flat['METRICS']['CHECKFLAT'] reference_exposures = check_flat['PARAMS']['CHECKFLAT_REF'] keynames = ["CHECKFLAT"] table = Table().single_table(keynames, [current_exposures], reference_exposures, nrg, wrg) layout = column(info_col, Div(), table, Div(), css_classes=["display-grid"]) return file_html(layout, CDN, "FIBERFLAT")
def load_qa(self): cam = self.selected_arm + str(self.selected_spectrograph) mergedqa = QLFModels().get_output(self.selected_process_id, cam) check_arc = mergedqa['TASKS']['CHECK_ARC'] nrg = check_arc['PARAMS']['CHECKARC_NORMAL_RANGE'] wrg = check_arc['PARAMS']['CHECKARC_WARN_RANGE'] info_col = Title().write_description('arc') # Prepare tables current_exposures = check_arc['METRICS']['CHECKARC'] reference_exposures = check_arc['PARAMS']['CHECKARC_REF'] keynames = ["CHECKARC"] table = Table().single_table(keynames, [current_exposures], reference_exposures, list(map(int, nrg)), list(map(int, wrg))) layout = column(info_col, Div(), table, Div(), css_classes=["display-grid"]) return file_html(layout, CDN, "ARC")
def load_qa(self): cam = self.selected_arm+str(self.selected_spectrograph) mergedqa = QLFModels().get_output(self.selected_process_id, cam) check_fibers = mergedqa['TASKS']['CHECK_FIBERS'] gen_info = mergedqa['GENERAL_INFO'] flavor = mergedqa['FLAVOR'] nrg = check_fibers['PARAMS']['XYSHIFTS_NORMAL_RANGE'] wrg = check_fibers['PARAMS']['XYSHIFTS_WARN_RANGE'] info_col = Title().write_description('xyshifts') # Prepare tables current_exposures = check_fibers['METRICS']['XYSHIFTS'] program = gen_info['PROGRAM'].upper() if flavor == 'science': reference_exposures = check_fibers['PARAMS']['XYSHIFTS_' + program + '_REF'] else: reference_exposures = [] keynames = ["X", "Y"] metric = Table().reference_table(keynames, current_exposures, reference_exposures) alert = Table().alert_table(nrg, wrg) layout = column(info_col, Div(), metric, alert, css_classes=["display-grid"]) return file_html(layout, CDN, "XYSHIFTS")
def load_qa(self): cam = self.selected_arm+str(self.selected_spectrograph) mergedqa = QLFModels().get_output(self.selected_process_id, cam) gen_info = mergedqa['GENERAL_INFO'] check_spectra = mergedqa['TASKS']['CHECK_SPECTRA'] nrg = check_spectra['PARAMS']['SKYRBAND_NORMAL_RANGE'] wrg = check_spectra['PARAMS']['SKYRBAND_WARN_RANGE'] info_col = Title().write_description('skyR') # Prepare Tables current_exposures = [check_spectra['METRICS']['SKYRBAND']] program = gen_info['PROGRAM'].upper() reference_exposures = check_spectra['PARAMS']['SKYRBAND_' + program + '_REF'] keynames = ["SKYRBAND" for i in range(len(current_exposures))] table = Table().single_table(keynames, current_exposures, reference_exposures, nrg, wrg) layout = column(info_col, Div(), table, Div(), css_classes=["display-grid"]) return file_html(layout, CDN, "SKYR")
def data_source(self, fmap): """ Creating data source for plots """ data_model = { 'goodfiber': [], 'status': [], 'color': [], 'cam': [], 'OBJ_TYPE': [], 'ra': [], 'dec': [], } process_id = self.selected_process_id joblist = [ entry.camera.camera for entry in Job.objects.filter(process_id=process_id) ] ra_tile = fmap.fiber_ra dec_tile = fmap.fiber_dec otype_tile = fmap.objtype y = [] color = [] status = [] cam_inst = [] for spec in list(range(10)): cam = self.selected_arm + str(spec) if cam in joblist: mergedqa = QLFModels().get_output(self.selected_process_id, cam) countbins = mergedqa['TASKS']['CHECK_FIBERS']['METRICS'][ 'GOOD_FIBERS'] y = y + countbins color = color + [ 'green' if idx == 1 else 'red' for idx in countbins ] status = status + [ 'GOOD' if idx == 1 else 'BAD' for idx in countbins ] else: y = y + 500 * [''] color = color + ['lightgray'] * 500 status = status + [''] * 500 cam_inst = cam_inst + [cam] * 500 data_model['goodfiber'] = y data_model['color'] = color data_model['status'] = status data_model['cam'] = cam_inst data_model['OBJ_TYPE'] = otype_tile data_model['ra'] = ra_tile data_model['dec'] = dec_tile source = ColumnDataSource(data=data_model) return source
def __init__(self, data): self.pipeline_name = 'Quick Look' self.data = data self.models = QLFModels() output_dir = os.path.join('exposures', self.data.get('night'), self.data.get('zfill')) output_full_dir = os.path.join(desi_spectro_redux, output_dir) # Remove old dir if os.path.isdir(output_full_dir): shutil.rmtree(output_full_dir) # Make output dir os.makedirs(output_full_dir) self.data['output_dir'] = output_dir
def qa_tests(self, process_id): qa_tests = list() for job in QLFModels().get_jobs_by_process_id(process_id): try: process = job.process exposure = process.exposure lm = LoadMetrics(process_id, job.camera_id, process.exposure_id, exposure.night) qa_tests.append({job.camera_id: lm.load_qa_tests()}) except: mainlogger.error('qa_tests error camera %s' % job.camera) return qa_tests
def load_scalar_metrics(self, process_id, cam): scalar_metrics = dict() try: process = QLFModels().get_process_by_process_id(process_id) exposure = process.exposure lm = LoadMetrics(process_id, cam, process.exposure_id, exposure.night) scalar_metrics['metrics'] = lm.metrics scalar_metrics['tests'] = lm.tests except: print('load_scalar_metrics error') return scalar_metrics
class QLFProcess(object): """ Class responsible for managing Quick Look pipeline process. """ def __init__(self, data): self.pipeline_name = 'Quick Look' self.data = data self.models = QLFModels() output_dir = os.path.join('exposures', self.data.get('night'), self.data.get('zfill')) output_full_dir = os.path.join(desi_spectro_redux, output_dir) # Remove old dir if os.path.isdir(output_full_dir): shutil.rmtree(output_full_dir) # Make output dir os.makedirs(output_full_dir) self.data['output_dir'] = output_dir def start_process(self): """ Start pipeline. """ self.data['start'] = datetime.datetime.now().replace(microsecond=0) # create process in database and obtain the process id process = self.models.insert_process(self.data, self.pipeline_name) self.data['process_id'] = process.id self.data['status'] = process.status # TODO: ingest configuration file used, this should be done by process # self.models.insert_config(process.id) pipe_logger.info('...{}'.format('\n' * 20)) pipe_logger.info('Process ID {}'.format(process.id)) pipe_logger.info('ExpID {} started.'.format(self.data.get('expid'))) return process.id def finish_process(self): """ Finish pipeline. """ self.data['end'] = datetime.datetime.now().replace(microsecond=0) self.data['duration'] = self.data.get('end') - self.data.get('start') pipe_logger.info("ExpID {} ended (runtime: {}).".format( self.data.get('expid'), str(self.data.get('duration')))) proc = Thread(target=self.ingest_parallel_qas) proc.start()
def __init__(self): super().__init__() self.running = Event() self.exit = Event() self.dos_monitor = DOSmonitor() self.last_night = str() self.current_exposure = None exposure = QLFModels().get_last_exposure() if exposure: self.last_night = exposure.night
def __init__(self, process_id, cam, exp=None, night=None, flavor=None): self.cam = cam self.exp = exp self.night = night self.process_id = process_id # This is True if the pipeline didn't generate some json file self.error = dict(zip(self.qa_name, ['False']*len(self.qa_name))) self.models = QLFModels() stages = self.load_flavors() self.steps_dic = dict() self.qas_status = dict() self.alert_keys = dict() for step in self.flavor_stages[flavor]['step_list']: self.steps_dic[step['name']] = list() self.qas_status[step['name']] = list() for qa in step['qa_list']: self.steps_dic[step['name']].append(qa['name']) self.qas_status[step['name']].append('None') self.alert_keys[qa['name']] = qa['status_key']
def stop(self): if self.is_running(): QLFModels().abort_current_process() if self.monitor and self.monitor.is_alive(): logger.debug("Stop pid %i" % self.monitor.pid) pid = self.monitor.pid self.monitor.shutdown() kill_proc_tree(pid, include_parent=False) del self.monitor gc.collect self.monitor = None else: logger.debug("Monitor is not initialized.")
def data_source(self, fmap): """ Creating data source for plots """ data_model = { 'x': [], 'w': [], 'cam': [], 'OBJ_TYPE': [], 'ra': [], 'dec': [], } process_id = self.selected_process_id joblist = [ entry.camera.camera for entry in Job.objects.filter(process_id=process_id) ] ra_tile = fmap.fiber_ra dec_tile = fmap.fiber_dec otype_tile = fmap.objtype y = [] w = [] cam_inst = [] for spec in list(range(10)): cam = self.selected_arm + str(spec) if cam in joblist: mergedqa = QLFModels().get_output(self.selected_process_id, cam) xwsig = mergedqa['TASKS']['CHECK_CCDs']['METRICS'][ 'XWSIGMA_FIB'] y = y + xwsig[0] w = w + xwsig[1] else: y = y + 500 * [np.nan] w = w + 500 * [np.nan] cam_inst = cam_inst + [cam] * 500 data_model['x'] = y data_model['w'] = w data_model['cam'] = cam_inst data_model['OBJ_TYPE'] = otype_tile data_model['ra'] = ra_tile data_model['dec'] = dec_tile source = ColumnDataSource(data=data_model) return source
def load_qa(self): cam = self.selected_arm + str(self.selected_spectrograph) mergedqa = QLFModels().get_output(self.selected_process_id, cam) gen_info = mergedqa['GENERAL_INFO'] check_spectra = mergedqa['TASKS']['CHECK_SPECTRA'] std_fiberid = mergedqa['GENERAL_INFO']['STAR_FIBERID'] nrg = check_spectra['PARAMS']['DELTAMAG_TGT_NORMAL_RANGE'] wrg = check_spectra['PARAMS']['DELTAMAG_TGT_WARN_RANGE'] fiber_mag = np.array(mergedqa['GENERAL_INFO']['FIBER_MAGS']) if 'b' in cam: arm_id = 0 elif 'r' in cam: arm_id = 1 else: arm_id = 2 fiber_mag = fiber_mag[arm_id] #.flatten() tooltip = """ <div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">INTEG: </span> <span style="font-size: 1vw; color: #515151;">@integ</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">FIBER ID: </span> <span style="font-size: 1vw; color: #515151;">@x</span> </div> </div> """ source = ColumnDataSource( data={ 'integ': [i if i > -999 else np.nan for i in fiber_mag], 'x': np.arange(len(fiber_mag)), }) print(len(fiber_mag)) yrange = [0, 1.1 * max(fiber_mag)] fiber_hist = Plot2d( yrange, x_label="Fibers", y_label="Integral (counts)", tooltip=tooltip, title="", width=600, height=400, yscale="auto", hover_mode="vline", ).vbar( source, y="integ", ) info_col = Title().write_description('integ') # Reading obj_type objlist = mergedqa["TASKS"]["CHECK_SPECTRA"]["METRICS"]["OBJLIST"] if 'SKY' in objlist: objlist.remove('SKY') # Prepare tables current_exposures = check_spectra['METRICS']['DELTAMAG_TGT'] program = gen_info['PROGRAM'].upper() reference_exposures = check_spectra['PARAMS']['DELTAMAG_TGT_' + program + '_REF'] keynames = ["DELTAMAG_TGT" + " ({})".format(i) for i in objlist] table = Table().single_table(keynames, current_exposures, reference_exposures, nrg, wrg) layout = column(info_col, Div(), table, Div(), column(fiber_hist, sizing_mode='scale_both', css_classes=["main-one"]), css_classes=["display-grid"]) return file_html(layout, CDN, "INTEG")
def load_qa(self): cam = self.selected_arm+str(self.selected_spectrograph) mergedqa = QLFModels().get_output(self.selected_process_id, cam) check_spectra = mergedqa['TASKS']['CHECK_SPECTRA'] gen_info = mergedqa['GENERAL_INFO'] ra = gen_info['RA'] dec = gen_info['DEC'] nrg = check_spectra['PARAMS']['PEAKCOUNT_NORMAL_RANGE'] wrg = check_spectra['PARAMS']['PEAKCOUNT_WARN_RANGE'] current_exposures = [check_spectra['METRICS']['PEAKCOUNT']] program = gen_info['PROGRAM'].upper() reference_exposures = check_spectra['PARAMS']['PEAKCOUNT_' + program + '_REF'] obj_type = sort_obj(gen_info) my_palette = get_palette("RdYlBu_r") peak_tooltip = """ <div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">PEAKCOUNT: </span> <span style="font-size: 1vw; color: #515151">@peakcount_fib</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">RA: </span> <span style="font-size: 1vw; color: #515151;">@x1</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">DEC: </span> <span style="font-size: 1vw; color: #515151;">@y1</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">Obj Type: </span> <span style="font-size: 1vw; color: #515151;">@OBJ_TYPE</span> </div> </div> """ url = "http://legacysurvey.org/viewer?ra=@ra&dec=@dec&zoom=16&layer=decals-dr5" qlf_fiberid = np.arange(0, 500) peak_hover = HoverTool(tooltips=peak_tooltip) peakcount_fib = check_spectra['METRICS']['PEAKCOUNT_FIB'] source = ColumnDataSource(data={ 'x1': ra, 'y1': dec, 'peakcount_fib': peakcount_fib, 'delta_peakcount_fib': np.array(peakcount_fib)-reference_exposures, 'QLF_FIBERID': qlf_fiberid, 'OBJ_TYPE': obj_type, }) low, high = wrg mapper = LinearColorMapper(palette=my_palette, low=low, #0.98*np.min(peakcount_fib), high=high, #1.02*np.max(peakcount_fib)) nan_color='darkgrey') radius = 0.0165 radius_hover = 0.02 # centralize wedges in plots: ra_center=0.5*(max(ra)+min(ra)) dec_center=0.5*(max(dec)+min(dec)) xrange_wedge = Range1d(start=ra_center + .95, end=ra_center-.95) yrange_wedge = Range1d(start=dec_center+.82, end=dec_center-.82) # axes limit xmin, xmax = [min(gen_info['RA'][:]), max(gen_info['RA'][:])] ymin, ymax = [min(gen_info['DEC'][:]), max(gen_info['DEC'][:])] xfac, yfac = [(xmax-xmin)*0.06, (ymax-ymin)*0.06] left, right = xmin - xfac, xmax+xfac bottom, top = ymin-yfac, ymax+yfac wedge_plot = Plot2d( x_range=xrange_wedge, y_range=yrange_wedge, x_label="RA", y_label="DEC", tooltip=peak_tooltip, title="PEAKCOUNT", width=500, height=380, ).wedge( source, x='x1', y='y1', field='delta_peakcount_fib', mapper=mapper, ).plot info_col = Title().write_description('skypeak') # ================================ # histogram hist_tooltip = """ <div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">Frequency: </span> <span style="font-size: 1vw; color: #515151">@hist</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">Peakcount: </span> <span style="font-size: 1vw; color: #515151;">[@left, @right]</span> </div> </div> """ hist, edges = np.histogram(peakcount_fib, bins="sqrt") source_hist = ColumnDataSource(data={ 'hist': hist, 'histplusone': hist+1, 'bottom': [0] * len(hist), 'bottomplusone': [1]*len(hist), 'left': edges[:-1], 'right': edges[1:] }) p_hist = Plot2d( y_range=(1, 11**(int(np.log10(max(hist)))+1)), x_label='PEAKCOUNT', y_label='Frequency + 1', tooltip=hist_tooltip, title="", width=550, height=300, yscale="log", hover_mode="vline", ).quad( source_hist, top='histplusone', bottom='bottomplusone', line_width=1, ) # Prepare tables keynames = ["PEAKCOUNT" for i in range(len(current_exposures))] metric = Table().reference_table(keynames, current_exposures, reference_exposures) alert = Table().alert_table(nrg, wrg) layout = column(info_col, Div(), metric, alert, column(wedge_plot, sizing_mode='scale_both'), column(p_hist, sizing_mode='scale_both'), css_classes=["display-grid"]) return file_html(layout, CDN, "SKYPEAK")
def wedge_plot(self, wedge_arm, fmap, common_source=None, sigma_kind='x'): ra_center = fmap.exposure.telra dec_center = fmap.exposure.teldec fiber_tooltip = """ <div> <div> <span style="font-size: 12px; font-weight: bold; color: #303030;">SIGMA: </span> <span style="font-size: 13px; color: #515151">@y</span> </div> <div> <span style="font-size: 12px; font-weight: bold; color: #303030;">RA: </span> <span style="font-size: 13px; color: #515151;">@ra</span> </div> <div> <span style="font-size: 12px; font-weight: bold; color: #303030;">DEC: </span> <span style="font-size: 13px; color: #515151;">@dec</span> </div> <div> <span style="font-size: 12px; font-weight: bold; color: #303030;">Obj Type: </span> <span style="font-size: 13px; color: #515151;">@OBJ_TYPE</span> </div> <div> <span style="font-size: 12px; font-weight: bold; color: #303030;">CAM: </span> <span style="font-size: 13px; color: #515151;">@cam</span> </div> """ fiber_tooltip = fiber_tooltip.replace('SIGMA:', '%sSIGMA:' % sigma_kind.upper()) hover = HoverTool(tooltips=fiber_tooltip) my_palette = get_palette("bwr") source = common_source process_id = self.selected_process_id joblist = [ entry.camera.camera for entry in Job.objects.filter(process_id=process_id) ] if len(joblist) > 0: cam = joblist[0] mergedqa = QLFModels().get_output(self.selected_process_id, cam) warn_range = mergedqa['TASKS']['CHECK_CCDs']['PARAMS'][ 'XWSIGMA_WARN_RANGE'] arg_kind = {'x': 0, 'w': 1} refvalue = mergedqa['TASKS']['CHECK_CCDs']['PARAMS'][ 'XWSIGMA_REF'][arg_kind[sigma_kind]] rng_warn_min, rng_warn_max = warn_range[0] + \ refvalue, warn_range[1] + refvalue sigma = source.data['{}'.format(sigma_kind)] rng_min, rng_max = np.nanmin(sigma), np.nanmax(sigma) rng = rng_max - rng_min if np.isnan(rng_min) or np.isnan(rng_max): fill_color = 'lightgray' else: mapper = LinearColorMapper(palette=my_palette, nan_color='lightgray', low=rng_warn_min, high=rng_warn_max) fill_color = {'field': '%s' % (sigma_kind), 'transform': mapper} radius = 0.017 radius_hover = 0.018 xrange = Range1d(start=ra_center + 2, end=ra_center - 2) yrange = Range1d(start=dec_center + 1.8, end=dec_center - 1.8) p = Figure(title='FOCUS %s (ARM %s)' % (sigma_kind.upper(), wedge_arm), x_axis_label='RA', y_axis_label='DEC', plot_width=600, plot_height=600, tools=[ hover, "box_zoom,pan,wheel_zoom,reset,lasso_select,crosshair" ], active_drag="box_zoom", x_range=xrange, y_range=yrange) p.title.align = 'center' p.circle('ra', 'dec', source=source, name="data", radius=radius, fill_color=fill_color, line_color='black', line_width=0.4, hover_line_color='red') p.circle('ra', 'dec', source=source, name="data", radius=radius_hover, hover_fill_color=fill_color, fill_color=None, line_color=None, line_width=3, hover_line_color='orange') if 'mapper' in locals(): cbar = Figure( height=p.plot_height, width=120, toolbar_location=None, min_border=0, outline_line_color=None, ) color_bar = ColorBar(color_mapper=mapper, label_standoff=14, major_label_text_font_style="bold", padding=26, major_label_text_align='right', major_label_text_font_size="10pt", location=(0, 0)) cbar.title.align = 'center' cbar.title.text_font_size = '10pt' cbar.add_layout(color_bar, 'left') p_list = [cbar, p] else: p_list = [p] return p_list
def data_source_arm(self, fmap, arm): """ Creating data source for plots """ data_model = { 'resids': [], 'hover': [], 'OBJ_TYPE': [], 'ra': [], 'dec': [], 'cam': [], } process_id = self.selected_process_id process = Process.objects.get(pk=process_id) joblist = [entry.camera.camera for entry in Job.objects.filter( process_id=process_id)] ra_tile = fmap.fiber_ra dec_tile = fmap.fiber_dec otype_tile = fmap.objtype objlist = sorted(set(otype_tile)) if 'SKY' in objlist: objlist.remove('SKY') ra_snr = [] dec_snr = [] resids_snr = [] ot_snr = [] cam_snr = [] for spect in list(range(10)): cam = arm+str(spect) ra_petal = [] dec_petal = [] resids_petal = [] ot_petal = [] cam_petal = [] if cam in joblist: mergedqa = QLFModels().get_output( self.selected_process_id, cam) med_snr = np.array( mergedqa['TASKS']['CHECK_SPECTRA']['METRICS']["MEDIAN_SNR"]) # color values: resids_petal = ( mergedqa['TASKS']['CHECK_SPECTRA']['METRICS']['SNR_RESID']) ra_petal = [] dec_petal = [] fibers_snr = [] obj = np.arange(len(objlist)) for t in range(len(obj)): otype = list(objlist)[t] oid = np.where(np.array(list(objlist)) == otype)[0][0] if otype == 'STD': fibers = mergedqa['GENERAL_INFO']['STD_FIBERID'] elif otype == 'TGT': fibers = mergedqa['GENERAL_INFO']['STAR_FIBERID'] # fibers = mergedqa['GENERAL_INFO']['%s_FIBERID'%otype] fibers_snr = fibers_snr + fibers for j in range(len(fibers)): ras = mergedqa['GENERAL_INFO']['RA'][fibers[j]] decs = mergedqa['GENERAL_INFO']['DEC'][fibers[j]] ra_petal.append(ras) dec_petal.append(decs) ot_petal.append(otype) ra_snr = ra_snr+ra_petal dec_snr = dec_snr+dec_petal resids_snr = resids_snr + \ [i if i > -999 else np.nan for i in resids_petal] ot_snr = ot_snr + ot_petal cam_snr = cam_snr + [cam]*len(ra_petal) else: ra_snr = ra_snr + list(ra_tile[500*spect: 500*(spect + 1)]) dec_snr = dec_snr + list(dec_tile[500*spect: 500*(spect + 1)]) resids_snr = resids_snr + 500*[np.nan] ot_snr = ot_snr + list(otype_tile[500*spect: 500*(spect + 1)]) cam_snr = cam_snr + [str(cam)]*500 data_model['ra'] = ra_snr # ra_tile data_model['dec'] = dec_snr # dec_tile data_model['resids'] = resids_snr data_model['hover'] = ['%4.3f' % (ires) for ires in resids_snr] data_model['OBJ_TYPE'] = ot_snr data_model['cam'] = cam_snr source = ColumnDataSource(data=data_model) return source
from qlf_models import QLFModels from migrate_jobs_outputs import migrate_job_outputs from copy import copy import collections import numpy import datetime import random qlf_models = QLFModels() FLAVORS = {'science': 1.0, 'flat': 0.1, 'arc': 0.1} METRICS = { 'CHECK_SPECTRA': [ 'FIBER_MAG', 'SNR_MAG_TGT', 'SKYCONT_FIBER', 'PEAKCOUNT', 'PEAKCOUNT_FIB', 'DELTAMAG_TGT', 'DELTAMAG' ], 'CHECK_CCDs': [ 'BIAS_AMP', 'NOISE_AMP', 'XWSIGMA_FIB', 'NOISE_OVERSCAN_AMP', 'XWSIGMA_AMP', 'XWSIGMA' ] } def generate_processes_by_exposure(exposure): """ Creates a simulated process based on an exposure. Arguments: exposure {object} -- Exposure model Raises:
class LoadMetrics: """ Read values from the json's files and return an alert (NORMAL, WARN or ALARM) associated to a given metric. It also attributes a color for a wedge in the interface CORRECTING WXSIGMA Functions: ---------- find_qa_check(qa) update_status(qa) """ silent = 'False' # Defining a silent mode prfx = 'ql-' qa_name = ['countpix', 'getbias', 'getrms', 'xwsigma', 'countbins', 'integ', 'skycont', 'skypeak', 'snr'] def __init__(self, process_id, cam, exp=None, night=None, flavor=None): self.cam = cam self.exp = exp self.night = night self.process_id = process_id # This is True if the pipeline didn't generate some json file self.error = dict(zip(self.qa_name, ['False']*len(self.qa_name))) self.models = QLFModels() stages = self.load_flavors() self.steps_dic = dict() self.qas_status = dict() self.alert_keys = dict() for step in self.flavor_stages[flavor]['step_list']: self.steps_dic[step['name']] = list() self.qas_status[step['name']] = list() for qa in step['qa_list']: self.steps_dic[step['name']].append(qa['name']) self.qas_status[step['name']].append('None') self.alert_keys[qa['name']] = qa['status_key'] def load_flavors(self): flavors = ['science', 'arc', 'flat'] self.flavor_stages = dict() for flavor in flavors: flavor_path = os.path.join(qlf_root, "framework", "ql_mapping", "{}.json".format(flavor)) try: stages_json = open(flavor_path).read() self.flavor_stages[flavor] = json.loads(stages_json) except Exception as err: logger.error("flavor file not found {}".format(err)) def find_qa_check(self, qa): for check in self.steps_dic: if qa in self.steps_dic[check]: return check return None def load_merged_qa(self): data = None if self.process_id is not None: data = self.models.get_output(self.process_id, self.cam) return data def update_status(self, qa): index = -1 current_step = None qa_status = None for step in list(self.steps_dic): try: index = self.steps_dic[step].index(qa) current_step = step break except ValueError: continue data = self.load_merged_qa() try: alert_key = self.alert_keys[qa] check = self.find_qa_check(qa) qa_status = data["TASKS"][check]["METRICS"][alert_key] if current_step: self.qas_status[current_step][index] = qa_status except Exception as err: logger.warning('Failed to update camera status: {}'.format(err)) def get_merged_qa_status(self): for qa in list(self.alert_keys): self.update_status(qa)
def load_qa(self): cam = self.selected_arm + str(self.selected_spectrograph) mergedqa = QLFModels().get_output(self.selected_process_id, cam) gen_info = mergedqa['GENERAL_INFO'] ra = gen_info['RA'] dec = gen_info['DEC'] check_spectra = mergedqa['TASKS']['CHECK_SPECTRA'] skycont = check_spectra['METRICS'] sky = skycont['SKYCONT_FIBER'] skyfibers = gen_info['SKY_FIBERID'] nrg = check_spectra['PARAMS']['SKYCONT_NORMAL_RANGE'] wrg = check_spectra['PARAMS']['SKYCONT_WARN_RANGE'] current_exposures = [check_spectra['METRICS']['SKYCONT']] program = gen_info['PROGRAM'].upper() reference_exposures = check_spectra['PARAMS']['SKYCONT_' + program + '_REF'] ra_sky = [ra[i % 500] for i in skyfibers] dec_sky = [dec[i % 500] for i in skyfibers] my_palette = get_palette("RdYlBu_r") skc_tooltips = """ <div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">SKY CONT: </span> <span style="font-size: 1vw; color: #515151;">@skycont</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">RA: </span> <span style="font-size: 1vw; color: #515151;">@ra</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">DEC: </span> <span style="font-size: 1vw; color: #515151;">@dec</span> </div> </div> """ url = "http://legacysurvey.org/viewer?ra=@ra&dec=@dec&zoom=16&layer=decals-dr5" qlf_fiberid = np.arange(0, 500) # sky continuum per sky fiber averaged over two continuum regions, # 'n' is number of sky fibers ra_not, dec_not = [], [] fiber_mod = [] for fiber in skyfibers: fiber_mod.append(fiber % 500) for i in range(500): if i not in fiber_mod: ra_not.append(ra[i]) dec_not.append(dec[i]) source = ColumnDataSource( data={ 'skycont': sky, 'delta_skycont': np.array(sky) - reference_exposures, 'fiberid': skyfibers, 'ra': ra_sky, 'dec': dec_sky }) source_not = ColumnDataSource(data={ 'ra': ra_not, 'dec': dec_not, 'skycont': [''] * len(dec_not) }) low, high = wrg mapper = LinearColorMapper( palette=my_palette, low=low, #np.min(sky), high=high, #np.max(sky), nan_color='darkgrey') radius = 0.0165 # 0.013 radius_hover = 0.02 # 0.015 # centralize wedges in plots: ra_center = 0.5 * (max(ra) + min(ra)) dec_center = 0.5 * (max(dec) + min(dec)) xrange_wedge = Range1d(start=ra_center + .95, end=ra_center - .95) yrange_wedge = Range1d(start=dec_center + .82, end=dec_center - .82) wedge_plot = Plot2d(x_range=xrange_wedge, y_range=yrange_wedge, x_label="RA", y_label="DEC", tooltip=skc_tooltips, title="SKY_CONT", width=500, height=380, yscale="auto").wedge( source, x='ra', y='dec', field='delta_skycont', mapper=mapper, ).wedge( source_not, x='ra', y='dec', ).plot taptool = wedge_plot.select(type=TapTool) taptool.callback = OpenURL(url=url) # infos info_col = Title().write_description('skycont') # Prepare tables keynames = ["SKYCONT" for i in range(len(current_exposures))] table = Table().single_table(keynames, current_exposures, reference_exposures, nrg, wrg) layout = column(info_col, Div(), table, Div(), column(wedge_plot, sizing_mode='scale_both', css_classes=["main-one"]), css_classes=["display-grid"]) # End of Bokeh Block return file_html(layout, CDN, "SKYCONT")
def load_qa(self): cam = self.selected_arm + str(self.selected_spectrograph) mergedqa = QLFModels().get_output(self.selected_process_id, cam) # list of available object in petal objlist = mergedqa["TASKS"]["CHECK_SPECTRA"]["METRICS"]["OBJLIST"] if 'SKY' in objlist: objlist.remove('SKY') gen_info = mergedqa['GENERAL_INFO'] ra = gen_info['RA'] dec = gen_info['DEC'] check_spectra = mergedqa['TASKS']['CHECK_SPECTRA'] snr = check_spectra['METRICS'] nrg = check_spectra['PARAMS']['FIDSNR_TGT_NORMAL_RANGE'] wrg = check_spectra['PARAMS']['FIDSNR_TGT_WARN_RANGE'] # Object identification in fibers: obj_fiber = sort_obj(gen_info) qlf_obj = ['ELG', 'LRG', 'QSO', 'STAR'] avobj = ['STAR' if x == 'STD' else x for x in objlist] obj_idx = {} for o in qlf_obj: try: obj_idx.update({o: avobj.index(o)}) except: obj_idx.update({o: None}) try: exptime = gen_info['EXPTIME'] name_warn = '' except: exptime = 1000 name_warn = ' (exptime fixed)' # Sort objects for QLF: obj_idx = {} for o in qlf_obj: try: obj_idx.update({o: avobj.index(o)}) except: obj_idx.update({o: None}) def good_idx(mag, snr): # Filtering measurements with good SNR & good MAG # Treating None (inf and Nan already treated in db) mag_2 = np.array([-9998 if x is None else x for x in mag]) snr_2 = np.array([-9998 if x is None else x for x in snr]) idx = np.arange(len(snr_2)) # Filtering values with good mag AND snr return list(idx[(mag_2 > -999) & (snr_2 > 0)]) # Treating bad snr and mag mag_snr = {} for o in avobj: snr_ql, mag_ql = snr['SNR_MAG_TGT'][obj_idx[o]] idx = good_idx(mag_ql, snr_ql) x = [mag_ql[i] for i in idx] y = [snr_ql[i] for i in idx] mag_snr.update({o: [y, x]}) # Preparing xy_plot data: if obj_idx['ELG'] is not None: elg_snr = mag_snr['ELG'] if obj_idx['LRG'] is not None: lrg_snr = mag_snr['LRG'] if obj_idx['QSO'] is not None: qso_snr = mag_snr['QSO'] if obj_idx['STAR'] is not None: star_snr = mag_snr['STAR'] #lrg_snr = mag_snr['LRG'] def fit_func(xdata, coeff): """ astro fit """ r1 = 0.0 # read noise a, b = coeff x = np.linspace(min(xdata), max(xdata), 1000) Flux = 10**(-0.4 * (x - 22.5)) y = a * Flux * exptime / np.sqrt(a * Flux * exptime + b * exptime + r1**2) return x, y data_model = { 'x': [], 'y': [], 'y2': [], 'fiber_id': [], 'ra': [], 'dec': [], } elg = ColumnDataSource(data=data_model.copy()) lrg = ColumnDataSource(data=data_model.copy()) qso = ColumnDataSource(data=data_model.copy()) star = ColumnDataSource(data=data_model.copy()) data_fit = { 'x': [], 'y': [], 'y2': [], 'fiber_id': [], 'ra': [], 'dec': [] } elg_fit = ColumnDataSource(data=data_fit.copy()) lrg_fit = ColumnDataSource(data=data_fit.copy()) qso_fit = ColumnDataSource(data=data_fit.copy()) star_fit = ColumnDataSource(data=data_fit.copy()) if obj_idx['ELG'] is not None: elg.data['x'] = elg_snr[1] elg.data['y'] = np.array(elg_snr[0]) elg.data['y2'] = np.array(elg_snr[0])**2 elg.data['fiber_id'] = gen_info['ELG_FIBERID'] elg.data['ra'] = [ra[i % 500] for i in gen_info['ELG_FIBERID']] elg.data['dec'] = [dec[i % 500] for i in gen_info['ELG_FIBERID']] xfit, yfit = fit_func(elg_snr[1], snr['FITCOEFF_TGT'][obj_idx['ELG']]) elg_fit.data['x'] = xfit elg_fit.data['y'] = np.array(yfit) elg_fit.data['y2'] = np.array(yfit)**2 for key in ['fiber_id', 'ra', 'dec']: elg_fit.data[key] = [''] * len(yfit) if obj_idx['LRG'] is not None: lrg.data['x'] = lrg_snr[1] lrg.data['y'] = np.array(lrg_snr[0]) lrg.data['y2'] = np.array(lrg_snr[0])**2 lrg.data['fiber_id'] = gen_info['LRG_FIBERID'] lrg.data['ra'] = [ra[i % 500] for i in gen_info['LRG_FIBERID']] lrg.data['dec'] = [dec[i % 500] for i in gen_info['LRG_FIBERID']] xfit, yfit = fit_func(lrg_snr[1], snr['FITCOEFF_TGT'][obj_idx['LRG']]) lrg_fit.data['x'] = xfit lrg_fit.data['y'] = np.array(yfit) lrg_fit.data['y2'] = np.array(yfit)**2 for key in ['fiber_id', 'ra', 'dec']: lrg_fit.data[key] = [''] * len(yfit) if obj_idx['QSO'] is not None: qso.data['x'] = qso_snr[1] qso.data['y'] = np.array(qso_snr[0]) qso.data['y2'] = np.array(qso_snr[0])**2 qso.data['fiber_id'] = gen_info['QSO_FIBERID'] qso.data['ra'] = [ra[i % 500] for i in gen_info['QSO_FIBERID']] qso.data['dec'] = [dec[i % 500] for i in gen_info['QSO_FIBERID']] xfit, yfit = fit_func(qso_snr[1], snr['FITCOEFF_TGT'][obj_idx['QSO']]) qso_fit.data['x'] = xfit qso_fit.data['y'] = np.array(yfit) qso_fit.data['y2'] = np.array(yfit)**2 for key in ['fiber_id', 'ra', 'dec']: qso_fit.data[key] = [''] * len(yfit) if obj_idx['STAR'] is not None: star.data['x'] = star_snr[1] star.data['y'] = np.array(star_snr[0]) star.data['y2'] = np.array(star_snr[0])**2 star.data['fiber_id'] = gen_info['STAR_FIBERID'] star.data['ra'] = [ra[i % 500] for i in gen_info['STAR_FIBERID']] star.data['dec'] = [dec[i % 500] for i in gen_info['STAR_FIBERID']] xfit, yfit = fit_func(star_snr[1], snr['FITCOEFF_TGT'][obj_idx['STAR']]) star_fit.data['x'] = xfit star_fit.data['y'] = np.array(yfit) star_fit.data['y2'] = np.array(yfit)**2 for key in ['fiber_id', 'ra', 'dec']: star_fit.data[key] = [''] * len(yfit) html_tooltip = """ <div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">SNR: </span> <span style="font-size: 1vw; color: #515151;">@y</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">DECAM_{}: </span> <span style="font-size: 1vw; color: #515151;">@x</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">Fiber ID: </span> <span style="font-size: 1vw; color: #515151;">@fiber_id</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">RA: </span> <span style="font-size: 1vw; color: #515151;">@ra</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">Dec: </span> <span style="font-size: 1vw; color: #515151">@dec</span> </div> </div> """.format(str(self.selected_arm).upper()) url = "http://legacysurvey.org/viewer?ra=@ra&dec=@dec&zoom=16&layer=decals-dr5" elg_plot = Plot2d( x_label="DECAM_{}".format(str(self.selected_arm).upper()), y_label="MEDIAN SNR^2", tooltip=html_tooltip, title="ELG", width=500, height=380, yscale="log", ).line( source=elg_fit, y='y2', ).circle( source=elg, size=8, y='y2', fill_color="blue", ).plot taptool = elg_plot.select(type=TapTool) taptool.callback = OpenURL(url=url) lrg_plot = Plot2d(x_label="DECAM_{}".format( str(self.selected_arm).upper()), y_label="MEDIAN SNR^2", tooltip=html_tooltip, title="LRG", width=500, height=380, yscale="log").line( source=lrg_fit, y='y2', ).circle( source=lrg, size=8, y='y2', fill_color="red", ).plot taptool = lrg_plot.select(type=TapTool) taptool.callback = OpenURL(url=url) qso_plot = Plot2d(x_label="DECAM_{}".format( str(self.selected_arm).upper()), y_label="MEDIAN SNR^2", tooltip=html_tooltip, title="QSO", width=500, height=380, yscale="log").line( source=qso_fit, y='y2', ).circle( source=qso, size=8, y='y2', fill_color="green", ).plot taptool = qso_plot.select(type=TapTool) taptool.callback = OpenURL(url=url) star_plot = Plot2d( x_label="DECAM_{}".format(str(self.selected_arm).upper()), y_label="MEDIAN SNR^2", tooltip=html_tooltip, title="STAR", width=500, height=380, yscale="log", ).line(source=star_fit, y='y2').circle( source=star, size=8, y='y2', fill_color="yellow", ).plot taptool = star_plot.select(type=TapTool) taptool.callback = OpenURL(url=url) # infos info_col = Title().write_description('snr') # ----------------- # WEDGES snr_tooltip = """ <div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">Resid: </span> <span style="font-size: 1vw; color: #515151">@resid_snr</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">Obj Type: </span> <span style="font-size: 1vw; color: #515151;">@OBJ_TYPE</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">RA: </span> <span style="font-size: 1vw; color: #515151;">@x1</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">DEC: </span> <span style="font-size: 1vw; color: #515151;">@y1</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">FIBER ID: </span> <span style="font-size: 1vw; color: #515151;">@QLF_FIBERID</span> </div> </div> """ median = snr['MEDIAN_SNR'] resid = snr['SNR_RESID'] qlf_fiberid = range(0, 500) my_palette = get_palette('bwr') fibersnr_tgt = [] for i in avobj: fibersnr_tgt.append(gen_info[i + '_FIBERID']) fibersnr = [] for i in list(range(len(fibersnr_tgt))): fibersnr = fibersnr + fibersnr_tgt[i] source = ColumnDataSource( data={ 'x1': [ra[i % 500] for i in fibersnr], 'y1': [dec[i % 500] for i in fibersnr], 'resid_snr': resid, 'QLF_FIBERID': fibersnr, 'OBJ_TYPE': [obj_fiber[i % 500] for i in fibersnr], 'median': median }) ra_not = [] dec_not = [] obj_not = [] fiber_not = [] fiber_mod = [] for fiber in fibersnr: fiber_mod.append(fiber % 500) for i in range(500): if i not in fiber_mod: ra_not.append(ra[i]) dec_not.append(dec[i]) fiber_not.append(i) obj_not.append(obj_fiber[i]) source_not = ColumnDataSource( data={ 'x1': ra_not, 'y1': dec_not, 'resid_snr': [''] * len(dec_not), 'QLF_FIBERID': fiber_not, 'OBJ_TYPE': obj_not }) rmax, rmin = np.nanmax(resid), np.nanmin(resid) if np.isnan(rmax) or np.isnan(rmin): fill_color = 'lightgray' else: dy = (rmax - rmin) * 0.1 mapper = LinearColorMapper(palette=my_palette, nan_color='darkgray', low=rmin - dy, high=rmax + dy) fill_color = {'field': 'resid_snr', 'transform': mapper} radius = 0.0165 radius_hover = 0.02 # centralize wedges in plots: ra_center = 0.5 * (max(ra) + min(ra)) dec_center = 0.5 * (max(dec) + min(dec)) xrange_wedge = Range1d(start=ra_center + .95, end=ra_center - .95) yrange_wedge = Range1d(start=dec_center + .82, end=dec_center - .82) # axes limit xmin, xmax = [min(ra[:]), max(ra[:])] ymin, ymax = [min(dec[:]), max(dec[:])] xfac, yfac = [(xmax - xmin) * 0.06, (ymax - ymin) * 0.06] left, right = xmin - xfac, xmax + xfac bottom, top = ymin - yfac, ymax + yfac # WEDGE RESIDUAL plots wedge_plot = Plot2d(x_range=xrange_wedge, y_range=yrange_wedge, x_label="RA", y_label="DEC", tooltip=snr_tooltip, title='Residual SNR' + name_warn, width=500, height=380, yscale="auto").wedge( source, x='x1', y='y1', field='resid_snr', mapper=mapper, ).wedge( source_not, x='x1', y='y1', ).plot taptool = wedge_plot.select(type=TapTool) taptool.callback = OpenURL(url=url) # ------------------- # Median plot median_tooltip = """ <div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">MEDIAN: </span> <span style="font-size: 1vw; color: #515151">@median</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">Resid: </span> <span style="font-size: 1vw; color: #515151">@resid_snr</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">Obj Type: </span> <span style="font-size: 1vw; color: #515151;">@OBJ_TYPE</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">RA: </span> <span style="font-size: 1vw; color: #515151;">@x1</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">DEC: </span> <span style="font-size: 1vw; color: #515151;">@y1</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">FIBER ID: </span> <span style="font-size: 1vw; color: #515151;">@QLF_FIBERID</span> </div> </div> """ mediam_plot = Plot2d( x_label="Fiber", y_label='Median SNR', tooltip=median_tooltip, title="", width=600, height=400, yscale="auto", hover_mode="vline", ).vbar( source, y="median", x="QLF_FIBERID", line_width=0.4, ) # Prepare tables current_exposures = check_spectra['METRICS']['FIDSNR_TGT'] program = gen_info['PROGRAM'].upper() reference_exposures = check_spectra['PARAMS']['FIDSNR_TGT_' + program + '_REF'] keynames = ["FIDSNR_TGT" + " ({})".format(i) for i in objlist] metric = Table().reference_table(keynames, current_exposures, reference_exposures) alert = Table().alert_table(nrg, wrg) layout = column(info_col, Div(), metric, alert, column(elg_plot, sizing_mode='scale_both'), column(lrg_plot, sizing_mode='scale_both'), column(qso_plot, sizing_mode='scale_both'), column(star_plot, sizing_mode='scale_both'), column(mediam_plot, sizing_mode='scale_both'), column(wedge_plot, sizing_mode='scale_both'), css_classes=["display-grid"]) return file_html(layout, CDN, "MEDIAN SNR")
def __init__(self, data): self.pipeline_name = 'Quick Look' self.data = data self.models = QLFModels()
def data_source(self, fmap): """ Creating data source for plots """ data_model = { 'x_b': [], 'x_r': [], 'x_z': [], 'OBJ_TYPE': [], 'ra': [], 'dec': [], 'cam_b': [], 'cam_r': [], 'cam_z': [] } process_id = self.selected_process_id joblist = [entry.camera.camera for entry in Job.objects.filter( process_id=process_id)] ra_tile = fmap.fiber_ra dec_tile = fmap.fiber_dec otype_tile = fmap.objtype objlist = sorted(set(otype_tile)) if 'SKY' in objlist: objlist.remove('SKY') fibaux = [] y = [] cam_inst = [] ra_all = [] dec_all = [] for spect in list(range(1)): cam = self.selected_arm+str(spect) if cam in joblist: mergedqa = QLFModels().get_output( self.selected_process_id, cam) # Assign available objects objtype = otype_tile[500*spect: 500*(spect + 1)] med_snr = np.array( mergedqa['TASKS']['CHECK_SPECTRA']['METRICS']["MEDIAN_SNR"]) resids = mergedqa['TASKS']['CHECK_SPECTRA']['METRICS']['SNR_RESID'] obj = np.arange(len(objlist)) rayes = [] decyes = [] for t in range(len(obj)): otype = list(objlist)[t] oid = np.where(np.array(list(objlist)) == otype)[0][0] if otype == 'STD': fibers = mergedqa['GENERAL_INFO']['STAR_FIBERID'] else: fibers = mergedqa['GENERAL_INFO']['%s_FIBERID' % otype] fibaux = fibaux + [500*spect + i for i in fibers] for i in range(len(fibers)): ras = mergedqa['GENERAL_INFO']['RA'][fibers[i]] decs = mergedqa['GENERAL_INFO']['DEC'][fibers[i]] rayes.append(ras) decyes.append(decs) ra_all = ra_all + rayes dec_all = dec_all + decyes nanresids = [i if i > -9999. else np.nan for i in resids] y = y + nanresids else: y = y + 500*[np.nan] ra_all = ra_all + \ [i for i in ra_tile[500*spect:500*(spect + 1)]] dec_all = dec_all + \ [i for i in dec_tile[500*spect:500*(spect + 1)]] fibaux = fibaux + list(range(500*spect, 500*(spect+1))) cam_inst = cam_inst + [cam]*500 data_model['x_' + cam[0]] = y data_model['cam_'+cam[0]] = cam_inst data_model['OBJ_TYPE'] = [otype_tile[ii] for ii in fibaux] data_model['ra'] = ra_all data_model['dec'] = dec_all source = ColumnDataSource(data=data_model) return source
def load_qa(self): cam = self.selected_arm + str(self.selected_spectrograph) mergedqa = QLFModels().get_output(self.selected_process_id, cam) gen_info = mergedqa['GENERAL_INFO'] flavor = mergedqa['FLAVOR'] if flavor == "science": ra = gen_info['RA'] dec = gen_info['DEC'] check_ccds = mergedqa['TASKS']['CHECK_CCDs'] xwsigma = check_ccds['METRICS']['XWSIGMA'] xw_amp = check_ccds['METRICS']['XWSIGMA_AMP'] xw_fib = check_ccds['METRICS']['XWSIGMA_FIB'] nrg = check_ccds['PARAMS']['XWSIGMA_NORMAL_RANGE'] wrg = check_ccds['PARAMS']['XWSIGMA_WARN_RANGE'] obj_type = sort_obj(gen_info) # [""]*500 if mergedqa['FLAVOR'].upper() == 'SCIENCE': program = mergedqa['GENERAL_INFO']['PROGRAM'].upper() program_prefix = '_' + program else: program_prefix = '' xw_ref = check_ccds['PARAMS']['XWSIGMA' + program_prefix + '_REF'] xsigma = xw_fib[0] wsigma = xw_fib[1] delta_rg = wrg[1] - wrg[0] hist_rg = (wrg[0] - 0.1 * delta_rg, wrg[1] + 0.1 * delta_rg) my_palette = get_palette("RdYlBu_r") xfiber = np.arange(len(xsigma)) wfiber = np.arange(len(wsigma)) if mergedqa['FLAVOR'].upper() != 'SCIENCE': ra = np.full(500, np.nan) dec = np.full(500, np.nan) source = ColumnDataSource( data={ 'x1': ra, 'y1': dec, 'xsigma': xsigma, 'wsigma': wsigma, 'delta_xsigma': np.array(xsigma) - xw_ref[0], 'delta_wsigma': np.array(wsigma) - xw_ref[1], 'xref': [xw_ref[0]] * len(xsigma), 'wref': [xw_ref[1]] * len(xsigma), 'xfiber': xfiber, 'wfiber': wfiber, 'OBJ_TYPE': obj_type, 'left': np.arange(0, 500) - 0.4, 'right': np.arange(0, 500) + 0.4, 'bottom': [0] * 500 }) xsigma_tooltip = """ <div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">XSigma: </span> <span style="font-size: 1vw; color: #515151">@xsigma</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">Reference: </span> <span style="font-size: 1vw; color: #515151">@xref</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">Obj Type: </span> <span style="font-size: 1vw; color: #515151;">@OBJ_TYPE</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">RA: </span> <span style="font-size: 1vw; color: #515151;">@x1</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">DEC: </span> <span style="font-size: 1vw; color: #515151;">@y1</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">FIBER ID: </span> <span style="font-size: 1vw; color: #515151;">@xfiber</span> </div> </div> """ wsigma_tooltip = """ <div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">WSigma: </span> <span style="font-size: 1vw; color: #515151">@wsigma</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">Reference: </span> <span style="font-size: 1vw; color: #515151">@wref</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">Obj Type: </span> <span style="font-size: 1vw; color: #515151;">@OBJ_TYPE</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">RA: </span> <span style="font-size: 1vw; color: #515151;">@x1</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">DEC: </span> <span style="font-size: 1vw; color: #515151;">@y1</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">FIBER ID: </span> <span style="font-size: 1vw; color: #515151;">@wfiber</span> </div> </div> """ url = "http://legacysurvey.org/viewer?ra=@ra&dec=@dec&zoom=16&layer=decals-dr5" # determining the position of selected cam fibers: obj_type = sort_obj(gen_info) # --------------------------------- if flavor == "science": # axes limit xmin, xmax = [min(ra[:]), max(ra[:])] ymin, ymax = [min(dec[:]), max(dec[:])] xfac, yfac = [(xmax - xmin) * 0.06, (ymax - ymin) * 0.06] left, right = xmin - xfac, xmax + xfac bottom, top = ymin - yfac, ymax + yfac low, high = wrg xmapper = LinearColorMapper(palette=my_palette, low=low, high=high, nan_color="darkgrey") wmapper = LinearColorMapper(palette=my_palette, low=low, high=high, nan_color="darkgrey") # ============ # XSIGMA WEDGE radius = 0.0165 radius_hover = 0.02 # centralize wedges in plots: ra_center = 0.5 * (max(ra) + min(ra)) dec_center = 0.5 * (max(dec) + min(dec)) xrange_wedge = Range1d(start=ra_center + .95, end=ra_center - .95) yrange_wedge = Range1d(start=dec_center + .82, end=dec_center - .82) wedge_plot_x = Plot2d(x_range=xrange_wedge, y_range=yrange_wedge, x_label="RA", y_label="DEC", tooltip=xsigma_tooltip, title="XSIGMA", width=500, height=380, yscale="auto").wedge( source, x='x1', y='y1', field='delta_xsigma', mapper=xmapper, colorbar_title='xsigma').plot taptool = wedge_plot_x.select(type=TapTool) taptool.callback = OpenURL(url=url) # ============ # WSIGMA WEDGE wedge_plot_w = Plot2d(x_range=xrange_wedge, y_range=yrange_wedge, x_label="RA", y_label="DEC", tooltip=wsigma_tooltip, title="WSIGMA", width=500, height=380, yscale="auto").wedge( source, x='x1', y='y1', field='delta_wsigma', mapper=wmapper, colorbar_title='wsigma').plot taptool = wedge_plot_w.select(type=TapTool) taptool.callback = OpenURL(url=url) # ================================ # Stat histogram # x_fiber_hist d_yplt = (max(xsigma) - min(xsigma)) * 0.1 yrange = [0, max(xsigma) + d_yplt] xhist = Plot2d( yrange, x_label="Fiber number", y_label="X std dev (number of pixels)", tooltip=xsigma_tooltip, title="", width=600, height=300, yscale="auto", hover_mode="vline", ).quad( source, top='xsigma', ) # w_fiber_hist d_yplt = (max(wsigma) - min(wsigma)) * 0.1 yrange = [0, max(wsigma) + d_yplt] whist = Plot2d( yrange, x_label="Fiber number", y_label="W std dev (number of pixels)", tooltip=xsigma_tooltip, title="", width=600, height=300, yscale="auto", hover_mode="vline", ).quad( source, top='wsigma', ) # ================================ # Stat histogram def histpar(yscale, hist): if yscale == 'log': ylabel = "Frequency + 1" yrange = (1, 11**(int(np.log10(max(hist))) + 1)) bottomval = 'bottomplusone' histval = 'histplusone' else: ylabel = "Frequency" yrange = (0.0 * max(hist), 1.1 * max(hist)) bottomval = 'bottom' histval = 'hist' return [ylabel, yrange, bottomval, histval] xhistlabel = "XSIGMA" yscale = "auto" hist_tooltip_x = """ <div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">Frequency: </span> <span style="font-size: 1vw; color: #515151">@hist</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">XSIGMA: </span> <span style="font-size: 1vw; color: #515151;">[@left, @right]</span> </div> </div> """ hist, edges = np.histogram(xsigma, 'sqrt') source_hist = ColumnDataSource( data={ 'hist': hist, 'histplusone': hist + 1, 'bottom': [0] * len(hist), 'bottomplusone': [1] * len(hist), 'left': edges[:-1], 'right': edges[1:] }) ylabel, yrange, bottomval, histval = histpar(yscale, hist) p_hist_x = Plot2d( x_label=xhistlabel, y_label=ylabel, tooltip=hist_tooltip_x, title="", width=600, height=300, yscale="auto", y_range=yrange, x_range=(hist_rg[0] + xw_ref[0], hist_rg[1] + xw_ref[0]), hover_mode="vline", ).quad( source_hist, top=histval, bottom=bottomval, line_width=0.4, ) # Histogram 2 xhistlabel = "WSIGMA" hist_tooltip_w = """ <div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">Frequency: </span> <span style="font-size: 1vw; color: #515151">@hist</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">WSIGMA: </span> <span style="font-size: 1vw; color: #515151;">[@left, @right]</span> </div> </div> """ hist, edges = np.histogram(wsigma, 'sqrt') source_hist = ColumnDataSource( data={ 'hist': hist, 'histplusone': hist + 1, 'bottom': [0] * len(hist), 'bottomplusone': [1] * len(hist), 'left': edges[:-1], 'right': edges[1:] }) ylabel, yrange, bottomval, histval = histpar(yscale, hist) yrangew = yrange p_hist_w = Plot2d( x_label=xhistlabel, y_label=ylabel, tooltip=hist_tooltip_w, title="", width=600, height=300, yscale="auto", y_range=yrange, x_range=(hist_rg[0] + xw_ref[1], hist_rg[1] + xw_ref[1]), hover_mode="vline", ).quad( source_hist, top=histval, bottom=bottomval, line_width=0.8, ) # -------------------------------------------------------------- # vlines ranges: bname = 'XWSIGMA' for ialert in nrg: # par[bname+'_NORMAL_RANGE']: spans = Span(location=ialert + xw_ref[0], dimension='height', line_color='green', line_dash='dashed', line_width=2) p_hist_x.add_layout(spans) my_label = Label(x=ialert + xw_ref[0], y=yrange[-1] / 2.2, y_units='data', text='Normal Range', text_color='green', angle=np.pi / 2.) p_hist_x.add_layout(my_label) for ialert in wrg: # par[bname+'_WARN_RANGE']: spans = Span(location=ialert + xw_ref[0], dimension='height', line_color='tomato', line_dash='dotdash', line_width=2) p_hist_x.add_layout(spans) my_label = Label(x=ialert + xw_ref[0], y=yrange[-1] / 2.2, y_units='data', text='Warning Range', text_color='tomato', angle=np.pi / 2.) p_hist_x.add_layout(my_label) for ialert in nrg: # par[bname+'_NORMAL_RANGE']: spans = Span(location=ialert + xw_ref[1], dimension='height', line_color='green', line_dash='dashed', line_width=2) p_hist_w.add_layout(spans) my_label = Label(x=ialert + xw_ref[1], y=yrangew[-1] / 2.2, y_units='data', text='Normal Range', text_color='green', angle=np.pi / 2.) p_hist_w.add_layout(my_label) for ialert in wrg: # par[bname+'_WARN_RANGE']: spans = Span(location=ialert + xw_ref[1], dimension='height', line_color='tomato', line_dash='dotdash', line_width=2) p_hist_w.add_layout(spans) my_label = Label(x=ialert + xw_ref[1], y=yrangew[-1] / 2.2, y_units='data', text='Warning Range', text_color='tomato', angle=np.pi / 2.) p_hist_w.add_layout(my_label) # amp 1 xamp = Patch().plot_amp( dz=xw_amp[0], refexp=[xw_ref[0]] * 4, name="XSIGMA AMP", description="X standard deviation per Amp (number of pixels)", wrg=wrg) # amp 2 wamp = Patch().plot_amp( dz=xw_amp[1], refexp=[xw_ref[1]] * 4, name="WSIGMA AMP", description="W standard deviation per Amp (number of pixels)", wrg=wrg) # ------------------------------------------------------------------------- info_col = Title().write_description('xwsigma') current_exposures = check_ccds['METRICS']['XWSIGMA'] alert_x = Table().alert_table(nrg, wrg) alert_w = Table().alert_table(nrg, wrg) if flavor == 'science': program = gen_info['PROGRAM'].upper() reference_exposures = check_ccds['PARAMS']['XWSIGMA_' + program + '_REF'] keynames = ["XSIGMA"] x_metric = Table().reference_table(keynames, [current_exposures[0]], [reference_exposures[0]]) keynames = ["WSIGMA"] w_metric = Table().reference_table(keynames, [current_exposures[1]], [reference_exposures[1]]) layout = column( info_col, widgetbox(Div(), css_classes=["tableranges"]), widgetbox( Div(text= '<h2 align=center style="text-align:center;"> XSIGMA </h2>' )), widgetbox( Div(text= '<h2 align=center style="text-align:center;"> WSIGMA </h2>' )), x_metric, w_metric, alert_x, alert_w, column(wedge_plot_x, sizing_mode='scale_both'), column(wedge_plot_w, sizing_mode='scale_both'), column(xhist, sizing_mode='scale_both'), column(whist, sizing_mode='scale_both'), column(p_hist_x, sizing_mode='scale_both'), column(p_hist_w, sizing_mode='scale_both'), column(xamp, sizing_mode='scale_both'), column(wamp, sizing_mode='scale_both'), css_classes=["display-grid"], sizing_mode='scale_width') else: reference_exposures = check_ccds['PARAMS']['XWSIGMA_REF'] keynames = ["XSIGMA"] x_metric = Table().reference_table(keynames, [current_exposures[0]], [reference_exposures[0]]) keynames = ["WSIGMA"] w_metric = Table().reference_table(keynames, [current_exposures[1]], [reference_exposures[1]]) layout = column( info_col, widgetbox(Div(), css_classes=["tableranges"]), widgetbox( Div(text= '<h2 align=center style="text-align:center;"> XSIGMA </h2>' )), widgetbox( Div(text= '<h2 align=center style="text-align:center;"> WSIGMA </h2>' )), x_metric, w_metric, alert_x, alert_w, column(xhist, sizing_mode='scale_both'), column(whist, sizing_mode='scale_both'), column(p_hist_x, sizing_mode='scale_both'), column(p_hist_w, sizing_mode='scale_both'), column(xamp, sizing_mode='scale_both'), column(wamp, sizing_mode='scale_both'), css_classes=["display-grid"], sizing_mode='scale_width') return file_html(layout, CDN, "XWSIGMA")
class Regression(): def __init__(self, xaxis, yaxis, start, end, camera): self.yaxis = yaxis self.xaxis = xaxis self.start = datetime.strptime(start, '%Y%m%d').strftime('%Y-%m-%d') self.end = datetime.strptime(end, '%Y%m%d').strftime('%Y-%m-%d') self.camera = camera self.models = QLFModels() def render_plot(self, outputs_x, outputs_y): metrics_path = os.path.join(qlf_root, "framework", "ql_mapping", "metrics.json") with open(metrics_path) as f: metrics = json.load(f) y_data = metrics[self.yaxis] x_data = metrics[self.xaxis] df_x = pd.DataFrame(list(outputs_x)) df_y = pd.DataFrame(list(outputs_y)) source = ColumnDataSource( data=dict(x=df_x['value'].apply(lambda x: x[0]), y=df_y['value'].apply(lambda x: x[0]), exposure=df_x['exposure_id'], camera=df_x['camera'])) TOOLTIPS = """ <div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">{}: </span> <span style="font-size: 1vw; color: #515151">@y</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">{}: </span> <span style="font-size: 1vw; color: #515151;">@x</span> </div> <div> <span style="font-size: 1vw; font-weight: bold; color: #303030;">Exposure: </span> <span style="font-size: 1vw; color: #515151;">@exposure</span> </div> </div> """.format(y_data['display'], x_data['display']) hover = HoverTool(tooltips=TOOLTIPS) plot = figure( title="Camera: {}".format(self.camera), toolbar_location='above', active_drag="box_zoom", plot_height=300, # plot_width=500, x_axis_label=x_data['display'], y_axis_label=y_data['display'], tools=[hover, 'box_zoom,pan,wheel_zoom,box_select,reset'], sizing_mode='scale_width') q = plot.circle('x', 'y', source=source, size=8, fill_color='dodgerblue', hover_fill_color='blue', line_color='black') font_size = "1.2vw" plot.xaxis.major_label_text_font_size = font_size plot.yaxis.major_label_text_font_size = font_size plot.xaxis.axis_label_text_font_size = font_size plot.yaxis.axis_label_text_font_size = font_size plot.legend.label_text_font_size = font_size plot.title.text_font_size = font_size self.layout = plot def render(self): outputs_y = self.models.get_product_metrics_by_camera( self.yaxis, self.camera, begin_date=self.start, end_date=self.end) outputs_x = self.models.get_product_metrics_by_camera( self.xaxis, self.camera, begin_date=self.start, end_date=self.end) self.render_plot(outputs_x, outputs_y) return file_html(self.layout, CDN, "Regression")