qa_validation_dir)) os.mkdir(qa_validation_dir) # # 'create another directory to store the pybdsf output # qa_validation_dir = '{0:s}/validation'.format(qa_validation_dir) # if not os.path.exists(qa_validation_dir): # print("Directory {0:s} does not exist and will be created".format( # qa_validation_dir)) # os.mkdir(qa_validation_dir) # Run validation depending on the chosen mode # +++++++++++++++++++++++++++++++++++++++++++ # Create logging file lib.setup_logger('debug', logfile='{0:s}/{1:s}_{2:s}_validation.log'.format( qa_validation_dir, obs_id, run_mode)) logger = logging.getLogger(__name__) # logging.basicConfig(filename='{0:s}/{1:d}_{2:s}_pybdsf.log'.format(qa_validation_dir, obs_id, run_mode), level=logging.DEBUG, # format='%(asctime)s - %(levelname)s: %(message)s') # logger = logging.getLogger(__name__) # run through continuum mode if run_mode == 'continuum': # base directory for data if args.trigger_mode: logger.info( "--> Running continuum QA in trigger mode. Looking only for data processed by Apercal on {0:s} <--"
if args.path is None: if args.basedir is not None: qa_dir = get_default_imagepath(args.scan, basedir=args.basedir) else: qa_dir = get_default_imagepath(args.scan) # check that selfcal qa directory exists qa_preflag_dir = os.path.join(qa_dir, "preflag") if not os.path.exists(qa_preflag_dir): os.mkdir(qa_preflag_dir) else: qa_preflag_dir = args.path # Create log file lib.setup_logger('info', logfile=os.path.join(qa_preflag_dir, 'run_preflag_qa.log')) logger = logging.getLogger(__name__) logger.info("Running preflag QA") # now combine the plots try: start_time = time.time() preflag_plots.combine_preflag_plots(qa_preflag_dir, trigger_mode=args.trigger_mode) except Exception as e: logger.warning("Running preflag QA failed") logger.exception(e) else: logger.warning( "Running preflag QA ... Done ({0:.0f}s)".format(time.time() -
args = parser.parse_args() # If no path is given change to default QA path if args.path is None: output_path = get_default_imagepath(args.scan, basedir=args.basedir) # check that selfcal qa directory exists output_path = os.path.join(output_path, "selfcal/") if not os.path.exists(output_path): os.mkdir(output_path) else: output_path = args.path # Create log file lib.setup_logger('info', logfile='{0:s}run_scal_plots.log'.format(output_path)) logger = logging.getLogger(__name__) # Get selfcal maps if args.maps: try: logger.info("#### Creating selfcal maps ...") start_time_maps = time.time() get_selfcal_maps(args.scan, output_path, trigger_mode=args.trigger_mode) logger.info("#### Creating selfcal maps. Done ({0:.0f}s)".format( time.time() - start_time_maps)) except Exception as e: logger.error(e) logger.error("#### Creating selfcal maps failed")
def main(): start_time = time.time() args = parse_args() obs_id = args.obs_id flux_cal = args.calibrator qa_dir = args.path base_dir = args.base_dir n_threads = args.threads subband_step = args.subband_step # set output directory if qa_dir is None: if base_dir is not None: qa_dir = get_default_imagepath(obs_id, basedir=base_dir) else: qa_dir = get_default_imagepath(obs_id) # check that path exists if not os.path.exists(qa_dir): print("Directory {0:s} does not exist and will be created".format( qa_dir)) os.makedirs(qa_dir) data_dir = os.path.dirname(qa_dir).rsplit("qa")[0] # check the mode to run the validation qa_beamweights_dir = os.path.join(qa_dir, "beamweights") # check that this directory exists (just in case) if not os.path.exists(qa_beamweights_dir): print("Directory {0:s} does not exist and will be created".format( qa_beamweights_dir)) os.makedirs(qa_beamweights_dir) lib.setup_logger( 'debug', logfile='{0:s}/create_beamweights.log'.format(qa_beamweights_dir)) logger = logging.getLogger(__name__) logger.info("Getting beamweight plots for {}".format(flux_cal)) # get a list of beams if no beam was provided if args.beam is None: data_dir_beam_list = glob.glob(os.path.join(data_dir, "[0-3][0-9]")) # check that there are beams if len(data_dir_beam_list) == 0: logger.warning("No beams found in {}".format(data_dir)) return None else: beam_list = [ int(os.path.basename(beam)) for beam in data_dir_beam_list ] else: beam_list = [args.beam] # now go through the beams for beam_nr in beam_list: start_time_beam = time.time() logger.info("Processing beam {}".format(beam_nr)) # check that the given calibrator exists data_cal_dir = os.path.join(data_dir, "{0:02d}".format(beam_nr)) # calibrator file cal_file = os.path.join(data_cal_dir, "raw/{}.MS".format(flux_cal)) # check that it exists if not os.path.exists(cal_file): logger.warning( "Could not find calibrator {}. Continue with next beam".format( cal_file)) continue else: logger.info("Found calibrator {}".format(cal_file)) # set output directory for plots qa_beamweights_beam_dir = os.path.join(qa_beamweights_dir, "{0:02d}".format(beam_nr)) # check that this directory exists (just in case) if not os.path.exists(qa_beamweights_beam_dir): logger.info( "Directory {0:s} does not exist and will be created".format( qa_beamweights_beam_dir)) os.makedirs(qa_beamweights_beam_dir) # Start with one measurement set to set up the size of the array # # cal = pt.table( # "/data/hess/apertif/{}/{}/WSRTA{}_B000.MS/APERTIF_CALIBRATION".format(args.cal_date, args.taskid, args.taskid), # ack=False) cal = pt.table(os.path.join(cal_file, "APERTIF_CALIBRATION"), ack=False) num_beams = 40 num_subbands = pt.taql( 'select distinct SPECTRAL_WINDOW_ID FROM $cal').nrows() num_antennas = pt.taql('select distinct ANTENNA_ID FROM $cal').nrows() beamweights = np.zeros((num_beams, num_subbands, num_antennas, 11, 11), dtype=np.complex64) logger.info("Number of subbands in {0} is {1}".format( os.path.basename(cal_file), num_subbands)) # in case there are no subbands or antennas better check if num_subbands != 0 and num_antennas != 0: # Old implementation looped over beams (and I just picked a subband for simplicity, but this could be expanded to loop over subbands) # # plot_sub = 350 # for beam_nr in range(40): # ms_name = "/data/hess/apertif/{}/{}/WSRTA{}_B0{:02}.MS/APERTIF_CALIBRATION".format(args.cal_date, args.taskid, # args.taskid, beam_nr) # print(ms_name) # cal = pt.table(ms_name, ack=False) # weights_gershape = cal.getcol('BEAM_FORMER_WEIGHTS').reshape((num_subbands, -1, 2, 64)) # # for subband in range(num_subbands): # for antenna in range(num_antennas): # beamweights[beam_nr, subband, antenna] = convert_weights(weights_gershape[subband, antenna]) # # print("BEAM NUMBER {}".format(beam_nr)) # # fig, axs = plt.subplots(3, 4, figsize=(15, 11)) # fig, axs = plt.subplots(3, 4, figsize=(10, 7)) # fig.suptitle("Beam {}; Subband {}".format(beam_nr, plot_sub), fontsize=14) # for ax, plot_ant in zip(np.array(axs).flatten(), range(num_antennas)): # ax.imshow(np.abs(beamweights[beam_nr, plot_sub, plot_ant]), cmap='plasma') # ax.set_title("Antenna " + str(plot_ant)) # if plot_ant < 8: # ax.set_xticklabels([]) # for i in range(61): # x, y = give_coord('X', i) # ax.text(x - 0.35, y + 0.18, 'X' + str(i), color='white', fontsize=5) # x, y = give_coord('Y', i) # ax.text(x - 0.35, y + 0.18, 'Y' + str(i), color='white', fontsize=5) # # plt.savefig('/data/hess/apertif/{}/{}_B0{:02}_S{:03}_weights.png'.format(args.cal_date, args.cal_date, # beam_nr, plot_sub)) # plt.close() # New implementation because I was just thinking of using a single beam and plotting a bunch of subbands. (quick and dirty solution) # Beam is chosen by the user and saved in args.beam # ms_name = "/home/hess/apertif/{}/{:02}/3C147.MS/APERTIF_CALIBRATION".format( # args.taskid, beam_nr) # cal = pt.table(ms_name, ack=False) logger.info("Getting weights") weights_gershape = cal.getcol('BEAM_FORMER_WEIGHTS').reshape( (num_subbands, -1, 2, 64)) logger.info("Getting weights ... Done") # parallelise it to plot faster with pymp.Parallel(n_threads) as p: # go throught the subband for subband in p.range(0, num_subbands, subband_step): for antenna in range(num_antennas): beamweights[beam_nr, subband, antenna] = convert_weights( weights_gershape[subband, antenna]) fig, axs = plt.subplots(3, 4, figsize=(10, 7)) fig.suptitle("Beam {}; Subband {}".format( beam_nr, subband), fontsize=14) for ax, plot_ant in zip( np.array(axs).flatten(), range(num_antennas)): ax.imshow(np.abs(beamweights[beam_nr, subband, plot_ant]), cmap='plasma') ax.set_title("Antenna " + str(plot_ant)) if plot_ant < 8: ax.set_xticklabels([]) for i in range(61): x, y = give_coord('X', i) ax.text(x - 0.35, y + 0.18, 'X' + str(i), color='white', fontsize=5) x, y = give_coord('Y', i) ax.text(x - 0.35, y + 0.18, 'Y' + str(i), color='white', fontsize=5) plot_name = os.path.join( qa_beamweights_beam_dir, "{0}_{1}_B{2:02d}_S{3:03d}_weights.png".format( obs_id, flux_cal, beam_nr, subband)) # plt.savefig('/home/hess/apertif/{}/{}_B0{:02}_S{:03}_weights.png'.format(args.taskid, args.cal_date, # beam_nr, subband)) plt.savefig(plot_name, overwrite=True) logger.info("Saving plot {}".format(plot_name)) plt.close('all') logger.info("Processing beam {0} ... Done ({1:.0f}s)".format( beam_nr, time.time() - start_time_beam)) else: logger.warning( "Found {0} subbands and {1} antennas for beam {2} in {3}". format(num_subbands, num_antennas, beam_nr, flux_cal)) logger.info("Getting beamweight plots for {0} ... Done ({1:.0f}s)".format( flux_cal, time.time() - start_time))
""" Trigger the start of a fluxcal pipeline. Returns immediately. Args: taskid_* (int): something like 181204020 name_* (str): something like '3C295' beamlist_* (List[int]): something like [0, 1, 2, ..., 9] Returns: bool: True if apercal started successfully """ basedir = '/data/apertif/{}/'.format(taskid_target) if not os.path.exists(basedir): os.mkdir(basedir) lib.setup_logger('debug', logfile=os.path.join(basedir, 'apercal.log')) logger = logging.getLogger(__name__) gitinfo = subprocess.check_output('cd ' + os.path.dirname(apercal.__file__) + '&& git describe --tag; cd', shell=True).strip() logger.info("Apercal version: " + gitinfo) name_fluxcal = str(name_fluxcal).strip() name_polcal = str(name_polcal).strip() name_target = str(name_target).strip() p0 = prepare() p0.basedir = basedir p0.fluxcal = name_fluxcal + ".MS" p0.polcal = name_polcal + ".MS" p0.target = name_target + ".MS"
def run_triggered_qa(targets, fluxcals, polcals, steps=None, basedir=None, osa=''): """Function to run all QA steps. | Args: | target (list(int, str, list(str))): the target field to be reduced | fluxcals (list(int, str, int)): the flux calibrators for the beams to be reduced | polcals (list(int, str, int)): the polarisation calibrator for the beams to be reduced | steps (list(str)): The QA steps that should be exectuted | basedir (str): The directory where the taskid is located | osa (str): The OSA of the given taskid | Function is called from autocal as | ``run_triggered_qa(tdict['target'], tdict['cal1'], tdict['cal2'])`` | With the first three variables defined (the same way as autocal) as | targets = (190505048, 'LH_WSRT', array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])) fluxcals = [(190505017, '3C147_9', 9), (190505016, '3C147_8', 8), (190505015, '3C147_7', 7), (190505014, '3C147_6', 6), (190505013, '3C147_5', 5), (190505012, '3C147_4', 4), (190505011, '3C147_3', 3), (190505010, '3C147_2', 2), (190505009, '3C147_1', 1), (190505008, '3C147_0', 0)] polcals = [(190506001, '3C286_0', 0), (190506002, '3C286_1', 1), (190506003, '3C286_2', 2), (190506004, '3C286_3', 3), (190506005,'3C286_4', 4), (190506006, '3C286_5', 5), (190506007, '3C286_6', 6), (190506008, '3C286_7', 7), (190506009, '3C286_8', 8), (190506010, '3C286_9', 9)] | If steps is not provided then all steps except mosaic will be performed: | steps = ['inspection_plots', 'preflag', 'crosscal', 'selfcal', 'continuum', 'line', 'report'] | It is possible to select a certain step: | steps = ['inspection_plots'] | test call can look like this: | ``from dataqa.run_qa import run_triggered_qa`` | ``run_triggered_qa((190505048, 'LH_WSRT', [0]), [(190505048, '3C147_10', 10)], [(190505048, '3C286_10', 10)], steps=['report'])`` """ # for time measurement start_time = time.time() # Process input parameters # (same as in start_apercal_pipeline) # ======================== (taskid_target, name_target, beamlist_target) = targets if fluxcals: name_fluxcal = str(fluxcals[0][1]).strip().split('_')[0].upper() else: name_fluxcal = '' if polcals: name_polcal = str(polcals[0][1]).strip().split('_')[0].upper() else: name_polcal = '' if steps is None: # steps = ['preflag', 'crosscal', 'selfcal', # 'continuum', 'line', 'mosaic', 'report'] # steps = ['inspection_plots', 'beamweights', 'preflag', 'crosscal', 'selfcal', # 'continuum', 'line', 'report'] # due to isssue with beamweights script, this step is currently not performed steps = [ 'inspection_plots', 'preflag', 'crosscal', 'selfcal', 'continuum', 'line', 'report' ] # Set up # ====== # Get host name host_name = socket.gethostname() # QA directory if basedir is not None: qa_dir = get_default_imagepath(taskid_target, basedir=basedir) else: qa_dir = get_default_imagepath(taskid_target) basedir = "/data/apertif" # check that path exists if not os.path.exists(qa_dir): print("Directory {0:s} does not exist and will be created".format( qa_dir)) try: os.mkdir(qa_dir) except Exception as e: print(e) # start log file # logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', # filename='{0:s}{1:s}_triggered_qa.log'.format(qa_dir, host_name), level=logging.DEBUG) lib.setup_logger('debug', logfile='{0:s}{1:s}_triggered_qa.log'.format( qa_dir, host_name)) logger = logging.getLogger(__name__) logger.info("#######################") logger.info("Input parameters:") logger.info("target={0:s}".format(str(targets))) logger.info("fluxcals={0:s}".format(str(fluxcals))) logger.info("polcals={0:s}".format(str(polcals))) logger.info("#######################") logger.info('#######################') logger.info('#### Running all QA steps on {0:s}'.format(host_name)) logger.info('#######################') # If both fluxcal and polcal polarized, remove polcal # (taken from start_pipeline) if subs_calmodels.is_polarised( name_polcal) and subs_calmodels.is_polarised(name_fluxcal): name_polcal = "" if (fluxcals and fluxcals != '') and (polcals and polcals != ''): assert (len(fluxcals) == len(polcals)) # avoid symmetry bias, if there is only a polcal but no fluxcal, switch them if fluxcals is None and polcals is not None: logger.info( "Only polcal was provided. Setting polcal {} to fluxcal".format( name_polcal)) fluxcals, polcals = polcals, fluxcals name_fluxcal, name_polcal = name_polcal, name_fluxcal name_polcal = "" # Exchange polcal and fluxcal if specified in the wrong order # (taken from start_pipeline) # (except for how the names are switched) elif not subs_calmodels.is_polarised(name_polcal) and name_polcal != '': if subs_calmodels.is_polarised(name_fluxcal): logger.debug("Switching polcal and fluxcal because " + name_polcal + " is not polarised") fluxcals, polcals = polcals, fluxcals name_fluxcal, name_polcal = name_polcal, name_fluxcal #name_polcal = str(polcals[0][1]).strip() else: logger.debug("Setting polcal to '' since " + name_polcal + " is not polarised") name_polcal = "" elif name_polcal != '': logger.debug("Polcal " + name_polcal + " is polarised, all good") logger.info( "## Observation of target: {0:s}, flux calibrator: {1:s}, polarisation calibrator: {2:s}" .format(name_target, name_fluxcal, name_polcal)) # Write information about the observation into a txt file for later # This information is important for the OSA report # ================================================================= # flux calibrator tid list flux_cal_tid_list = [cal[0] for cal in fluxcals] # pol calibrator tid list if name_polcal != '': pol_cal_tid_list = [cal[0] for cal in polcals] else: pol_cal_tid_list = [] summary_table = Table( [[taskid_target], [name_target], [name_fluxcal], [str(flux_cal_tid_list).replace("]", "").replace("[", "")], [name_polcal], [str(pol_cal_tid_list).replace("]", "").replace("[", "")], [osa]], names=('Obs_ID', 'Target', 'Flux_Calibrator', 'Flux_Calibrator_Obs_IDs', 'Pol_Calibrator', 'Pol_Calibrator_Obs_IDs', 'OSA')) table_name = "{0}_obs.ecsv".format(taskid_target) table_name_with_path = os.path.join(qa_dir, table_name) try: summary_table.write(table_name_with_path, format='ascii.ecsv', overwrite=True) except Exception as e: logger.warning("Saving observation information in {0} failed.".format( table_name_with_path)) logger.exception(e) else: logger.info(("Saving observation information in {0} ... Done.".format( table_name_with_path))) # Inspection Plots # ================ if 'inspection_plots' in steps: start_time_inspection_plot = time.time() # for the target it is enough to do it only for happili-01 # as they do not depend on the beam # for the flux and pol calibrator, they have to be run on every node # get inspection plots for target if host_name == "happili-01": logger.info( "#### Inspection plot QA for {}...".format(name_target)) try: inspection_plot_msg = os.system( 'python /home/apercal/dataqa/run_inspection_plot.py {0:d} {1:s} --basedir={2}' .format(taskid_target, name_target, basedir)) logger.info( "Getting inspection plots finished with msg {0}".format( inspection_plot_msg)) logger.info( "#### Inspection plot QA {0}... Done ".format(name_target)) except Exception as e: logger.warning( "Inspection plot QA for {} failed.".format(name_target)) logger.exception(e) # get inspection plot for flux calibrator logger.info("#### Inspection plot QA for {}...".format(name_fluxcal)) for (taskid_cal, name_cal, beamnr_cal) in fluxcals: try: inspection_plot_msg = os.system( 'python /home/apercal/dataqa/run_inspection_plot.py {0:d} {1:s} -c --beam={2:d} --cal_id={3:d} --basedir={4}' .format(taskid_target, name_fluxcal, beamnr_cal, taskid_cal, basedir)) logger.info( "Getting inspection plots finished with msg {0}".format( inspection_plot_msg)) logger.info( "#### Inspection plot QA for {0} beam {1} ... Done".format( name_fluxcal, beamnr_cal)) except Exception as e: logger.warning( "Inspection plot QA for {} beam {1} failed.".format( name_fluxcal, beamnr_cal)) logger.exception(e) # get inspection plot for pol calibrator if it exists if name_polcal != '': logger.info( "#### Inspection plot QA for {}...".format(name_polcal)) for (taskid_cal, name_cal, beamnr_cal) in polcals: try: inspection_plot_msg = os.system( 'python /home/apercal/dataqa/run_inspection_plot.py {0:d} {1:s} -c --beam={2:d} --cal_id={3:d} --basedir={4}' .format(taskid_target, name_polcal, beamnr_cal, taskid_cal, basedir)) logger.info( "Getting inspection plots finished with msg {0}". format(inspection_plot_msg)) logger.info( "#### Inspection plot QA for {0} beam {1} ... Done". format(name_polcal, beamnr_cal)) except Exception as e: logger.warning( "Inspection plot QA for {} beam {1} failed.".format( name_polcal, beamnr_cal)) logger.exception(e) logger.info("#### Inspection plot QA ... Done (time {0:.1f}s)".format( time.time() - start_time_inspection_plot)) else: logger.warning("#### Did not perform inspection plot QA") # Beamweights Plots # ================= if 'beamweights' in steps: start_time_beamweights = time.time() # this needs to run on every node logger.info("#### Beamweights QA for {}...".format(name_fluxcal)) try: beamweights_msg = os.system( 'python /home/apercal/dataqa/run_beamweights_plots.py {0:d} {1:s} -t 20' .format(taskid_target, name_fluxcal)) logger.info("Getting Beamweightss finished with msg {0}".format( beamweights_msg)) except Exception as e: logger.warning( "Beamweights QA for {} failed.".format(name_fluxcal)) logger.exception(e) else: logger.info("#### Beamweights QA ... Done (time {0:.1f}s)".format( time.time() - start_time_beamweights)) else: logger.warning("#### Did not perform Beamweights QA") # Preflag QA # ========== if 'preflag' in steps and host_name == "happili-01": logger.info("#### Running preflag QA ...") start_time_preflag = time.time() try: preflag_msg = os.system( 'python /home/apercal/dataqa/run_preflag_qa.py {0:d} --basedir={1}' .format(taskid_target, basedir)) logger.info("Preflag QA finished with msg {0}".format(preflag_msg)) logger.info( "#### Running preflag QA ... Done (time {0:.1f}s)".format( time.time() - start_time_preflag)) except Exception as e: logger.warning("Preflag QA failed. Continue with next QA") logger.exception(e) # Disabled rfinder # try: # preflag_msg = os.system( # 'python /home/apercal/dataqa/run_rfinder.py {0:d} {1:s} --trigger_mode'.format(taskid_target, name_fluxcal)) # logger.info( # "Preflag QA finished with msg {0}".format(preflag_msg)) # logger.info("#### Running preflag QA ... Done (time {0:.1f}s)".format( # time.time()-start_time_preflag)) # except Exception as e: # logger.warning("Preflag QA failed. Continue with next QA") # logger.exception(e) else: logger.warning("#### Did not perform preflag QA") # Crosscal QA # =========== if 'crosscal' in steps and name_fluxcal != '': logger.info('#### Running crosscal QA ...') start_time_crosscal = time.time() try: crosscal_msg = os.system( 'python /home/apercal/dataqa/run_ccal_plots.py {0:d} "{1:s}" "{2:s}" --basedir={3} --trigger_mode' .format(taskid_target, name_fluxcal, name_polcal, basedir)) logger.info( "Crosscal QA finished with msg {0}".format(crosscal_msg)) logger.info( "#### Running crosscal QA ... Done (time {0:.1f}s)".format( time.time() - start_time_crosscal)) except Exception as e: logger.warning("Crosscal QA failed. Continue with next QA") logger.exception(e) else: logger.warning("#### Did not perform crosscal QA") # Selfcal QA # ========== if 'selfcal' in steps: logger.info('#### Running selfcal QA ...') start_time_selfcal = time.time() try: selfcal_msg = os.system( 'python /home/apercal/dataqa/run_scal_plots.py {0:d} {1:s} --basedir={2} --trigger_mode' .format(taskid_target, name_target, basedir)) logger.info("Selfcal QA finished with msg {0}".format(selfcal_msg)) logger.info( "#### Running selfcal QA ... Done (time {0:.1f}s)".format( time.time() - start_time_selfcal)) except Exception as e: logger.warning("Selfcal QA failed. Continue with next QA") logger.exception(e) else: logger.warning("#### Did not perform selfcal QA") # Mosaic QA # ========== if 'mosaic' in steps and host_name == 'happili-01': logger.info('#### Mosaic QA is currently not available ...') # logger.info('#### Running mosaic QA ...') # start_time_mosaic = time.time() # try: # # Create the mosaic # logger.info('## Making the mosaic ...') # start_time_make_mosaic = time.time() # make_mosaic_msg = os.system( # 'python /home/apercal/dataqa/make_mosaic_image.py {0:d}'.format(taskid_target)) # logger.info( # "Making mosaic finished with msg {0}".format(make_mosaic_msg)) # logger.info("## Making the mosaic ... Done (time {0:.1f}s)".format( # time.time()-start_time_make_mosaic)) # # Run the validation tool # logger.info('## Run validation ...') # start_time_mosaic_validation = time.time() # mosaic_validation_msg = os.system( # 'python /home/apercal/dataqa/run_continuum_validation.py {0:d} --for_mosaic'.format(taskid_target)) # logger.info( # "Mosaic validation finished with msg {0}".format(mosaic_validation_msg)) # logger.info("## Run validation ... Done (time {0:.1f}s)".format( # time.time()-start_time_mosaic_validation)) # logger.info("#### Running mosaic QA ... Done (time {0:.1f}s)".format( # time.time()-start_time_mosaic)) # except Exception as e: # logger.warning("Mosaic QA failed. Continue with next QA") # logger.exception(e) else: logger.warning("#### Did not perform mosaic QA") # Line QA # ======= if 'line' in steps: logger.info('#### Running line QA ...') start_time_line = time.time() try: # Get cube statistic without continuum subtraction logger.info('## Get cube statistic ...') start_time_get_cube_stat = time.time() cube_stat_msg = os.system( 'python /home/apercal/dataqa/run_cube_stats.py {0:d} --basedir={1} --trigger_mode' .format(taskid_target, basedir)) logger.info( "Cube stat finished with msg {0}".format(cube_stat_msg)) logger.info( "## Get cube statistic ... Done (time {0:.1f}s)".format( time.time() - start_time_get_cube_stat)) # Subtract continuum # logger.info('## Subtract continuum ...') # start_time_subtract_continuum = time.time() # subtract_cont_msg = os.system( # 'python /home/apercal/dataqa/subtract_continuum.py {0:d} --trigger_mode'.format(taskid_target)) # logger.info( # "Continuum subtraction finished with msg {0}".format(subtract_cont_msg)) # logger.info("## Subtract continuum ... Done (time {0:.1f}s)".format( # time.time()-start_time_subtract_continuum)) # # Get cube statistic after continuum subtraction # logger.info( # '## Get cube statistic after continuum subtraction ...') # start_time_get_cube_stat_cont = time.time() # get_cube_stat_cont_msg = os.system( # 'python /home/apercal/dataqa/run_cube_stats_cont.py {0:d} --trigger_mode'.format(taskid_target)) # logger.info( # "Cube stat cont finished with msg {0}".format(get_cube_stat_cont_msg)) # logger.info("## Get cube statistic after continuum subtraction ... Done (time {0:.1f}s)".format( # time.time()-start_time_get_cube_stat_cont)) logger.info("#### Running line QA ... Done (time {0:.1f}s)".format( time.time() - start_time_line)) except Exception as e: logger.warning("Line QA failed. Continue with next QA") logger.exception(e) else: logger.warning("#### Did not perform line QA") # Continuum QA # ============ if 'continuum' in steps: logger.info('#### Running continuum QA ...') start_time_continuum = time.time() try: continuum_msg = os.system( 'python /home/apercal/dataqa/run_continuum_validation.py {0:d} --basedir={1} --trigger_mode' .format(taskid_target, basedir)) logger.info( "Continuum QA finished with msg {0}".format(continuum_msg)) logger.info( "#### Running continuum QA ... Done (time {0:.1f}s)".format( time.time() - start_time_continuum)) except Exception as e: logger.warning("Continuum QA failed. Continue with next QA") logger.exception(e) else: logger.warning("#### Did not perform continuum QA") # Create report # ============= if 'report' in steps: # merge the crosscal and selfcal plots for the report if host_name == 'happili-01': logger.info('#### Merge crosscal and selfcal plots...') start_time_merge = time.time() try: report_msg = os.system( 'python /home/apercal/dataqa/run_merge_plots.py {0:d} --basedir={1} --do_ccal --do_scal --run_parallel' .format(taskid_target, basedir)) logger.info("Merging finished with msg {0}".format(report_msg)) logger.info( "#### Merge crosscal and selfcal plots ... Done (time {0:.1f}s)" .format(time.time() - start_time_merge)) except Exception as e: logger.warning("Merge crosscal and selfcal plots failed.") logger.exception(e) # now create the report logger.info('#### Create report ...') start_time_report = time.time() try: report_msg = os.system( 'python /home/apercal/dataqa/create_report.py {0:d} --basedir={1} --trigger_mode' .format(taskid_target, basedir)) logger.info("Report finished with msg {0}".format(report_msg)) logger.info("#### Create report ... Done (time {0:.1f}s)".format( time.time() - start_time_report)) except Exception as e: logger.warning("Creating report failed.") logger.exception(e) else: logger.warning("#### Did not create a report") # Finish # ====== logger.info('#######################') logger.info( '#### Running all QA steps on {0:s} ... Done (time {1:.1f}s)'.format( host_name, time.time() - start_time)) logger.info('#######################')
def run_manual_processing(): """Function to run Apercal and QA""" # make sure a use base directory if basedir is None: print("ERROR: No basedir specified. Abort.") return -1 start_time = time.time() host_name = socket.gethostname() cwd = os.getcwd() name_fluxcal = str(fluxcals[0][1]).strip().split('_')[0].upper() (task_id, name_target, beamlist_target) = targets logfile = os.path.join(cwd, "{0}_{1}_apercal_qa.log".format(task_id, host_name)) # Setting up log file lib.setup_logger('debug', logfile=logfile) logger = logging.getLogger(__name__) if do_apercal: logger.info("Running apercal manually") logger.info( "When apercal runs, the logging information will be in the apercal log" ) logger.info("Using task id: {0}".format(task_id)) logger.info("target = {}".format(str(targets))) logger.info("flux_cal = {}".format(str(fluxcals))) logger.info("pol_cal = {}".format(str(polcals))) # logger.info("Steps: {}".format(str(steps))) try: start_time = time.time() logger.info('Running apercal') # return_msg = start_apercal_pipeline( # targets, fluxcals, polcals, basedir=basedir, steps=steps, configfilename=configfile) return_msg = start_apercal_pipeline(targets, fluxcals, polcals, basedir=basedir, steps=steps, configfilename=configfile) # return_msg = start_apercal_pipeline( # targets, fluxcals, polcals, steps=steps) except Exception as e: lib.setup_logger('debug', logfile=logfile) logger = logging.getLogger(__name__) logger.warning("Running apercal failed ({0:.3f}h)".format( (time.time() - start_time) / 3600.)) logger.warning(return_msg) logger.exception(e) else: lib.setup_logger('debug', logfile=logfile) logger = logging.getLogger(__name__) logger.info("Running apercal ... Done ({0:.3f}h)".format( (time.time() - start_time) / 3600.)) if do_qa: logger.info("Running QA manually") logger.info( "When apercal runs, the logging information will be in the apercal log" ) logger.info("Using task id: {0}".format(task_id)) logger.info("target = {}".format(str(targets))) logger.info("flux_cal = {}".format(str(fluxcals))) logger.info("pol_cal = {}".format(str(polcals))) logger.info("Running all QA steps") try: start_time = time.time() logger.info('Running QA') return_msg = run_triggered_qa(targets, fluxcals, polcals, osa=osa, basedir=os.path.dirname(basedir), steps=steps_qa) except Exception as e: lib.setup_logger('debug', logfile=logfile) logger = logging.getLogger(__name__) logger.warning("Running QA failed ({0:.3f}h)".format( (time.time() - start_time) / 3600.)) logger.warning(return_msg) logger.exception(e) else: lib.setup_logger('debug', logfile=logfile) logger = logging.getLogger(__name__) logger.info("Running QA ... Done ({0:.3f}h)".format( (time.time() - start_time) / 3600.))
def run_manual_processing(): """Function to run Apercal, QA and Apergest""" start_time = time.time() host_name = socket.gethostname() cwd = os.getcwd() host_name = socket.gethostname() name_fluxcal = str(fluxcals[0][1]).strip().split('_')[0].upper() (task_id, name_target, beamlist_target) = targets logfile = os.path.join( cwd, "{0}_{1}_apercal_qa_apergest.log".format(task_id, host_name)) # Setting up log file lib.setup_logger('debug', logfile=logfile) logger = logging.getLogger(__name__) if do_apercal: logger.info("Running apercal manually") logger.info( "When apercal runs, the logging information will be in the apercal log" ) logger.info("Using task id: {0}".format(task_id)) logger.info("target = {}".format(str(targets))) logger.info("flux_cal = {}".format(str(fluxcals))) logger.info("pol_cal = {}".format(str(polcals))) # logger.info("Steps: {}".format(str(steps))) try: start_time = time.time() logger.info('Running apercal') # return_msg = start_apercal_pipeline( # targets, fluxcals, polcals, basedir=basedir, steps=steps, configfilename=configfile) start_apercal_pipeline(targets, fluxcals, polcals, configfilename=configfile, steps=steps) # return_msg = start_apercal_pipeline( # targets, fluxcals, polcals, steps=steps) except Exception as e: lib.setup_logger('debug', logfile=logfile) logger = logging.getLogger(__name__) logger.warning("Running apercal failed ({0:.3f}h)".format( (time.time() - start_time) / 3600.)) logger.exception(e) else: lib.setup_logger('debug', logfile=logfile) logger = logging.getLogger(__name__) logger.info("Running apercal ... Done ({0:.3f}h)".format( (time.time() - start_time) / 3600.)) if do_qa: logger.info("Running QA manually") logger.info( "When apercal runs, the logging information will be in the apercal log" ) logger.info("Using task id: {0}".format(task_id)) logger.info("target = {}".format(str(targets))) logger.info("flux_cal = {}".format(str(fluxcals))) logger.info("pol_cal = {}".format(str(polcals))) logger.info("Running all QA steps") try: start_time = time.time() logger.info('Running QA') run_triggered_qa(targets, fluxcals, polcals, osa=osa) except Exception as e: lib.setup_logger('debug', logfile=logfile) logger = logging.getLogger(__name__) logger.warning("Running QA failed ({0:.3f}h)".format( (time.time() - start_time) / 3600.)) logger.exception(e) else: lib.setup_logger('debug', logfile=logfile) logger = logging.getLogger(__name__) logger.info("Running QA ... Done ({0:.3f}h)".format( (time.time() - start_time) / 3600.)) if do_apergest: logger.info("Running Apergest manually") apergest_dir = "/home/schulz/apercal/apergest/{}".format(host_name) os.chdir(apergest_dir) try: apergest(task_id, do_make_jsons=True, do_prepare_ingest=True, do_run_ingest=True, do_delete_data=False) except Exception as e: os.chdir(cwd) lib.setup_logger('debug', logfile=logfile) logger = logging.getLogger(__name__) logger.warning("Running Apergest failed ({0:.3f}h)".format( (time.time() - start_time) / 3600.)) logger.warning(return_msg) logger.exception(e) else: os.chdir(cwd) lib.setup_logger('debug', logfile=logfile) logger = logging.getLogger(__name__) logger.info("Running Apergest ... Done ({0:.3f}h)".format( (time.time() - start_time) / 3600.)) # just to make sure to get back os.chdir(cwd)
def main(): start = timer() parser = argparse.ArgumentParser(description='Generate selfcal QA plots') # 1st argument: File name parser.add_argument("obs_id", help='ID of observation of target field') parser.add_argument( "src_name", help='Name of the calibrator or target of the plots') parser.add_argument("-c", "--calibrator", action="store_true", default=False, help='Set if a calibrator is used. Also requires beam and cal_id') parser.add_argument("--beam", type=int, default=None, help='If src_name is a calibrator set the beam number') parser.add_argument("--cal_id", type=str, default=None, help='Obs ID of the calibrator') parser.add_argument('-p', '--path', default=None, help='Destination for images') parser.add_argument('-b', '--basedir', default=None, help='Directory of obs id') # this mode will make the script look only for the beams processed by Apercal on a given node # parser.add_argument("--trigger_mode", action="store_true", default=False, # help='Set it to run Autocal triggering mode automatically after Apercal.') args = parser.parse_args() # If no path is given change to default QA path if args.path is None: if args.basedir is not None: output_path = get_default_imagepath( args.obs_id, basedir=args.basedir) else: output_path = get_default_imagepath(args.obs_id) # check that selfcal qa directory exists qa_plot_dir = os.path.join(output_path, "inspection_plots") if not os.path.exists(qa_plot_dir): os.mkdir(qa_plot_dir) else: qa_plot_dir = args.path # create a directory with the src_name to put if args.src_name is not None: qa_plot_dir = os.path.join(qa_plot_dir, args.src_name) if not os.path.exists(qa_plot_dir): os.mkdir(qa_plot_dir) # if it is a calibrator then put the plots into a beam directory if args.calibrator: if args.beam is None: print("ERROR: Please specify beam of calibrator") return -1 elif args.cal_id is None: print("ERROR: Please specify id of calibrator") return -1 else: is_calibrator = True qa_plot_dir = os.path.join( qa_plot_dir, "{0:02d}".format(args.beam)) if not os.path.exists(qa_plot_dir): os.mkdir(qa_plot_dir) else: is_calibrator = False # Create log file lib.setup_logger( 'info', logfile=os.path.join(qa_plot_dir, 'get_inspection_plot.log')) logger = logging.getLogger(__name__) # Run function to get plots try: logger.info("#### Getting inspection plots ...") start_time_plots = time.time() get_inspection_plots(args.obs_id, qa_plot_dir, is_calibrator=is_calibrator, cal_id=args.cal_id) except Exception as e: logger.error(e) logger.error("#### Getting inspection plots failed") else: logger.info("#### Getting inspection plots... Done ({0:.0f}s)".format( time.time()-start_time_plots))
default=False, help='Set to run the script in parallel') parser.add_argument('-b', '--basedir', default=None, help='Data directory') parser.add_argument('--n_cores', default=5, help='Data directory') args = parser.parse_args() # get the QA directory qa_dir = get_default_imagepath(args.scan, basedir=args.basedir) # start logging # Create logging file lib.setup_logger('info', logfile=os.path.join(qa_dir, 'merge_plots.log')) logger = logging.getLogger(__name__) start_time = time() logger.info("#### Merging plots ...") try: run_merge_plots(qa_dir, do_ccal=args.do_ccal, do_scal=args.do_scal, run_parallel=args.run_parallel, n_cores=args.n_cores) except Exception as e: logger.warning( "#### Merging plots ... Failed ({0:.0f}s)".format(time() -
from merge_ccal_scal_plots import run_merge_plots import numpy as np import os from apercal.libs import lib import logging lib.setup_logger('debug', logfile='test_merge_plots.log') logger = logging.getLogger(__name__) basedir = '/data/apertif/190602049_flag-strategy-test/qa' do_ccal = True do_scal = False # file_list = np.array([os.path.join(basedir, img) for img in img_list]) # new_file_name = os.path.join(basedir, "merge_test.png") run_merge_plots(basedir, do_ccal=do_ccal, do_scal=do_scal)
def main(): start_time = time.time() # Create and parse argument list # ++++++++++++++++++++++++++++++ parser = argparse.ArgumentParser( description='Create overview for QA') # 1st argument: Observation number parser.add_argument("obs_id", type=str, help='Observation Number') parser.add_argument("--target", type=str, default='', help='Name of the target') parser.add_argument("--fluxcal", type=str, default='', help='Name of the flux calibrator') parser.add_argument("--polcal", type=str, default='', help='Name of the polarisation calibrator') parser.add_argument("--osa", type=str, default='', help='Name of the OSA') parser.add_argument("-p", "--path", type=str, help='Path to QA output') parser.add_argument("-b", "--basedir", type=str, help='Base directory where the obs id is') parser.add_argument("--tank", action="store_true", default=False, help='Create the report on new volume') parser.add_argument("-a", "--add_osa_report", action="store_true", default=False, help='Add only the osa report to the webpage') parser.add_argument("-c", "--combine", action="store_true", default=False, help='(Depracated) Set to create a combined report from all happilis on happili-01. It will overwrite the report on happili-01') parser.add_argument("--no_merge", action="store_true", default=False, help='Set to merge selfcal and crosscal plots') parser.add_argument("--do_not_read_timing", action="store_true", default=False, help='Set to avoid reading timing information. Makes only sense if script is run multiple times or for debugging') parser.add_argument("--page_only", action="store_true", default=False, help='Set only create the webpages themselves') # this mode will make the script look only for the beams processed by Apercal on a given node parser.add_argument("--trigger_mode", action="store_true", default=False, help='Set it to run Autocal triggering mode automatically after Apercal.') parser.add_argument("--single_node", action="store_true", default=False, help='Set it to run QA on a single node and get same result as if running like the OSA. Note, this is different from trigger mode.') args = parser.parse_args() obs_id = args.obs_id qa_dir = args.path base_dir = args.basedir do_combine = args.combine add_osa_report = args.add_osa_report # directory where the output will be of pybdsf will be stored if qa_dir is None: if base_dir is not None: qa_dir = get_default_imagepath(obs_id, basedir=base_dir) else: qa_dir = get_default_imagepath(obs_id) # check that path exists if not os.path.exists(qa_dir): print( "Directory {0:s} does not exist and will be created".format(qa_dir)) os.makedirs(qa_dir) # change the base directory from /data to /tank if args.tank and "/data" in qa_dir: print("Switching to /tank") qa_dir = qa_dir.replace("/data", "/tank") # check the mode to run the validation qa_report_dir = "{0:s}report".format( qa_dir) # check that this directory exists (just in case) if not os.path.exists(qa_report_dir): print("Directory {0:s} does not exist and will be created".format( qa_report_dir)) os.makedirs(qa_report_dir) lib.setup_logger( 'debug', logfile='{0:s}/create_report.log'.format(qa_report_dir)) logger = logging.getLogger(__name__) # if osa report should be added, check it is available if add_osa_report: # name of the osa report for this observation osa_report = os.path.join( qa_report_dir, "OSA_Report/{}_OSA_report.ecsv".format(obs_id)) # check that the file is actually there if not os.path.exists(osa_report): logger.error("No OSA report found. Abort") return -1 else: osa_report = '' # Saving observation information if they do not exist yet # ======================================================= table_name = "{0}_obs.ecsv".format(obs_id) table_name_with_path = os.path.join(qa_dir, table_name) if not os.path.exists(table_name_with_path): obs_info = Table([ [obs_id], [args.target], [args.fluxcal], [''], [args.polcal], [''], [args.osa]], names=( 'Obs_ID', 'Target', 'Flux_Calibrator', 'Flux_Calibrator_Obs_IDs', 'Pol_Calibrator', 'Pol_Calibrator_Obs_IDs', 'OSA')) try: obs_info.write( table_name_with_path, format='ascii.ecsv', overwrite=True) except Exception as e: logger.warning("Saving observation information in {0} failed.".format( table_name_with_path)) logger.exception(e) else: logger.info( ("Saving observation information in {0} ... Done.".format(table_name_with_path))) else: logger.info( ("Observation information already exists. Reading {0}.".format(table_name_with_path))) obs_info = Table.read(table_name_with_path, format="ascii.ecsv") # check on which happili we are: host_name = socket.gethostname() if args.trigger_mode: logger.info( "--> Running report QA in trigger mode. Looking only for data processed by Apercal on {0:s} <--".format(host_name)) elif args.single_node: logger.info( "--> Running report QA in single-node mode. Looking only for data processed by Apercal on {0:s} <--".format(host_name)) elif do_combine: logger.info("Combining QAs from different happilis") if host_name != "happili-01": logger.warning("You are not working on happili-01.") logger.warning("Cannot combine QA from different happilis") do_combine = False elif host_name != "happili-01" and not args.trigger_mode: logger.warning("You are not working on happili-01.") logger.warning("The script will not process all beams") logger.warning("Please switch to happili-01") apercal_log_file = "/data/apertif/{0:s}/apercal.log".format( obs_id) # logging.basicConfig(filename='{0:s}/create_report.log'.format(qa_dir), level=logging.DEBUG, # format='%(asctime)s - %(levelname)s: %(message)s') # getting timing measurment for apercal only in trigger mode # if not add_osa_report and not args.do_not_read_timing: if args.trigger_mode or args.single_node: try: get_pipeline_run_time(obs_id, trigger_mode=args.trigger_mode) except Exception as e: logger.exception(e) # the subpages to be created subpages = ['observing_log', 'summary', 'beamweights', 'inspection_plots', 'preflag', 'crosscal', 'selfcal', 'continuum', 'polarisation', 'line', 'mosaic', 'apercal_log'] logger.info("#### Create report directory structure") # copy the js and css files js_file_name = "{0:s}/report_fct.js".format( hp.__file__.split("/html_report.py")[0]) css_file_name = "{0:s}/report_style.css".format( hp.__file__.split("/html_report.py")[0]) # for copying osa_files: osa_nb_file = "{0:s}/OSA_report.ipynb".format( hp.__file__.split("/html_report.py")[0]) osa_py_file = "{0:s}/osa_functions.py".format( hp.__file__.split("/html_report.py")[0]) osa_files = [osa_nb_file, osa_py_file] # Check that directory of the qa exists if not os.path.exists(qa_dir): logger.error( "Directory {0:s} does not exists. Abort".format(qa_report_dir)) return -1 else: # do things that should only happen on happili-01 when the OSA runs this function if not args.trigger_mode and not args.page_only: if host_name == "happili-01" or args.single_node: # go through some of the subpages and process numpy files for page in subpages: # exclude non-apercal modules (and mosaic) if page != "apercal_log" or page != "inspection_plots" or page != "summary" or page != "mosaic": # just run it on preflag for now if page == "preflag" or page == "crosscal" or page == "convert" or page == "selfcal" or page == "continuum": # get information from numpy files try: logger.info( "## Getting summary table for {}".format(page)) make_nptabel_csv( obs_id, page, qa_dir, output_path=os.path.join(qa_dir, page)) except Exception as e: logger.warning( "## Getting summary table for {} failed".format(page)) logger.exception(e) else: logger.info( "## Getting summary table for {} ... Done".format(page)) # merge plots if not args.no_merge and not args.single_node: try: logger.info( "## Merging selfcal and crosscal plots") run_merge_plots( qa_dir, do_ccal=True, do_scal=True, run_parallel=True, n_cores=5) except Exception as e: logger.warning( "## Merging selfcal and crosscal plots ... Failed") logger.exception(e) else: logger.info( "## Merging selfcal and crosscal plots ... Done") # merge the continuum image properties if page == 'continuum': try: merge_continuum_image_properties_table( obs_id, qa_dir, single_node=args.single_node) except Exception as e: logger.warning( "Merging continuum image properties ... Failed") logger.exception(e) else: logger.info( "Merging continuum image properties ... Done") # get line statistics if page == 'line': try: combine_cube_stats( obs_id, qa_dir, single_node=args.single_node) except Exception as e: logger.warning( "Getting cube statistics ... Failed") logger.exception(e) else: logger.info( "Getting cube statistics ... Done") # create dish delay plot try: logger.info("Getting dish delay plot") get_dish_delay_plots( obs_id, obs_info['Flux_Calibrator'][0], basedir=args.basedir) except Exception as e: logger.warning("Getting dish delay plot ... Failed") logger.exception(e) else: logger.info("Getting dish delay plot ... Done") # create compound beam plots try: logger.info("Getting compound beam plots") make_cb_plots_for_report(obs_id, qa_dir) except Exception as e: logger.warning("Getting compound beam plots ... Failed") logger.exception(e) else: logger.info("Getting compound beam plots ... Done") # Create directory structure for the report if not add_osa_report: logger.info("#### Creating directory structrure") try: hpd.create_report_dirs( obs_id, qa_dir, subpages, css_file=css_file_name, js_file=js_file_name, trigger_mode=args.trigger_mode, single_node=args.single_node, do_combine=do_combine, obs_info=obs_info, osa_files=osa_files) except Exception as e: logger.error(e) else: logger.info("#### Creating directory structrure ... Done") logger.info("#### Creating report") try: hp.create_main_html(qa_report_dir, obs_id, subpages, css_file=css_file_name, js_file=js_file_name, obs_info=obs_info, osa_report=osa_report) except Exception as e: logger.error(e) logger.info("#### Report. Done ({0:.0f}s)".format( time.time()-start_time))
from apercal.modules.mosaic_v2 import mosaic import os import socket import time start_time = time.time() # set path to configfile # note: remove settings below if configfile is used # configfile = "/home/schulz/pipeline/apercal_tests/mosaic/mosaic_v2_190428055.cfg" configfile = None # Setting up log file task_id = 190428055 logfile = os.path.join(os.getcwd(), "{0}_mosaic.log".format(task_id)) lib.setup_logger('debug', logfile=logfile) logger = logging.getLogger(__name__) # Mosaic module mo = mosaic(file_=configfile) # List of settings (Change as necessary) # ====================================== # set the output directory mo.basedir = '/data/schulz/mosaic_test/190428055/' # enable continuum mosaic mo.mosaic_continuum_mf = True # set the taskid of observation to mosaic
# get taskid/obs_id/scan obs_id = args.obs_id # get the QA directory for this observation qa_dir = get_default_imagepath(obs_id, basedir=args.basedir) # get the line QA directory for this observation qa_line_dir = "{0:s}line".format(qa_dir) if not os.path.exists(qa_line_dir): print("Creating directory {0:s}".format(qa_line_dir)) os.mkdir(qa_line_dir) # Create logging file lib.setup_logger('debug', logfile='{0:s}/get_cube_stats.log'.format(qa_line_dir)) logger = logging.getLogger(__name__) # check host name host_name = socket.gethostname() # get data directories depending on the host name if args.trigger_mode: logger.info( "--> Running line QA in trigger mode. Looking only for data processed by Apercal on {0:s} <--" .format(host_name)) data_base_dir_list = ['/data/apertif/{0:s}'.format(obs_id)] elif host_name != "happili-01" and not args.trigger_mode: logger.warning("You are not working on happili-01.") logger.warning("The script will not process all beams") logger.warning("Please switch to happili-01")
args = parser.parse_args() # If no path is given change to default QA path if args.path is None: output_path = get_default_imagepath(args.taskID, basedir=args.basedir) # check that preflag qa directory exists output_path = "{0:s}preflag/".format(output_path) if not os.path.exists(output_path): os.mkdir(output_path) else: output_path = args.path # Create logging file lib.setup_logger('debug', logfile='{0:s}run_rfinder.log'.format(output_path)) logger = logging.getLogger(__name__) # get data directories depending on the host name host_name = socket.gethostname() if args.trigger_mode: logger.info( "--> Running line QA in trigger mode. Looking only for data processed by Apercal on {0:s} <--" .format(host_name)) data_beam_dir_list = glob.glob("/data/apertif/{}/[0-3][0-9]".format( args.taskID)) elif host_name != "happili-01" and not args.trigger_mode: logger.warning("You are not working on happili-01.") logger.warning("The script will not process all beams") logger.warning("Please switch to happili-01") data_beam_dir_list = glob.glob("/data/apertif/{}/[0-3][0-9]".format(
def make_mosaic(task_id, basedir, centre_ra=None, centre_dec=None, mosaic_beams=None, primary_beam_map_dir=None, step_limit=None, use_noise_correlation=False, do_validation=False, continuum_image_dir=None, do_not_cleanup=False): start_time = time.time() # configfile # configfile = "/home/schulz/pipeline/apercal_tests/mosaic/mosaic_v2_190428055.cfg" configfile = None # Setting up log file # task_id = 190822046 logfile = os.path.join(os.getcwd(), "{0}_mosaic.log".format(task_id)) lib.setup_logger('debug', logfile=logfile) logger = logging.getLogger(__name__) host_name = socket.gethostname() # Mosaic module mo = mosaic(file_=configfile) # Settings # ======== # set the number of steps for the mosaic to go through mo.mosaic_step_limit = step_limit # set the output directory mo.basedir = os.path.join(basedir, '{}'.format(task_id)) # enable continuum mosaic mo.mosaic_continuum_mf = True #disable line and polarization mosaics mo.mosaic_line = False mo.mosaic_polarisation = False # set the taskid of observation to mosaic mo.mosaic_taskid = "{}".format(task_id) # set the list of beams or all if mosaic_beams is None: mo.mosaic_beams = 'all' else: mo.mosaic_beams = mosaic_beams # set location of continuum images, default is ALTA # continuum images must be located in <path>/<beam_nr>/<image_name>.fits if continuum_image_dir is None: mo.mosaic_continuum_image_origin = "ALTA" else: mo.mosaic_continuum_image_origin = continuum_image_dir # mo.mosaic_continuum_image_origin = "/data/pisano/190428055/continuum/raw/" # set the type of primary beam to be used mo.mosaic_primary_beam_type = 'Correct' # if the correct primary beam is suppose to be used # set the path to files (do not change) if primary_beam_map_dir is None: mo.mosaic_primary_beam_shape_files_location = "/tank/apertif/driftscans/fits_files/191023/chann_5" else: mo.mosaic_primary_beam_shape_files_location = primary_beam_map_dir # set the cutoff mo.mosaic_beam_map_cutoff = 0.1 # set the projection centre if centre_ra is None and centre_dec is None: # using the a given beam mo.mosaic_continuum_projection_centre_beam = '00' else: # using ra and dec (untested) mo.mosaic_continuum_projection_centre_ra = centre_ra mo.mosaic_continuum_projection_centre_dec = centre_dec # type of beam for convolution mo.mosaic_common_beam_type = 'circular' # turn on noise correlation if use_noise_correlation: mo.mosaic_use_askap_based_matrix = True else: mo.mosaic_use_askap_based_matrix = False # run the image validation tool on the mosaic if do_validation: mo.mosaic_continuum_image_validation = True else: mo.mosaic_continuum_image_validation = False # clean up mo.mosaic_continuum_clean_up_level = 1 if do_not_cleanup: mo.mosaic_continuum_clean_up = False else: mo.mosaic_continuum_clean_up = True # create the mosaic # ================= mo.go() logger.info("Finished after {0:.0f}s".format(time.time() - start_time))
help='Set it to run Autocal triggering mode automatically after Apercal.') args = parser.parse_args() # If no path is given change to default QA path if args.path is None: output_path = get_default_imagepath(args.scan, basedir=args.basedir) # check that crosscal qa directory exists output_path = "{0:s}crosscal/".format(output_path) if not os.path.exists(output_path): os.mkdir(output_path) else: output_path = args.path # Create logging file lib.setup_logger( 'debug', logfile='{0:s}run_ccal_plots.log'.format(output_path)) logger = logging.getLogger(__name__) # logging.basicConfig(level=logging.DEBUG) # Create crosscal plots crosscal_plots.make_all_ccal_plots( args.scan, args.fluxcal, args.polcal, output_path=output_path, basedir=args.basedir, trigger_mode=args.trigger_mode) end = timer() logger.info('Elapsed time to generate cross-calibration data QA inpection plots is {} minutes'.format( (end - start)/60.)) #time in minutes
def start_apercal_pipeline(targets, fluxcals, polcals, dry_run=False, basedir=None, flip_ra=False, steps=None, configfilename=None): """ Trigger the start of a fluxcal pipeline. Returns when pipeline is done. Example for taskid, name, beamnr: (190108926, '3C147_36', 36) Fluxcals and polcals can be specified in the wrong order, if the polcal is not polarised they will be flipped. If both polcals and fluxcals are set, they should both be the same length. A list of config files can be provided, i.e., one for each beam. If a single config file is given, copies of it will be created so that there is one config per beam. If no config file is given, the default one is used and copies for each beam are made. Args: targets (Tuple[int, str, List[int]]): taskid, name, list of beamnrs fluxcals (List[Tuple[int, str, int]]): fluxcals: taskid, name, beamnr polcals (List[Tuple[int, str, int]]): polcals: taskid, name, beamnr (can be None) dry_run (bool): interpret arguments, do not actually run pipeline basedir (str): base directory; if not specified will be /data/apertif/{target_taskid} flip_ra (bool): flip RA (for old measurement sets where beamweights were flipped) steps (List[str]): list of steps to perform configfilename (List[str]): Custom configfile (should be full path for now) Returns: Tuple[Dict[int, List[str]], str], str: Tuple of a dict, the formatted runtime, and possibly an exception. The dict contains beam numbers (ints) as keys, a list of failed steps as values. Failed is defined here as 'threw an exception', only for target steps. Please also read logs. """ if steps is None: steps = [ "prepare", "split", "preflag", "ccal", "convert", "scal", "continuum", "polarisation", "line", "transfer" ] (taskid_target, name_target, beamlist_target) = targets # set the base directory if none was provided if not basedir: basedir = '/data/apertif/{}/'.format(taskid_target) elif len(basedir) > 0 and basedir[-1] != '/': basedir = basedir + '/' if not os.path.exists(basedir): os.mkdir(basedir) logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) gitinfo = subprocess.check_output('cd ' + os.path.dirname(apercal.__file__) + '&& git describe --tag; cd', shell=True).strip() logger.info("Apercal version: " + gitinfo) logger.info( "start_apercal called with arguments targets={}; fluxcals={}; polcals={}" .format(targets, fluxcals, polcals)) logger.info("steps = {}".format(steps)) # number of beams to process n_beams = len(beamlist_target) # check the input config file # get the default configfile if none was provided if not configfilename: logger.info("No config file provided, getting default config") # create a list of config file name configfilename_list = [ os.path.join( basedir, "{0}_B{1}_Apercal_settings.cfg".format(taskid_target, str(beam).zfill(2))) for beam in beamlist_target ] # get the default config settings config = lib.get_default_config() # go through the config files and create them for beam_index in range(n_beams): with open(configfilename_list[beam_index], "w") as fp: config.write(fp) logger.info("Beam {} config file saved to {}".format( beamlist_target[beam_index], configfilename_list[beam_index])) # if configfile(s) are given as a list elif type(configfilename) is list: # if it is just one, create copies for each beam in the base directory if len(configfilename) == 1: logger.info( "A single config file was provided. Creating copies of {}". format(configfilename[0])) configfilename_list = [ os.path.join( basedir, "{0}_B{1}_Apercal_settings.cfg".format( taskid_target, str(beam).zfill(2))) for beam in beamlist_target ] # make the copies for config in configfilename_list: lib.basher("cp " + str(configfilename[0]) + " " + str(config)) elif len(configfilename) == n_beams: logger.info("Number of config files and target beams match.") configfilename_list = configfilename else: error = "Number of config files and target beams did not match. Abort" logger.error(error) raise RuntimeError(error) # if configfilename is just a string elif type(configfilename) is str: logger.info( "A single config file was provided. Creating copies of {}".format( configfilename)) configfilename_list = [ os.path.join( basedir, "{0}_B{1}_Apercal_settings.cfg".format(taskid_target, str(beam).zfill(2))) for beam in beamlist_target ] # make the copies for config in configfilename_list: lib.basher("cp " + str(configfilename) + " " + str(config)) else: error = "Unknown input for configfilename. Abort" logger.error(error) raise RuntimeError(error) status = pymp.shared.dict({beamnr: [] for beamnr in beamlist_target}) if fluxcals: name_fluxcal = str(fluxcals[0][1]).strip().split('_')[0].upper() else: name_fluxcal = '' if polcals: name_polcal = str(polcals[0][1]).strip().split('_')[0].upper() else: name_polcal = '' name_target = str(name_target).strip() # .upper() # If both fluxcal and polcal polarized, remove polcal if subs_calmodels.is_polarised( name_polcal) and subs_calmodels.is_polarised(name_fluxcal): name_polcal = "" if (fluxcals and fluxcals != '') and (polcals and polcals != ''): assert (len(fluxcals) == len(polcals)) # avoid symmetry bias, if there is only a polcal but no fluxcal, switch them if fluxcals is None and polcals is not None: logger.info( "Only polcal was provided. Setting polcal {} to fluxcal".format( name_polcal)) fluxcals, polcals = polcals, fluxcals name_polcal = "" # Exchange polcal and fluxcal if specified in the wrong order elif not subs_calmodels.is_polarised(name_polcal) and name_polcal != '': if subs_calmodels.is_polarised(name_fluxcal): logger.info("Switching polcal and fluxcal because " + name_polcal + " is not polarised") fluxcals, polcals = polcals, fluxcals name_polcal = str(polcals[0][1]).strip() else: logger.info("Setting polcal to '' since " + name_polcal + " is not polarised") name_polcal = "" elif name_polcal != '': logger.info("Polcal " + name_polcal + " is polarised, all good") def name_to_ms(name): if not name: return '' elif '3C' in name: return name.upper().strip().split('_')[0] + '.MS' else: return name + '.MS' def name_to_mir(name): if not name: return '' elif '3C' in name: return name.upper().strip().split('_')[0] + '.mir' else: return name + '.mir' def set_files(p): """ Set the basedir, fluxcal, polcal, target properties Args: p (BaseModule): apercal step object (e.g. prepare) Returns: None """ p.basedir = basedir p.fluxcal = name_to_ms(name_fluxcal) p.polcal = name_to_ms(name_polcal) p.target = name_to_ms(name_target) # debug_msg = """ # p.basedir = basedir = {0}; # p.fluxcal = name_to_ms(name_fluxcal) = {1}; # p.polcal = name_to_ms(name_polcal) = {2}; # p.target = name_to_ms(name_target) = {3}; # """.format(basedir, name_to_ms(name_fluxcal), name_to_ms(name_polcal), name_to_ms(name_target)) # logger.debug(debug_msg) beamnrs_fluxcal = [f[2] for f in fluxcals] if len(fluxcals) > 1: # Check every target beam has a fluxcal beam for beamnr_target in beamlist_target: assert (beamnr_target in beamnrs_fluxcal) # creating a copy of the target beamlist as a normal array # to avoid using np.where() for such a small thing if type(beamlist_target) == np.ndarray: beamlist_target_for_config = beamlist_target.tolist() else: beamlist_target_for_config = beamlist_target time_start = time() try: # ======= # Prepare # ======= # keep a start-finish record of step in the main log file if "prepare" in steps: logger.info("Running prepare") start_time_prepare = time() else: logger.info("Skipping prepare") # Prepare fluxcals for (taskid_fluxcal, name_fluxcal, beamnr_fluxcal) in fluxcals: p0 = prepare(file_=configfilename_list[ beamlist_target_for_config.index(beamnr_fluxcal)]) p0.basedir = basedir #set_files(p0) p0.prepare_flip_ra = flip_ra # the following two need to be empty strings for prepare p0.fluxcal = '' p0.polcal = '' p0.target = name_to_ms(name_fluxcal) p0.prepare_target_beams = str(beamnr_fluxcal) p0.prepare_date = str(taskid_fluxcal)[:6] p0.prepare_obsnum_target = validate_taskid(taskid_fluxcal) if "prepare" in steps and not dry_run: try: p0.go() except Exception as e: logger.warning("Prepare failed for fluxcal " + str(taskid_fluxcal) + " beam " + str(beamnr_fluxcal)) logger.exception(e) if 'prepare' in steps: # copy the param file generated here param_file = os.path.join(basedir, 'param.npy') director(p0, 'rn', param_file.replace( ".npy", "_prepare_{}.npy".format(name_fluxcal.split('_')[0])), file_=param_file, ignore_nonexistent=True) # Prepare polcals if name_polcal != '': for (taskid_polcal, name_polcal, beamnr_polcal) in polcals: p0 = prepare(file_=configfilename_list[ beamlist_target_for_config.index(beamnr_polcal)]) p0.basedir = basedir #set_files(p0) p0.prepare_flip_ra = flip_ra # the following two need to be empty strings for prepare p0.fluxcal = '' p0.polcal = '' p0.target = name_to_ms(name_polcal) p0.prepare_target_beams = str(beamnr_polcal) p0.prepare_date = str(taskid_polcal)[:6] p0.prepare_obsnum_target = validate_taskid(taskid_polcal) if "prepare" in steps and not dry_run: try: p0.go() except Exception as e: logger.warning("Prepare failed for polcal " + str(taskid_polcal) + " beam " + str(beamnr_polcal)) logger.exception(e) if 'prepare' in steps: # copy the param file generated here param_file = os.path.join(basedir, 'param.npy') director(p0, 'rn', param_file.replace( ".npy", "_prepare_{}.npy".format( name_polcal.split('_')[0])), file_=param_file, ignore_nonexistent=True) # Prepare target for beamnr in beamlist_target: p0 = prepare(file_=configfilename_list[ beamlist_target_for_config.index(beamnr)]) p0.basedir = basedir # set_files(p0) p0.prepare_flip_ra = flip_ra # the following two need to be empty strings for prepare p0.fluxcal = '' p0.polcal = '' p0.target = name_to_ms(name_target) p0.prepare_date = str(taskid_target)[:6] p0.prepare_obsnum_target = validate_taskid(taskid_target) p0.prepare_target_beams = ','.join( ['{:02d}'.format(beamnr) for beamnr in beamlist_target]) if "prepare" in steps and not dry_run: try: p0.go() except Exception as e: logger.warning("Prepare failed for target " + str(taskid_target) + " beam " + str(beamnr)) logger.exception(e) status[beamnr] += ['prepare'] # keep a start-finish record of step in the main log file if "prepare" in steps: logger.info("Running prepare ... Done ({0:.0f}s)".format( time() - start_time_prepare)) # copy the param file generated here param_file = os.path.join(basedir, 'param.npy') director(p0, 'rn', param_file.replace(".npy", "_prepare_{}.npy".format(name_target)), file_=param_file, ignore_nonexistent=True) # ===== # Split # ===== # keep a start-finish record of step in the main log file if 'split' in steps: logger.info("Running split") start_time_split = time() else: logger.info("Skipping split") # Splitting a small chunk of data for quicklook pipeline # at the moment it all relies on the target beams # what if there are more calibrator than target beams-> realistic? with pymp.Parallel(5) as p: for beam_index in p.range(n_beams): beamnr = beamlist_target[beam_index] # individual logfiles for each process logfilepath = os.path.join(basedir, 'apercal{:02d}.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.debug("Starting logfile for beam " + str(beamnr)) try: s0 = split(file_=configfilename_list[beam_index]) set_files(s0) s0.beam = "{:02d}".format(beamnr) if "split" in steps and not dry_run: s0.go() except Exception as e: logger.warning("Split failed for {0} beam {1}".format( str(taskid_target), str(beamnr))) logger.exception(e) # not sure if following line is necessary status[beamnr] += ['split'] # keep a start-finish record of step in the main log file if "split" in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info( "Running split ... Done ({0:.0f}s)".format(time() - start_time_split)) # copy the param file generated here # param_file = os.path.join(basedir, 'param.npy') # director( # p0, 'rn', param_file.replace(".npy", "_split.npy"), file_=param_file, ignore_nonexistent=True) # ======= # Preflag # ======= # keep a record of the parallalised step in the main log file if "preflag" in steps: logger.info("Running preflag") start_time_preflag = time() else: logger.info("Skipping preflag") # In order to run in parallel, the bandpass table needs to exists # doing it here is not elegant but requires the least amount of changes # to preflage # with pymp.Parallel(10) as p: # for beam_index in p.range(n_beams): # beamnr = beamlist_target[beam_index] # # individual logfiles for each process # logfilepath = os.path.join( # basedir, 'apercal{:02d}.log'.format(beamnr)) # lib.setup_logger('debug', logfile=logfilepath) # logger = logging.getLogger(__name__) # logger.debug("Starting logfile for beam " + str(beamnr)) # p1 = preflag(filename=configfilename) # p1.paramfilename = 'param_{:02d}.npy'.format(beamnr) # p1.basedir = basedir # p1.fluxcal = '' # p1.polcal = '' # p1.target = name_to_ms(name_fluxcal) # p1.beam = "{:02d}".format(beamnr) # p1.preflag_targetbeams = "{:02d}".format(beamnr) # if "preflag" in steps and not dry_run: # try: # bandpass_start_time = time() # logger.info("Running aoflagger bandpass for flux calibrator {0} in beam {1}".format( # p1.target, p1.beam)) # # director( # # p1, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) # p1.go() # # director(p1, 'rm', basedir + '/param.npy', # # ignore_nonexistent=True) # # it is necessary to move the param files in order to keep them # param_file = basedir + \ # '/param_{:02d}.npy'.format(beamnr) # director( # p1, 'mv', param_file, file_=param_file.replace(".npy", "_preflag_{0}.npy".format(name_fluxcal)), ignore_nonexistent=True) # p1.aoflagger_bandpass() # except Exception as e: # logger.warning("Running aoflagger bandpass for flux calibrator {0} in beam {1} ... Failed ({2:.0f}s)".format( # p1.target, p1.beam, time() - bandpass_start_time)) # logger.exception(e) # status[beamnr] += ['preflag_bandpass'] # else: # logger.info("Running aoflagger bandpass for flux calibrator {0} in beam {1} ... Done ({2:.0f}s)".format( # p1.target, p1.beam, time() - bandpass_start_time)) # Flag fluxcal (pretending it's a target, parallelised version) # 5 in parallel with pymp.Parallel(5) as p: for beam_index in p.range(n_beams): beamnr = beamlist_target[beam_index] # individual logfiles for each process logfilepath = os.path.join(basedir, 'apercal{:02d}.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.debug("Starting logfile for beam " + str(beamnr)) try: p1 = preflag(filename=configfilename_list[beam_index]) p1.paramfilename = 'param_{:02d}.npy'.format(beamnr) p1.basedir = basedir p1.fluxcal = '' p1.polcal = '' p1.target = name_to_ms(name_fluxcal) p1.beam = "{:02d}".format(beamnr) p1.preflag_targetbeams = "{:02d}".format(beamnr) if beam_index < 2: p1.preflag_aoflagger_threads = 9 else: p1.preflag_aoflagger_threads = 10 if "preflag" in steps and not dry_run: logger.info( "Running preflag for flux calibrator {0} in beam {1}" .format(p1.target, p1.beam)) preflag_flux_cal_start_time = time() # director( # p1, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) p1.go() # it is necessary to move the param files in order to keep them param_file = os.path.join( basedir, 'param_{:02d}.npy'.format(beamnr)) director(p1, 'rn', param_file.replace( ".npy", "_preflag_{0}.npy".format( name_fluxcal.split('_')[0])), file_=param_file, ignore_nonexistent=True) logger.info( "Running preflag for flux calibrator {0} in beam {1} ... Done ({2:.0f}s)" .format(p1.target, p1.beam, time() - preflag_flux_cal_start_time)) except Exception as e: logger.warning( "Running preflag for flux calibrator {0} in beam {1} ... Failed ({2:.0f}s)" .format(p1.target, p1.beam, time() - preflag_flux_cal_start_time)) logger.exception(e) status[beamnr] += ['preflag'] # Flag polcal (pretending it's a target, parallel version) # 5 in parallel with pymp.Parallel(5) as p: for beam_index in p.range(n_beams): beamnr = beamlist_target[beam_index] # individual logfiles for each process logfilepath = os.path.join(basedir, 'apercal{:02d}.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.debug("Starting logfile for beam " + str(beamnr)) try: p1 = preflag(filename=configfilename_list[beam_index]) # remove next line in final version p1.preflag_aoflagger_version = 'local' p1.basedir = basedir p1.paramfilename = 'param_{:02d}.npy'.format(beamnr) p1.basedir = basedir if name_polcal != '': p1.fluxcal = '' p1.polcal = '' p1.target = name_to_ms(name_polcal) p1.beam = "{:02d}".format(beamnr) p1.preflag_targetbeams = "{:02d}".format(beamnr) if beam_index < 2: p1.preflag_aoflagger_threads = 9 else: p1.preflag_aoflagger_threads = 10 if "preflag" in steps and not dry_run: logger.info( "Running preflag for pol calibrator {0} in beam {1}" .format(p1.target, p1.beam)) preflag_pol_cal_start_time = time() # director( # p1, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) p1.go() # it is necessary to move the param files in order to keep them param_file = os.path.join( basedir, 'param_{:02d}.npy'.format(beamnr)) director(p1, 'rn', param_file.replace( ".npy", "_preflag_{0}.npy".format( name_polcal.split('_')[0])), file_=param_file, ignore_nonexistent=True) logger.info( "Running preflag for pol calibrator {0} in beam {1} ... Done ({2:.0f}s)" .format(p1.target, p1.beam, time() - preflag_pol_cal_start_time)) except Exception as e: logger.warning( "Running preflag for pol calibrator {0} in beam {1} ... Failed ({2:.0f}s)" .format(p1.target, p1.beam, time() - preflag_pol_cal_start_time)) logger.exception(e) status[beamnr] += ['preflag'] # Flag target # 5 in parallel with pymp.Parallel(5) as p: for beam_index in p.range(n_beams): beamnr = beamlist_target[beam_index] # individual logfiles for each process logfilepath = os.path.join(basedir, 'apercal{:02d}.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.debug("Starting logfile for beam " + str(beamnr)) try: p1 = preflag(filename=configfilename_list[beam_index]) # remove next line in final version p1.preflag_aoflagger_version = 'local' p1.paramfilename = 'param_{:02d}.npy'.format(beamnr) p1.basedir = basedir p1.fluxcal = '' p1.polcal = '' p1.target = name_to_ms(name_target) p1.beam = "{:02d}".format(beamnr) p1.preflag_targetbeams = "{:02d}".format(beamnr) if beam_index < 2: p1.preflag_aoflagger_threads = 9 else: p1.preflag_aoflagger_threads = 10 if "preflag" in steps and not dry_run: logger.info( "Running preflag for target {0} in beam {1}". format(p1.target, p1.beam)) preflag_target_start_time = time() # director( # p1, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) p1.go() # it is necessary to move the param files in order to keep them param_file = os.path.join( basedir, 'param_{:02d}.npy'.format(beamnr)) director(p1, 'rn', param_file.replace( ".npy", "_preflag_{0}.npy".format(name_target)), file_=param_file, ignore_nonexistent=True) logger.info( "Running preflag for target {0} in beam {1} ... Done ({2:.0f}s)" .format(p1.target, p1.beam, time() - preflag_target_start_time)) except Exception as e: logger.info( "Running preflag for target {0} in beam {1} ... Failed ({2:.0f}s)" .format(p1.target, p1.beam, time() - preflag_target_start_time)) logger.exception(e) status[beamnr] += ['preflag'] # keep a record of the parallalised step in the main log file if "preflag" in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Running preflag ... Done ({0:.0f}s)".format( time() - start_time_preflag)) # =============== # Crosscal # =============== # keep a record of the parallalised step in the main log file if 'ccal' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Running crosscal") start_time_crosscal = time() else: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Skipping crosscal") if len(fluxcals) == 1 and fluxcals[0][-1] == 0 and n_beams > 1: raise ApercalException( "Sorry, one fluxcal is not supported anymore at the moment") with pymp.Parallel(10) as p: for beam_index in p.range(n_beams): beamnr = beamlist_target[beam_index] logfilepath = os.path.join(basedir, 'apercal{:02d}.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.debug("Starting logfile for beam " + str(beamnr)) try: p2 = ccal(file_=configfilename_list[beam_index]) p2.paramfilename = 'param_{:02d}.npy'.format(beamnr) set_files(p2) p2.beam = "{:02d}".format(beamnr) p2.crosscal_transfer_to_target_targetbeams = "{:02d}".format( beamnr) if "ccal" in steps and not dry_run: # director( # p2, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) p2.go() # it is necessary to move the param files in order to keep them param_file = os.path.join( basedir, 'param_{:02d}.npy'.format(beamnr)) director(p2, 'rn', param_file.replace(".npy", "_crosscal.npy"), file_=param_file, ignore_nonexistent=True) except Exception as e: # Exception was already logged just before logger.warning( "Failed beam {}, skipping that from crosscal".format( beamnr)) logger.exception(e) status[beamnr] += ['crosscal'] # keep a record of the parallalised step in the main log file if 'ccal' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Running crosscal ... Done ({0:.0f}s)".format( time() - start_time_crosscal)) # ======= # Convert # ======= # keep a record of the parallalised step in the main log file if 'convert' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Running convert") start_time_convert = time() else: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Skipping convert") # 5 threads to not hammer the disks too much, convert is only IO with pymp.Parallel(5) as p: for beam_index in p.range(n_beams): beamnr = beamlist_target[beam_index] logfilepath = os.path.join(basedir, 'apercal{:02d}.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) try: p3 = convert(file_=configfilename_list[beam_index]) p3.paramfilename = 'param_{:02d}.npy'.format(beamnr) set_files(p3) p3.beam = "{:02d}".format(beamnr) p3.convert_targetbeams = "{:02d}".format(beamnr) if "convert" in steps and not dry_run: # director( # p3, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) p3.go() # it is necessary to move the param files in order to keep them param_file = os.path.join( basedir, 'param_{:02d}.npy'.format(beamnr)) director(p3, 'rn', param_file.replace(".npy", "_convert.npy"), file_=param_file, ignore_nonexistent=True) # director( # p3, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) except Exception as e: logger.warning( "Failed beam {}, skipping that from convert".format( beamnr)) logger.exception(e) status[beamnr] += ['convert'] if 'convert' in steps: # keep a record of the parallalised step in the main log file logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Running convert ... Done ({0:.0f}s)".format( time() - start_time_convert)) # ================================== # Selfcal + Continuum + Polarisation # ================================== # keep a record of the parallalised step in the main log file if 'scal' in steps or 'continuum' in steps or 'polarisation' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Running selfcal and/or continuum and/or polarisation") start_time_selfcal_continuum_polarisation = time() else: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Skipping selfcal and continuum and polarisation") with pymp.Parallel(10) as p: for beam_index in p.range(n_beams): beamnr = beamlist_target[beam_index] logfilepath = os.path.join(basedir, 'apercal{:02d}.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) try: p4 = scal(file_=configfilename_list[beam_index]) p4.paramfilename = 'param_{:02d}.npy'.format(beamnr) p4.basedir = basedir p4.beam = "{:02d}".format(beamnr) p4.target = name_target + '.mir' if "scal" in steps and not dry_run: p4.go() except Exception as e: # Exception was already logged just before logger.warning( "Failed beam {}, skipping that from scal".format( beamnr)) logger.exception(e) status[beamnr] += ['scal'] try: p5 = continuum(file_=configfilename_list[beam_index]) p5.paramfilename = 'param_{:02d}.npy'.format(beamnr) p5.basedir = basedir p5.beam = "{:02d}".format(beamnr) p5.target = name_target + '.mir' if "continuum" in steps and not dry_run: p5.go() except Exception as e: # Exception was already logged just before logger.warning( "Failed beam {}, skipping that from continuum".format( beamnr)) logger.exception(e) status[beamnr] += ['continuum'] try: p6 = polarisation(file_=configfilename_list[beam_index]) p6.paramfilename = 'param_{:02d}.npy'.format(beamnr) p6.basedir = basedir p6.beam = "{:02d}".format(beamnr) p6.polcal = name_to_mir(name_polcal) p6.target = name_to_mir(name_target) if "polarisation" in steps and not dry_run: p6.go() except Exception as e: # Exception was already logged just before logger.warning( "Failed beam {}, skipping that from polarisation". format(beamnr)) logger.exception(e) status[beamnr] += ['polarisation'] # keep a record of the parallalised step in the main log file if 'scal' in steps or 'continuum' in steps or 'polarisation' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info( "Running selfcal and/or continuum and/or polarisation ... Done ({0:.0f}s)" .format(time() - start_time_selfcal_continuum_polarisation)) # ==== # Line # ==== # keep a record of the parallalised step in the main log file if 'line' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger.info("Running line") start_time_line = time() else: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger.info("Skipping line") for beamnr in beamlist_target: # Because of the amount of information coming from line # this module gets its own logfile logfilepath = os.path.join(basedir, 'apercal{:02d}_line.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) try: p7 = line(file_=configfilename_list[ beamlist_target_for_config.index(beamnr)]) if beamnr not in p7.line_beams: logger.debug( "Skipping line imaging for beam {}".format(beamnr)) continue p7.basedir = basedir p7.beam = "{:02d}".format(beamnr) p7.target = name_target + '.mir' if "line" in steps and not dry_run: p7.go() except Exception as e: # Exception was already logged just before logger.warning( "Failed beam {}, skipping that from line".format(beamnr)) logger.exception(e) status[beamnr] += ['line'] # with pymp.Parallel(5) as p: # for beam_index in p.range(n_beams): # beamnr = beamlist_target[beam_index] # logfilepath = os.path.join( # basedir, 'apercal{:02d}.log'.format(beamnr)) # lib.setup_logger('debug', logfile=logfilepath) # logger = logging.getLogger(__name__) # try: # p7 = line(file_=configfilename) # if beamnr not in p7.line_beams: # logger.debug( # "Skipping line imaging for beam {}".format(beamnr)) # continue # p7.basedir = basedir # p7.beam = "{:02d}".format(beamnr) # p7.target = name_target + '.mir' # if "line" in steps and not dry_run: # p7.go() # except Exception as e: # # Exception was already logged just before # logger.warning( # "Failed beam {}, skipping that from line".format(beamnr)) # logger.exception(e) # status[beamnr] += ['line'] # keep a record of the parallalised step in the main log file if 'line' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info( "Running line ... Done ({0:.0f}s)".format(time() - start_time_line)) # ======== # Transfer # ======== # keep a record of the parallalised step in the main log file if 'transfer' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger.info("Running transfer") start_time_transfer = time() else: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger.info("Skipping transfer") # 5 threads to not hammer the disks too much during copying with pymp.Parallel(5) as p: for beam_index in p.range(n_beams): beamnr = beamlist_target[beam_index] logfilepath = os.path.join(basedir, 'apercal{:02d}.log'.format(beamnr)) lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) try: p8 = transfer(file_=configfilename_list[beam_index]) p8.paramfilename = 'param_{:02d}.npy'.format(beamnr) p8.basedir = basedir p8.target = name_target + '.mir' p8.beam = "{:02d}".format(beamnr) if "transfer" in steps and not dry_run: # director( # p8, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) p8.go() except Exception as e: logger.warning( "Failed beam {}, skipping that from transfer".format( beamnr)) logger.exception(e) status[beamnr] += ['transfer'] # keep a record of the parallalised step in the main log file if 'transfer' in steps: logfilepath = os.path.join(basedir, 'apercal.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) logger.info("Running transfer ... Done ({0:.0f}s)".format( time() - start_time_transfer)) # Polarisation # ============ # keep a record of the parallalised step in the main log file # if 'polarisation' in steps: # logfilepath = os.path.join(basedir, 'apercal.log') # lib.setup_logger('debug', logfile=logfilepath) # logger = logging.getLogger(__name__) # logger.info("Running polarisation") # start_time_polarisation = time() # else: # logfilepath = os.path.join(basedir, 'apercal.log') # lib.setup_logger('debug', logfile=logfilepath) # logger = logging.getLogger(__name__) # logger.info("Skipping polarisation") # with pymp.Parallel(5) as p: # for beam_index in p.range(n_beams): # beamnr = beamlist_target[beam_index] # logfilepath = os.path.join( # basedir, 'apercal{:02d}.log'.format(beamnr)) # lib.setup_logger('debug', logfile=logfilepath) # logger = logging.getLogger(__name__) # try: # p7 = polarisation(file_=configfilename) # p7.paramfilename = 'param_{:02d}.npy'.format(beamnr) # p7.basedir = basedir # p7.beam = "{:02d}".format(beamnr) # p7.target = name_to_mir(name_target) # if "polarisation" in steps and not dry_run: # p7.go() # except Exception as e: # # Exception was already logged just before # logger.warning( # "Failed beam {}, skipping that from polarisation".format(beamnr)) # logger.exception(e) # status[beamnr] += ['polarisation'] # # keep a record of the parallalised step in the main log file # if 'polarisation' in steps: # logfilepath = os.path.join(basedir, 'apercal.log') # lib.setup_logger('debug', logfile=logfilepath) # logger = logging.getLogger(__name__) # logger.info("Running polarisation ... Done ({0:.0f}s)".format( # time() - start_time_polarisation)) # if "ccalqa" in steps and not dry_run: # logger.info("Starting crosscal QA plots") # try: # make_all_ccal_plots( # taskid_target, name_fluxcal.upper().strip().split('_')[0]) # except Exception as e: # logger.warning("Failed crosscal QA plots") # logger.exception(e) # logger.info("Done with crosscal QA plots") status = status.copy() # Convert pymp shared dict to a normal one msg = "Apercal finished after " + \ str(timedelta(seconds=time() - time_start)) logger.info(msg) return status, str(timedelta(seconds=time() - time_start)), None except Exception as e: msg = "Apercal threw an error after " + \ str(timedelta(seconds=time() - time_start)) logger.exception(msg) return status, str(timedelta(seconds=time() - time_start)), str(e)
def apercc(cal_list=None, task_id=None, cal_name=None, base_dir=None, search_all_nodes=False, steps=None): """ Main function to run the cross-calibration stability evaluation. For a list of calibrator scans or a given task id and calibrator name, this functions runs the cross-calibration evaluation. The function can get the data from ALTA and flags them using the Apercal modules prepare and preflag. It compares the bandpass solutions and gain factors between beams and between observations of the same calibrators The different steps can be selected individually. Example: scanid, source name, beam: [190108926, '3C147_36', 36] steps: ['prepare', 'preflag', 'crosscal', bpass_compare', 'gain_comare', 'bpass_compare_obs', 'gain_compare_obs'] function cal: apercc(cal_list=[[190108926, '3C147_36', 36], [190108927, '3C147_37', 37]) or apercc(task_id = 190409056, cal_name='3C196') Args: cal_list (List(List(int, str, int)): scan id, source name, beam, optional base_dir (str): Name of directory to store the data, if not specified it will be /data/apertif/crosscal/<scanid> when new data is fetched or /data/apertif/<scanid> when existing data is used task_id (int): ID of scan to be used as main ID and for the directory, if not specified it will be the first scan id cal_name (str): Name of the calibrator, if not specified the first name in the calibrator list will be used search_all_nodes (bool): steps (List(str)): List of steps in this task To Do: Use existing data using the task_id option and the name of the calibrator? """ # General setup # ============= # start time of this function start_time = time() # check input # if no list of a calibrators is given cal_list_mode = True if cal_list is None: # then it needs the task id and the calibrator name to look for existing data if task_id is not None and cal_name is not None: print( "Using task id and calibrator name. Assuming to use existing data. Will not run preflag, prepare and crosscal" ) # check that if steps were provided, they don't contain preflag, prepare and crosscal if steps is not None: if 'prepare' in steps: steps.remove('prepare') if 'preflag' in steps: steps.remove('preflag') if 'crosscal' in steps: steps.remove('crosscal') else: steps = [ 'bpass_compare', 'gain_comare', 'bpass_compare_obs', 'gain_compare_obs' ] # using existing data cal_list_mode = False # otherwise it won't do anything else: print( "Input parameters incorrect. Please specify either cal_list or task_id and cal_name. Abort" ) return -1 else: print("Using list of calibrators") if not steps: steps = [ 'prepare', 'preflag', 'crosscal', 'bpass_compare', 'gain_comare', 'bpass_compare_obs', 'gain_compare_obs' ] # # check that preflag is in it if prepare is run # else: # if 'prepare' in steps and not 'preflag' in steps: # steps.insert(1, 'preflag') # get the scan id to be used as the task id if not task_id: task_id = cal_list[0][0] else: task_id = task_id # create data directory unless specified using the first id unless otherwise specified # if no directory is specified if not base_dir: # and existing data is used, assume the base dir is an apercal processed data directory if task_id is not None and cal_name is not None: base_dir = '/data/apertif/{}/'.format(task_id) # if new data is fetched from the archive use a different default directory else: base_dir = '/data/apertif/crosscal/{}/'.format(task_id) elif len(base_dir) > 0 and base_dir[-1] != '/': base_dir = base_dir + '/' if not os.path.exists(base_dir) and cal_list_mode: try: os.mkdir(base_dir) except Exception as e: print("Creating the base directory failed. Abort") return -1 elif not os.path.exists(base_dir): print("Directory was not found. Abort") return -1 logfilepath = os.path.join(base_dir, 'apercc.log') lib.setup_logger('debug', logfile=logfilepath) logger = logging.getLogger(__name__) # gitinfo = subprocess.check_output('cd ' + os.path.dirname(apercal.__file__) + # '&& git describe --tag; cd', shell=True).strip() # logger.info("Apercal version: " + gitinfo) logger.info("Apertif cross-calibration stability evaluation") if cal_list_mode: logger.info("Using list of calibrators as input !!!!") else: logger.info("Using task id and calibrator name as input !!!!") logger.debug("apercc called with arguments ...") logger.debug("cal_list={}".format(cal_list)) logger.debug("task_id = {}".format(task_id)) logger.debug("cal_name = {}".format(cal_name)) logger.debug("base_dir = {}".format(base_dir)) logger.debug("search_all_nodes = {}".format(search_all_nodes)) logger.debug("steps = {}".format(steps)) # number of calibrators if cal_list is not None: n_cals = len(cal_list) # get a list of beams beam_list = np.array([cal_list[k][2] for k in range(n_cals)]) else: n_cals = 1 # get the name of the flux calibrator if cal_name is None: name_cal = str(cal_list[0][1]).strip().split('_')[0] else: name_cal = cal_name # Getting the data using prepare # ============================== if "prepare" in steps: start_time_prepare = time() logger.info("Getting data for calibrators") # go through the list of calibrators and run prepare for (task_id_cal, name_cal, beamnr_cal) in cal_list: logger.info("Running prepare for {0} of beam {1}".format( name_cal, beamnr_cal)) # create prepare object without config file prep = prepare(filename=None) # where to store the data prep.basedir = base_dir # give the calibrator as a target to prepare prep.fluxcal = '' prep.polcal = '' prep.target = name_cal.upper().strip().split('_')[0] + '.MS' prep.prepare_target_beams = str(beamnr_cal) prep.prepare_date = str(task_id_cal)[:6] prep.prepare_obsnum_target = str(task_id_cal)[-3:] try: prep.go() except Exception as e: logger.warning( "Prepare failed for calibrator {0} ({1}) beam {2}".format( str(task_id), name_cal, beamnr_cal)) logger.exception(e) else: logger.info("Prepare successful for {0} of beam {1}".format( name_cal, beamnr_cal)) logger.info("Getting data for calibrators ... Done ({0:.0f}s)".format( time() - start_time_prepare)) else: logger.info("Skipping getting data for calibrators") # Running preflag for calibrators # =============================== if 'preflag' in steps: start_time_flag = time() logger.info("Flagging data of calibrators") # Flag fluxcal (pretending it's a target) # needs to be changed for parallel preflag and make it a loop flag = preflag(filename=None) flag.basedir = base_dir flag.fluxcal = '' flag.polcal = '' flag.target = name_cal.upper().strip().split('_')[0] + '.MS' flag.beam = "{:02d}".format(beam_list[0]) flag.preflag_targetbeams = "{:02d}".format(beamnr) try: director(flag, 'rm', base_dir + '/param.npy', ignore_nonexistent=True) flag.go() except Exception as e: logger.warning("Preflag failed") logger.exception(e) else: logger.info( "Flagging data of calibrators ... Done ({0:.0f}s)".format( time() - start_time_flag)) else: logger.info("Skipping running preflag for calibrators") # Running crosscal for calibrators # =============================== if 'crosscal' in steps: start_time_crosscal = time() logger.info("Running crosscal for calibrators") for beam_nr in beam_list: logger.info("Running crosscal for beam {0}".format(beam_nr)) crosscal = ccal(file_=None) crosscal.basedir = base_dir crosscal.fluxcal = name_cal.upper().strip().split('_')[0] + '.MS' # p.polcal = name_to_ms(name_polcal) # p.target = name_to_ms(name_target) # p2.paramfilename = 'param_{:02d}.npy'.format(beamnr) crosscal.beam = "{:02d}".format(beam_nr) crosscal.crosscal_transfer_to_target = False # p2.crosscal_transfer_to_target_targetbeams = "{:02d}".format( # beamnr) try: director(crosscal, 'rm', base_dir + '/param.npy', ignore_nonexistent=True) # director( # p2, 'rm', basedir + '/param_{:02d}.npy'.format(beamnr), ignore_nonexistent=True) crosscal.go() except Exception as e: # Exception was already logged just before logger.warning("Failed crosscal for beam {}".format(beam_nr)) logger.exception(e) else: logger.info( "Running crosscal for beam {0} ... Done".format(beam_nr)) logger.info( "Running crosscal for calibrators ... Done ({0:.0f}s)".format( time() - start_time_crosscal)) else: logger.info("Skipping running crosscal for calibrators") # Running Bandbpass comparison # ============================ if 'bpass_compare' in steps: start_time_prepare = time() logger.info("Comparing bandpass") logger.info("#### Doing nothing here yet ####") logger.info( "Comparing bandpass ... Done ({0:.0f})".format(time() - start_time_prepare)) else: logger.info("Skipping comparing bandpass") # Running Bandbpass comparison # ============================ if 'gain_compare' in steps: start_time_gain = time() logger.info("Comparing gain solutions") logger.info("#### Doing nothing here yet ####") logger.info("Comparing gain solutions ... Done ({0:.0f})".format( time() - start_time_gain)) else: logger.info("Skipping comparing gain solutions") # Running Bandbpass comparison between observations # ================================================= if 'bpass_compare_obs' in steps: start_time_bandpass = time() logger.info("Comparing banpdass solutions across observations") logger.info("#### Doing nothing here yet ####") logger.info( "Comparing banpdass solutions across observations ... Done ({0:.0f})" .format(time() - start_time_bandpass)) else: logger.info( "Skipping comparing banpdass solutions across observations") # Running Bandbpass comparison between observations # ================================================= if 'bpass_compare_obs' in steps: start_time_gain = time() logger.info("Comparing banpdass solutions across observations") logger.info("#### Doing nothing here yet ####") logger.info( "Comparing banpdass solutions across observations ... Done ({0:.0f})" .format(time() - start_time_gain)) else: logger.info( "Skipping comparing banpdass solutions across observations") logger.info( "Apertif cross-calibration stability evaluation ... Done ({0:.0f}s)". format(time() - start_time))