def upload_cal_files(obsid, cal_id, cal_dir_to_tar, srclist, caltype=2): """ Uploads the calibrator solutions to the MWA pulsar database Parameters: ----------- obsid: int The observation ID cal_id: int The calibrator ID cal_dir_to_tar: string The location of the calibrator RTS directory srclist: string The path to the sourcelist caltype: int The type of calibrator. Default: 2 """ auth, web_address = get_db_auth_addr() client_files_dict = client.calibration_file_by_observation_id(web_address, auth, obsid=cal_id) if client_files_dict: logger.info( "This calibrator already has solutions on the database. Not uploading" ) else: zip_loc = zip_calibration_files(cal_dir_to_tar, cal_id, srclist) # Check if calibrator has to be made cal_list = client.calibrator_list(web_address, auth) if not cal_id in [c['observationid'] for c in cal_list]: client.calibrator_create(web_address, auth, observationid=str(cal_id)) # Upload Calibration file try: client.calibrator_file_upload(web_address, auth, observationid=str(cal_id), filepath=str(zip_loc), caltype=2) logger.info( "Uploaded calibrator solutions from {} to the database".format( cal_id)) except: logger.warn("Failed to upload calibration files") os.remove(zip_loc)
def flux_cal_and_submit(time_obs, metadata, bestprof_data, pul_ra, pul_dec, coh, auth, pulsar=None, trcvr=data_load.TRCVR_FILE): """ time_obs: the time in seconds of the dectection from the metadata metadata: list from the function get_obs_metadata bestprof_data: list from the function get_from_bestprof trcvr: the file location of antena temperatures """ #unpack the bestprof_data #[obsid, pulsar, dm, period, period_uncer, obsstart, obslength, profile, bin_num] obsid, prof_psr, dm, period, _, beg, t_int, profile, num_bins = bestprof_data if not pulsar: pulsar = prof_psr period = float(period) num_bins = int(num_bins) #get r_sys and gain t_sys, _, gain, u_gain = snfe.find_t_sys_gain(pulsar, obsid, obs_metadata=metadata,\ beg=beg, end=(t_int + beg - 1)) #estimate S/N try: prof_dict = prof_utils.auto_gfit(profile,\ period = period, plot_name="{0}_{1}_{2}_bins_gaussian_fit.png".format(obsid, pulsar, num_bins)) except (prof_utils.ProfileLengthError, prof_utils.NoFitError) as _: prof_dict = None if not prof_dict: logger.info( "Profile couldn't be fit. Using old style of profile analysis") prof_dict = prof_utils.auto_analyse_pulse_prof(profile, period) if prof_dict: sn = prof_dict["sn"] u_sn = prof_dict["sn_e"] w_equiv_bins = prof_dict["w_equiv_bins"] u_w_equiv_bins = prof_dict["w_equiv_bins_e"] w_equiv_ms = period / num_bins * w_equiv_bins u_w_equiv_ms = period / num_bins * u_w_equiv_bins scattering = prof_dict[ "scattering"] * period / num_bins / 1000 #convert to seconds u_scattering = prof_dict["scattering_e"] * period / num_bins / 1000 scattered = prof_dict["scattered"] else: logger.warn("Profile could not be analysed using any methods") sn = None u_sn = None w_equiv_bins = None u_w_equiv_bins = None w_equiv_ms = None u_w_equiv_ms = None scattering = None u_scattering = None scattered = None S_mean = None u_S_mean = None else: sn = prof_dict["sn"] u_sn = prof_dict["sn_e"] w_equiv_bins = prof_dict["Weq"] u_w_equiv_bins = prof_dict["Weq_e"] w_equiv_ms = period / num_bins * w_equiv_bins u_w_equiv_ms = period / num_bins * u_w_equiv_bins scattering = prof_dict[ "Wscat"] * period / num_bins / 1000 #convert to seconds u_scattering = prof_dict["Wscat_e"] * period / num_bins / 1000 scattered = prof_dict["scattered"] if prof_dict: logger.info("Profile scattered? {0}".format(scattered)) logger.info("S/N: {0} +/- {1}".format(sn, u_sn)) logger.debug("Gain {0} K/Jy".format(gain)) logger.debug("Equivalent width in bins: {0}".format(w_equiv_bins)) logger.debug("T_sys: {0} K".format(t_sys)) logger.debug("Bandwidth: {0}".format(bandwidth)) logger.debug("Detection time: {0}".format(t_int)) logger.debug("NUmber of bins: {0}".format(num_bins)) if scattered == False: #final calc of the mean fluxdesnity in mJy S_mean = sn * t_sys / ( gain * math.sqrt(2. * float(t_int) * bandwidth)) *\ math.sqrt( w_equiv_bins / (num_bins - w_equiv_bins)) * 1000. #constants to make uncertainty calc easier S_mean_cons = t_sys / ( math.sqrt(2. * float(t_int) * bandwidth)) *\ math.sqrt( w_equiv_bins / (num_bins - w_equiv_bins)) * 1000. u_S_mean = math.sqrt( math.pow(S_mean_cons * u_sn / gain , 2) +\ math.pow(sn * S_mean_cons * u_gain / math.pow(gain,2) , 2) ) logger.info('Smean {0:.2f} +/- {1:.2f} mJy'.format( S_mean, u_S_mean)) else: logger.info("Profile is scattered. Flux cannot be estimated") S_mean = None u_S_mean = None #calc obstype if (maxfreq - minfreq) == 23: obstype = 1 else: obstype = 2 subbands = get_subbands(metadata) web_address, auth = get_db_auth_addr() #get cal id if coh: cal_list = client.calibrator_list(web_address, auth) cal_already_created = False for c in cal_list: if (c[u'observationid'] == int( args.cal_id)) and (c[u'caltype'] == calibrator_type): cal_already_created = True cal_db_id = c[u'id'] if not cal_already_created: cal_db_id = int( client.calibrator_create(web_address, auth, observationid=str(args.cal_id), caltype=calibrator_type)[u'id']) else: cal_db_id = None if S_mean is not None: #prevent TypeError caused by trying to format Nones given to fluxes for highly scattered pulsars S_mean = float("{0:.2f}".format(S_mean)) u_S_mean = float("{0:.2f}".format(u_S_mean)) #format data for uploading if w_equiv_ms: w_equiv_ms = float("{0:.2f}".format(w_equiv_ms)) if u_w_equiv_ms: u_w_equiv_ms = float("{0:.2f}".format(u_w_equiv_ms)) if scattering: scattering = float("{0:.5f}".format(scattering)) if u_scattering: u_scattering = float("{0:.5f}".format(u_scattering)) det_kwargs = {} det_kwargs["observationid"] = int(obsid) det_kwargs["pulsar"] = str(pulsar) det_kwargs["subband"] = int(subbands) det_kwargs["coherent"] = coh, det_kwargs["observation_type"] = int(obstype) det_kwargs["calibrator"] = cal_db_id det_kwargs["startcchan"] = int(minfreq) det_kwargs["stopcchan"] = int(maxfreq) det_kwargs["flux"] = S_mean det_kwargs["flux_error"] = u_S_mean det_kwargs["width"] = w_equiv_ms det_kwargs["width_error"] = u_w_equiv_ms det_kwargs["scattering"] = scattering det_kwargs["scattering_error"] = u_scattering det_kwargs["dm"] = float(dm) try: client.detection_create(web_address, auth, **det_kwargs) except: logger.info("Detection already on database so updating the values") client.detection_update(web_address, auth, **det_kwargs) logger.info("Observation submitted to database") return subbands
) quit() args.calibration = zip_calibration_files(args.cal_dir_to_tar, args.cal_id, args.srclist) if args.pulsar and not (args.bestprof or args.ascii): #calc sub-bands subbands = 1 for b in range(len(channels)): if b == 0: continue if not (channels[b] - channels[b - 1]) == 1: subbands = subbands + 1 if coh: cal_list = client.calibrator_list(web_address, auth) cal_already_created = False for c in cal_list: if (c[u'observationid'] == int( args.cal_id)) and (c[u'caltype'] == calibrator_type): cal_already_created = True cal_db_id = c[u'id'] if not cal_already_created: cal_db_id = client.calibrator_create( web_address, auth, observationid=str(args.cal_id), caltype=calibrator_type)[u'id'] elif not args.cal_id: cal_db_id = None
def flux_cal_and_submit(time_detection, time_obs, metadata, bestprof_data, pul_ra, pul_dec, coh, auth, trcvr="/group/mwaops/PULSAR/MWA_Trcvr_tile_56.csv"): """ time_detection: the time in seconds of the dectection from the bestprof file time_obs: the time in seconds of the dectection from the metadata metadata: list from the function get_obs_metadata bestprof_data: list from the function get_from_bestprof trcvr: the file location of antena temperatures """ #unpack the bestprof_data #[obsid, pulsar, dm, period, period_uncer, obsstart, obslength, profile, bin_num] obsid, pulsar, _, period, _, beg, t_int, profile, num_bins = bestprof_data period=float(period) num_bins=int(num_bins) #get r_sys and gain t_sys, _, gain, u_gain = snfe.find_t_sys_gain(pulsar, obsid, obs_metadata=metadata,\ beg=beg, t_int=t_int) #estimate S/N sn, u_sn, _, w_equiv_bins, _, w_equiv_ms, u_w_equiv_ms, scattered = snfe.analyse_pulse_prof(prof_data=profile, period=period, verbose=True) logger.info("Profile scattered? {0}".format(scattered)) logger.info("S/N: {0} +/- {1}".format(sn, u_sn)) logger.debug("Gain {0} K/Jy".format(gain)) logger.debug("Equivalent width in bins: {0}".format(w_equiv_bins)) logger.debug("T_sys: {0} K".format(t_sys)) logger.debug("Bandwidth: {0}".format(bandwidth)) logger.debug("Detection time: {0}".format(time_detection)) logger.debug("NUmber of bins: {0}".format(num_bins)) if scattered == False: #final calc of the mean fluxdesnity in mJy S_mean = sn * t_sys / ( gain * math.sqrt(2. * float(time_detection) * bandwidth)) *\ math.sqrt( w_equiv_bins / (num_bins - w_equiv_bins)) * 1000. #constants to make uncertainty calc easier S_mean_cons = t_sys / ( math.sqrt(2. * float(time_detection) * bandwidth)) *\ math.sqrt( w_equiv_bins / (num_bins - w_equiv_bins)) * 1000. u_S_mean = math.sqrt( math.pow(S_mean_cons * u_sn / gain , 2) +\ math.pow(sn * S_mean_cons * u_gain / math.pow(gain,2) , 2) ) logger.info('Smean {0:.2f} +/- {1:.2f} mJy'.format(S_mean, u_S_mean)) else: logger.info("Profile is scattered. Flux cannot be estimated") S_mean = None u_S_mean = None #calc obstype if (maxfreq - minfreq) == 23: obstype = 1 else: obstype = 2 #calc scattering scat_height = max(profile) / 2.71828 scat_bins = 0 for p in profile: if p > scat_height: scat_bins = scat_bins + 1 scattering = float(scat_bins + 1) * float(period) /1000. #in s u_scattering = 1. * float(period) /1000. # assumes the uncertainty is one bin #calc sub-bands subbands = 1 for b in range(len(channels)): if b == 0: continue if not (channels[b] - channels[b-1]) == 1: subbands = subbands + 1 #get cal id if coh: cal_list = client.calibrator_list(web_address, auth) cal_already_created = False for c in cal_list: if ( c[u'observationid'] == int(args.cal_id) ) and ( c[u'caltype'] == calibrator_type ): cal_already_created = True cal_db_id = c[u'id'] if not cal_already_created: cal_db_id = int(client.calibrator_create(web_address, auth, observationid = str(args.cal_id), caltype = calibrator_type)[u'id']) else: cal_db_id = None if S_mean is not None: #prevent TypeError caused by trying to format Nones given to fluxes for #highly scattered pulsars S_mean = float("{0:.2f}".format(S_mean)) u_S_mean = float("{0:.2f}".format(u_S_mean)) try: client.detection_create(web_address, auth, observationid = int(obsid), pulsar = str(pulsar), subband = int(subbands), coherent = coh, observation_type = int(obstype), calibrator = cal_db_id, startcchan = int(minfreq), stopcchan = int(maxfreq), flux = S_mean, flux_error = u_S_mean, width = float("{0:.2f}".format(w_equiv_ms)), width_error = float("{0:.2f}".format(u_w_equiv_ms)), scattering = float("{0:.5f}".format(scattering)), scattering_error = float("{0:.5f}".format(u_scattering)), dm = float(dm)) except: logger.info("Detection already on database so updating the values") client.detection_update(web_address, auth, observationid = int(obsid), pulsar = str(pulsar), subband = str(subbands), coherent = coh, observation_type = int(obstype), calibrator = cal_db_id, startcchan = int(minfreq), stopcchan = int(maxfreq), flux = S_mean, flux_error = u_S_mean, width = float("{0:.2f}".format(w_equiv_ms)), width_error = float("{0:.2f}".format(u_w_equiv_ms)), scattering = float("{0:.5f}".format(scattering)), scattering_error = float("{0:.5f}".format(u_scattering)), dm = float(dm)) logger.info("Observation submitted to database") return subbands