def upload_cal_files(obsid, cal_id, cal_dir_to_tar, srclist, caltype=2):
"""
Uploads the calibrator solutions to the MWA pulsar database
Parameters:
-----------
obsid: int
The observation ID
cal_id: int
The calibrator ID
cal_dir_to_tar: string
The location of the calibrator RTS directory
srclist: string
The path to the sourcelist
caltype: int
The type of calibrator. Default: 2
"""
auth, web_address = get_db_auth_addr()
client_files_dict = client.calibration_file_by_observation_id(web_address,
auth,
obsid=cal_id)
if client_files_dict:
logger.info(
"This calibrator already has solutions on the database. Not uploading"
)
else:
zip_loc = zip_calibration_files(cal_dir_to_tar, cal_id, srclist)
# Check if calibrator has to be made
cal_list = client.calibrator_list(web_address, auth)
if not cal_id in [c['observationid'] for c in cal_list]:
client.calibrator_create(web_address,
auth,
observationid=str(cal_id))
# Upload Calibration file
try:
client.calibrator_file_upload(web_address,
auth,
observationid=str(cal_id),
filepath=str(zip_loc),
caltype=2)
logger.info(
"Uploaded calibrator solutions from {} to the database".format(
cal_id))
except:
logger.warn("Failed to upload calibration files")
os.remove(zip_loc)
def flux_cal_and_submit(time_obs,
metadata,
bestprof_data,
pul_ra,
pul_dec,
coh,
auth,
pulsar=None,
trcvr=data_load.TRCVR_FILE):
"""
time_obs: the time in seconds of the dectection from the metadata
metadata: list from the function get_obs_metadata
bestprof_data: list from the function get_from_bestprof
trcvr: the file location of antena temperatures
"""
#unpack the bestprof_data
#[obsid, pulsar, dm, period, period_uncer, obsstart, obslength, profile, bin_num]
obsid, prof_psr, dm, period, _, beg, t_int, profile, num_bins = bestprof_data
if not pulsar:
pulsar = prof_psr
period = float(period)
num_bins = int(num_bins)
#get r_sys and gain
t_sys, _, gain, u_gain = snfe.find_t_sys_gain(pulsar, obsid, obs_metadata=metadata,\
beg=beg, end=(t_int + beg - 1))
#estimate S/N
try:
prof_dict = prof_utils.auto_gfit(profile,\
period = period, plot_name="{0}_{1}_{2}_bins_gaussian_fit.png".format(obsid, pulsar, num_bins))
except (prof_utils.ProfileLengthError, prof_utils.NoFitError) as _:
prof_dict = None
if not prof_dict:
logger.info(
"Profile couldn't be fit. Using old style of profile analysis")
prof_dict = prof_utils.auto_analyse_pulse_prof(profile, period)
if prof_dict:
sn = prof_dict["sn"]
u_sn = prof_dict["sn_e"]
w_equiv_bins = prof_dict["w_equiv_bins"]
u_w_equiv_bins = prof_dict["w_equiv_bins_e"]
w_equiv_ms = period / num_bins * w_equiv_bins
u_w_equiv_ms = period / num_bins * u_w_equiv_bins
scattering = prof_dict[
"scattering"] * period / num_bins / 1000 #convert to seconds
u_scattering = prof_dict["scattering_e"] * period / num_bins / 1000
scattered = prof_dict["scattered"]
else:
logger.warn("Profile could not be analysed using any methods")
sn = None
u_sn = None
w_equiv_bins = None
u_w_equiv_bins = None
w_equiv_ms = None
u_w_equiv_ms = None
scattering = None
u_scattering = None
scattered = None
S_mean = None
u_S_mean = None
else:
sn = prof_dict["sn"]
u_sn = prof_dict["sn_e"]
w_equiv_bins = prof_dict["Weq"]
u_w_equiv_bins = prof_dict["Weq_e"]
w_equiv_ms = period / num_bins * w_equiv_bins
u_w_equiv_ms = period / num_bins * u_w_equiv_bins
scattering = prof_dict[
"Wscat"] * period / num_bins / 1000 #convert to seconds
u_scattering = prof_dict["Wscat_e"] * period / num_bins / 1000
scattered = prof_dict["scattered"]
if prof_dict:
logger.info("Profile scattered? {0}".format(scattered))
logger.info("S/N: {0} +/- {1}".format(sn, u_sn))
logger.debug("Gain {0} K/Jy".format(gain))
logger.debug("Equivalent width in bins: {0}".format(w_equiv_bins))
logger.debug("T_sys: {0} K".format(t_sys))
logger.debug("Bandwidth: {0}".format(bandwidth))
logger.debug("Detection time: {0}".format(t_int))
logger.debug("NUmber of bins: {0}".format(num_bins))
if scattered == False:
#final calc of the mean fluxdesnity in mJy
S_mean = sn * t_sys / ( gain * math.sqrt(2. * float(t_int) * bandwidth)) *\
math.sqrt( w_equiv_bins / (num_bins - w_equiv_bins)) * 1000.
#constants to make uncertainty calc easier
S_mean_cons = t_sys / ( math.sqrt(2. * float(t_int) * bandwidth)) *\
math.sqrt( w_equiv_bins / (num_bins - w_equiv_bins)) * 1000.
u_S_mean = math.sqrt( math.pow(S_mean_cons * u_sn / gain , 2) +\
math.pow(sn * S_mean_cons * u_gain / math.pow(gain,2) , 2) )
logger.info('Smean {0:.2f} +/- {1:.2f} mJy'.format(
S_mean, u_S_mean))
else:
logger.info("Profile is scattered. Flux cannot be estimated")
S_mean = None
u_S_mean = None
#calc obstype
if (maxfreq - minfreq) == 23:
obstype = 1
else:
obstype = 2
subbands = get_subbands(metadata)
web_address, auth = get_db_auth_addr()
#get cal id
if coh:
cal_list = client.calibrator_list(web_address, auth)
cal_already_created = False
for c in cal_list:
if (c[u'observationid'] == int(
args.cal_id)) and (c[u'caltype'] == calibrator_type):
cal_already_created = True
cal_db_id = c[u'id']
if not cal_already_created:
cal_db_id = int(
client.calibrator_create(web_address,
auth,
observationid=str(args.cal_id),
caltype=calibrator_type)[u'id'])
else:
cal_db_id = None
if S_mean is not None:
#prevent TypeError caused by trying to format Nones given to fluxes for highly scattered pulsars
S_mean = float("{0:.2f}".format(S_mean))
u_S_mean = float("{0:.2f}".format(u_S_mean))
#format data for uploading
if w_equiv_ms:
w_equiv_ms = float("{0:.2f}".format(w_equiv_ms))
if u_w_equiv_ms:
u_w_equiv_ms = float("{0:.2f}".format(u_w_equiv_ms))
if scattering:
scattering = float("{0:.5f}".format(scattering))
if u_scattering:
u_scattering = float("{0:.5f}".format(u_scattering))
det_kwargs = {}
det_kwargs["observationid"] = int(obsid)
det_kwargs["pulsar"] = str(pulsar)
det_kwargs["subband"] = int(subbands)
det_kwargs["coherent"] = coh,
det_kwargs["observation_type"] = int(obstype)
det_kwargs["calibrator"] = cal_db_id
det_kwargs["startcchan"] = int(minfreq)
det_kwargs["stopcchan"] = int(maxfreq)
det_kwargs["flux"] = S_mean
det_kwargs["flux_error"] = u_S_mean
det_kwargs["width"] = w_equiv_ms
det_kwargs["width_error"] = u_w_equiv_ms
det_kwargs["scattering"] = scattering
det_kwargs["scattering_error"] = u_scattering
det_kwargs["dm"] = float(dm)
try:
client.detection_create(web_address, auth, **det_kwargs)
except:
logger.info("Detection already on database so updating the values")
client.detection_update(web_address, auth, **det_kwargs)
logger.info("Observation submitted to database")
return subbands
continue
if not (channels[b] - channels[b - 1]) == 1:
subbands = subbands + 1
if coh:
cal_list = client.calibrator_list(web_address, auth)
cal_already_created = False
for c in cal_list:
if (c[u'observationid'] == int(
args.cal_id)) and (c[u'caltype'] == calibrator_type):
cal_already_created = True
cal_db_id = c[u'id']
if not cal_already_created:
cal_db_id = client.calibrator_create(
web_address,
auth,
observationid=str(args.cal_id),
caltype=calibrator_type)[u'id']
elif not args.cal_id:
cal_db_id = None
#uploads files to database if there's the no calc option
#checks if the observation is on the database
det_kwargs = {}
det_kwargs["observationid"] = str(args.obsid)
det_kwargs["pulsar"] = args.pulsar
det_kwargs["calibrator"] = int(cal_db_id)
det_kwargs["subband"] = int(subbands)
det_kwargs["coherent"] = coh
det_kwargs["startcchan"] = int(minfreq)
det_kwargs["stopcchan"] = int(maxfreq)
def flux_cal_and_submit(time_detection, time_obs, metadata, bestprof_data,
pul_ra, pul_dec, coh, auth,
trcvr="/group/mwaops/PULSAR/MWA_Trcvr_tile_56.csv"):
"""
time_detection: the time in seconds of the dectection from the bestprof file
time_obs: the time in seconds of the dectection from the metadata
metadata: list from the function get_obs_metadata
bestprof_data: list from the function get_from_bestprof
trcvr: the file location of antena temperatures
"""
#unpack the bestprof_data
#[obsid, pulsar, dm, period, period_uncer, obsstart, obslength, profile, bin_num]
obsid, pulsar, _, period, _, beg, t_int, profile, num_bins = bestprof_data
period=float(period)
num_bins=int(num_bins)
#get r_sys and gain
t_sys, _, gain, u_gain = snfe.find_t_sys_gain(pulsar, obsid, obs_metadata=metadata,\
beg=beg, t_int=t_int)
#estimate S/N
sn, u_sn, _, w_equiv_bins, _, w_equiv_ms, u_w_equiv_ms, scattered = snfe.analyse_pulse_prof(prof_data=profile, period=period, verbose=True)
logger.info("Profile scattered? {0}".format(scattered))
logger.info("S/N: {0} +/- {1}".format(sn, u_sn))
logger.debug("Gain {0} K/Jy".format(gain))
logger.debug("Equivalent width in bins: {0}".format(w_equiv_bins))
logger.debug("T_sys: {0} K".format(t_sys))
logger.debug("Bandwidth: {0}".format(bandwidth))
logger.debug("Detection time: {0}".format(time_detection))
logger.debug("NUmber of bins: {0}".format(num_bins))
if scattered == False:
#final calc of the mean fluxdesnity in mJy
S_mean = sn * t_sys / ( gain * math.sqrt(2. * float(time_detection) * bandwidth)) *\
math.sqrt( w_equiv_bins / (num_bins - w_equiv_bins)) * 1000.
#constants to make uncertainty calc easier
S_mean_cons = t_sys / ( math.sqrt(2. * float(time_detection) * bandwidth)) *\
math.sqrt( w_equiv_bins / (num_bins - w_equiv_bins)) * 1000.
u_S_mean = math.sqrt( math.pow(S_mean_cons * u_sn / gain , 2) +\
math.pow(sn * S_mean_cons * u_gain / math.pow(gain,2) , 2) )
logger.info('Smean {0:.2f} +/- {1:.2f} mJy'.format(S_mean, u_S_mean))
else:
logger.info("Profile is scattered. Flux cannot be estimated")
S_mean = None
u_S_mean = None
#calc obstype
if (maxfreq - minfreq) == 23:
obstype = 1
else:
obstype = 2
#calc scattering
scat_height = max(profile) / 2.71828
scat_bins = 0
for p in profile:
if p > scat_height:
scat_bins = scat_bins + 1
scattering = float(scat_bins + 1) * float(period) /1000. #in s
u_scattering = 1. * float(period) /1000. # assumes the uncertainty is one bin
#calc sub-bands
subbands = 1
for b in range(len(channels)):
if b == 0:
continue
if not (channels[b] - channels[b-1]) == 1:
subbands = subbands + 1
#get cal id
if coh:
cal_list = client.calibrator_list(web_address, auth)
cal_already_created = False
for c in cal_list:
if ( c[u'observationid'] == int(args.cal_id) ) and ( c[u'caltype'] == calibrator_type ):
cal_already_created = True
cal_db_id = c[u'id']
if not cal_already_created:
cal_db_id = int(client.calibrator_create(web_address, auth,
observationid = str(args.cal_id),
caltype = calibrator_type)[u'id'])
else:
cal_db_id = None
if S_mean is not None:
#prevent TypeError caused by trying to format Nones given to fluxes for
#highly scattered pulsars
S_mean = float("{0:.2f}".format(S_mean))
u_S_mean = float("{0:.2f}".format(u_S_mean))
try:
client.detection_create(web_address, auth,
observationid = int(obsid),
pulsar = str(pulsar),
subband = int(subbands),
coherent = coh,
observation_type = int(obstype),
calibrator = cal_db_id,
startcchan = int(minfreq), stopcchan = int(maxfreq),
flux = S_mean,
flux_error = u_S_mean,
width = float("{0:.2f}".format(w_equiv_ms)),
width_error = float("{0:.2f}".format(u_w_equiv_ms)),
scattering = float("{0:.5f}".format(scattering)),
scattering_error = float("{0:.5f}".format(u_scattering)),
dm = float(dm))
except:
logger.info("Detection already on database so updating the values")
client.detection_update(web_address, auth,
observationid = int(obsid),
pulsar = str(pulsar),
subband = str(subbands),
coherent = coh,
observation_type = int(obstype),
calibrator = cal_db_id,
startcchan = int(minfreq), stopcchan = int(maxfreq),
flux = S_mean,
flux_error = u_S_mean,
width = float("{0:.2f}".format(w_equiv_ms)),
width_error = float("{0:.2f}".format(u_w_equiv_ms)),
scattering = float("{0:.5f}".format(scattering)),
scattering_error = float("{0:.5f}".format(u_scattering)),
dm = float(dm))
logger.info("Observation submitted to database")
return subbands