def test_find_t_sys_gain():
"""
Tests the find_t_sys_gain function
"""
print("find_t_sys_gain")
obsid_1=1223042480
pulsar_1= "J2234+2114"
obsid_2=1222697776
pulsar_2= "J2330-2005"
md_1 = mwa_metadb_utils.get_common_obs_metadata(obsid_1) #this is to speed up the tests
md_2 = mwa_metadb_utils.get_common_obs_metadata(obsid_2)
q_1 = psrqpy.QueryATNF(psrs=[pulsar_1], loadfromdb=ATNF_LOC)
q_2 = psrqpy.QueryATNF(psrs=[pulsar_2], loadfromdb=ATNF_LOC)
test_cases = []
test_cases.append((pulsar_1, obsid_1, 1223042887, 600, q_1, md_1,\
325.17339020027805, 6.5034678040055613, 0.12547449408456718, 0.095633857616224477))
test_cases.append((pulsar_2, obsid_2, None, None, q_2, md_2,\
295.73944858721245, 5.9147889717442492, 0.29127750035795497, 0.049367868667752078))
for pulsar, obsid, beg, t_int, query, metadata,\
exp_t_sys, exp_t_sys_err, exp_gain, exp_gain_err in test_cases:
t_sys, t_sys_err, gain, gain_err = snfe.find_t_sys_gain(pulsar, obsid, beg=beg,
t_int=t_int, query=query, obs_metadata=metadata,
trcvr='database/MWA_Trcvr_tile_56.csv')
assert_almost_equal(exp_t_sys, t_sys, decimal=6)
assert_almost_equal(exp_t_sys_err, t_sys_err, decimal=6)
assert_almost_equal(exp_gain, gain, decimal=6)
assert_almost_equal(exp_gain_err, gain_err, decimal=6)
def test_find_t_sys_gain():
"""
Tests the find_t_sys_gain function
"""
print("find_t_sys_gain")
obsid_1 = 1223042480
pulsar_1 = "J2234+2114"
obsid_2 = 1222697776
pulsar_2 = "J2330-2005"
md_1 = md_dict[str(obsid_1)][0]
full_md_1 = md_dict[str(obsid_1)][1]
md_2 = md_dict[str(obsid_2)][0]
full_md_2 = md_dict[str(obsid_2)][1]
q_1 = psrqpy.QueryATNF(psrs=[pulsar_1], loadfromdb=data_load.ATNF_LOC)
q_2 = psrqpy.QueryATNF(psrs=[pulsar_2], loadfromdb=data_load.ATNF_LOC)
test_cases = []
test_cases.append((pulsar_1, obsid_1, 1223042887, q_1, md_1, full_md_1,\
325.16928459135534, 6.5034678040055613, 0.1405071004856564, 0.1065230048864121))
test_cases.append((pulsar_2, obsid_2, None, q_2, md_2, full_md_2,\
295.7376541472588, 5.914753082945176, 0.29127739909167133, 0.049367851504392074))
for pulsar, obsid, beg, query, metadata, full_meta,\
exp_t_sys, exp_t_sys_err, exp_gain, exp_gain_err in test_cases:
t_sys, t_sys_err, gain, gain_err = snfe.find_t_sys_gain(pulsar, obsid, beg=beg,\
query=query, obs_metadata=metadata, full_meta=full_meta)
assert_almost_equal(exp_t_sys, t_sys, decimal=4)
assert_almost_equal(exp_t_sys_err, t_sys_err, decimal=4)
assert_almost_equal(exp_gain, gain, decimal=4)
assert_almost_equal(exp_gain_err, gain_err, decimal=4)
def flux_cal_and_submit(time_obs,
metadata,
bestprof_data,
pul_ra,
pul_dec,
coh,
auth,
pulsar=None,
trcvr=data_load.TRCVR_FILE):
"""
time_obs: the time in seconds of the dectection from the metadata
metadata: list from the function get_obs_metadata
bestprof_data: list from the function get_from_bestprof
trcvr: the file location of antena temperatures
"""
#unpack the bestprof_data
#[obsid, pulsar, dm, period, period_uncer, obsstart, obslength, profile, bin_num]
obsid, prof_psr, dm, period, _, beg, t_int, profile, num_bins = bestprof_data
if not pulsar:
pulsar = prof_psr
period = float(period)
num_bins = int(num_bins)
#get r_sys and gain
t_sys, _, gain, u_gain = snfe.find_t_sys_gain(pulsar, obsid, obs_metadata=metadata,\
beg=beg, end=(t_int + beg - 1))
#estimate S/N
try:
prof_dict = prof_utils.auto_gfit(profile,\
period = period, plot_name="{0}_{1}_{2}_bins_gaussian_fit.png".format(obsid, pulsar, num_bins))
except (prof_utils.ProfileLengthError, prof_utils.NoFitError) as _:
prof_dict = None
if not prof_dict:
logger.info(
"Profile couldn't be fit. Using old style of profile analysis")
prof_dict = prof_utils.auto_analyse_pulse_prof(profile, period)
if prof_dict:
sn = prof_dict["sn"]
u_sn = prof_dict["sn_e"]
w_equiv_bins = prof_dict["w_equiv_bins"]
u_w_equiv_bins = prof_dict["w_equiv_bins_e"]
w_equiv_ms = period / num_bins * w_equiv_bins
u_w_equiv_ms = period / num_bins * u_w_equiv_bins
scattering = prof_dict[
"scattering"] * period / num_bins / 1000 #convert to seconds
u_scattering = prof_dict["scattering_e"] * period / num_bins / 1000
scattered = prof_dict["scattered"]
else:
logger.warn("Profile could not be analysed using any methods")
sn = None
u_sn = None
w_equiv_bins = None
u_w_equiv_bins = None
w_equiv_ms = None
u_w_equiv_ms = None
scattering = None
u_scattering = None
scattered = None
S_mean = None
u_S_mean = None
else:
sn = prof_dict["sn"]
u_sn = prof_dict["sn_e"]
w_equiv_bins = prof_dict["Weq"]
u_w_equiv_bins = prof_dict["Weq_e"]
w_equiv_ms = period / num_bins * w_equiv_bins
u_w_equiv_ms = period / num_bins * u_w_equiv_bins
scattering = prof_dict[
"Wscat"] * period / num_bins / 1000 #convert to seconds
u_scattering = prof_dict["Wscat_e"] * period / num_bins / 1000
scattered = prof_dict["scattered"]
if prof_dict:
logger.info("Profile scattered? {0}".format(scattered))
logger.info("S/N: {0} +/- {1}".format(sn, u_sn))
logger.debug("Gain {0} K/Jy".format(gain))
logger.debug("Equivalent width in bins: {0}".format(w_equiv_bins))
logger.debug("T_sys: {0} K".format(t_sys))
logger.debug("Bandwidth: {0}".format(bandwidth))
logger.debug("Detection time: {0}".format(t_int))
logger.debug("NUmber of bins: {0}".format(num_bins))
if scattered == False:
#final calc of the mean fluxdesnity in mJy
S_mean = sn * t_sys / ( gain * math.sqrt(2. * float(t_int) * bandwidth)) *\
math.sqrt( w_equiv_bins / (num_bins - w_equiv_bins)) * 1000.
#constants to make uncertainty calc easier
S_mean_cons = t_sys / ( math.sqrt(2. * float(t_int) * bandwidth)) *\
math.sqrt( w_equiv_bins / (num_bins - w_equiv_bins)) * 1000.
u_S_mean = math.sqrt( math.pow(S_mean_cons * u_sn / gain , 2) +\
math.pow(sn * S_mean_cons * u_gain / math.pow(gain,2) , 2) )
logger.info('Smean {0:.2f} +/- {1:.2f} mJy'.format(
S_mean, u_S_mean))
else:
logger.info("Profile is scattered. Flux cannot be estimated")
S_mean = None
u_S_mean = None
#calc obstype
if (maxfreq - minfreq) == 23:
obstype = 1
else:
obstype = 2
subbands = get_subbands(metadata)
web_address, auth = get_db_auth_addr()
#get cal id
if coh:
cal_list = client.calibrator_list(web_address, auth)
cal_already_created = False
for c in cal_list:
if (c[u'observationid'] == int(
args.cal_id)) and (c[u'caltype'] == calibrator_type):
cal_already_created = True
cal_db_id = c[u'id']
if not cal_already_created:
cal_db_id = int(
client.calibrator_create(web_address,
auth,
observationid=str(args.cal_id),
caltype=calibrator_type)[u'id'])
else:
cal_db_id = None
if S_mean is not None:
#prevent TypeError caused by trying to format Nones given to fluxes for highly scattered pulsars
S_mean = float("{0:.2f}".format(S_mean))
u_S_mean = float("{0:.2f}".format(u_S_mean))
#format data for uploading
if w_equiv_ms:
w_equiv_ms = float("{0:.2f}".format(w_equiv_ms))
if u_w_equiv_ms:
u_w_equiv_ms = float("{0:.2f}".format(u_w_equiv_ms))
if scattering:
scattering = float("{0:.5f}".format(scattering))
if u_scattering:
u_scattering = float("{0:.5f}".format(u_scattering))
det_kwargs = {}
det_kwargs["observationid"] = int(obsid)
det_kwargs["pulsar"] = str(pulsar)
det_kwargs["subband"] = int(subbands)
det_kwargs["coherent"] = coh,
det_kwargs["observation_type"] = int(obstype)
det_kwargs["calibrator"] = cal_db_id
det_kwargs["startcchan"] = int(minfreq)
det_kwargs["stopcchan"] = int(maxfreq)
det_kwargs["flux"] = S_mean
det_kwargs["flux_error"] = u_S_mean
det_kwargs["width"] = w_equiv_ms
det_kwargs["width_error"] = u_w_equiv_ms
det_kwargs["scattering"] = scattering
det_kwargs["scattering_error"] = u_scattering
det_kwargs["dm"] = float(dm)
try:
client.detection_create(web_address, auth, **det_kwargs)
except:
logger.info("Detection already on database so updating the values")
client.detection_update(web_address, auth, **det_kwargs)
logger.info("Observation submitted to database")
return subbands
def flux_cal_and_submit(time_detection, time_obs, metadata, bestprof_data,
pul_ra, pul_dec, coh, auth,
trcvr="/group/mwaops/PULSAR/MWA_Trcvr_tile_56.csv"):
"""
time_detection: the time in seconds of the dectection from the bestprof file
time_obs: the time in seconds of the dectection from the metadata
metadata: list from the function get_obs_metadata
bestprof_data: list from the function get_from_bestprof
trcvr: the file location of antena temperatures
"""
#unpack the bestprof_data
#[obsid, pulsar, dm, period, period_uncer, obsstart, obslength, profile, bin_num]
obsid, pulsar, _, period, _, beg, t_int, profile, num_bins = bestprof_data
period=float(period)
num_bins=int(num_bins)
#get r_sys and gain
t_sys, _, gain, u_gain = snfe.find_t_sys_gain(pulsar, obsid, obs_metadata=metadata,\
beg=beg, t_int=t_int)
#estimate S/N
sn, u_sn, _, w_equiv_bins, _, w_equiv_ms, u_w_equiv_ms, scattered = snfe.analyse_pulse_prof(prof_data=profile, period=period, verbose=True)
logger.info("Profile scattered? {0}".format(scattered))
logger.info("S/N: {0} +/- {1}".format(sn, u_sn))
logger.debug("Gain {0} K/Jy".format(gain))
logger.debug("Equivalent width in bins: {0}".format(w_equiv_bins))
logger.debug("T_sys: {0} K".format(t_sys))
logger.debug("Bandwidth: {0}".format(bandwidth))
logger.debug("Detection time: {0}".format(time_detection))
logger.debug("NUmber of bins: {0}".format(num_bins))
if scattered == False:
#final calc of the mean fluxdesnity in mJy
S_mean = sn * t_sys / ( gain * math.sqrt(2. * float(time_detection) * bandwidth)) *\
math.sqrt( w_equiv_bins / (num_bins - w_equiv_bins)) * 1000.
#constants to make uncertainty calc easier
S_mean_cons = t_sys / ( math.sqrt(2. * float(time_detection) * bandwidth)) *\
math.sqrt( w_equiv_bins / (num_bins - w_equiv_bins)) * 1000.
u_S_mean = math.sqrt( math.pow(S_mean_cons * u_sn / gain , 2) +\
math.pow(sn * S_mean_cons * u_gain / math.pow(gain,2) , 2) )
logger.info('Smean {0:.2f} +/- {1:.2f} mJy'.format(S_mean, u_S_mean))
else:
logger.info("Profile is scattered. Flux cannot be estimated")
S_mean = None
u_S_mean = None
#calc obstype
if (maxfreq - minfreq) == 23:
obstype = 1
else:
obstype = 2
#calc scattering
scat_height = max(profile) / 2.71828
scat_bins = 0
for p in profile:
if p > scat_height:
scat_bins = scat_bins + 1
scattering = float(scat_bins + 1) * float(period) /1000. #in s
u_scattering = 1. * float(period) /1000. # assumes the uncertainty is one bin
#calc sub-bands
subbands = 1
for b in range(len(channels)):
if b == 0:
continue
if not (channels[b] - channels[b-1]) == 1:
subbands = subbands + 1
#get cal id
if coh:
cal_list = client.calibrator_list(web_address, auth)
cal_already_created = False
for c in cal_list:
if ( c[u'observationid'] == int(args.cal_id) ) and ( c[u'caltype'] == calibrator_type ):
cal_already_created = True
cal_db_id = c[u'id']
if not cal_already_created:
cal_db_id = int(client.calibrator_create(web_address, auth,
observationid = str(args.cal_id),
caltype = calibrator_type)[u'id'])
else:
cal_db_id = None
if S_mean is not None:
#prevent TypeError caused by trying to format Nones given to fluxes for
#highly scattered pulsars
S_mean = float("{0:.2f}".format(S_mean))
u_S_mean = float("{0:.2f}".format(u_S_mean))
try:
client.detection_create(web_address, auth,
observationid = int(obsid),
pulsar = str(pulsar),
subband = int(subbands),
coherent = coh,
observation_type = int(obstype),
calibrator = cal_db_id,
startcchan = int(minfreq), stopcchan = int(maxfreq),
flux = S_mean,
flux_error = u_S_mean,
width = float("{0:.2f}".format(w_equiv_ms)),
width_error = float("{0:.2f}".format(u_w_equiv_ms)),
scattering = float("{0:.5f}".format(scattering)),
scattering_error = float("{0:.5f}".format(u_scattering)),
dm = float(dm))
except:
logger.info("Detection already on database so updating the values")
client.detection_update(web_address, auth,
observationid = int(obsid),
pulsar = str(pulsar),
subband = str(subbands),
coherent = coh,
observation_type = int(obstype),
calibrator = cal_db_id,
startcchan = int(minfreq), stopcchan = int(maxfreq),
flux = S_mean,
flux_error = u_S_mean,
width = float("{0:.2f}".format(w_equiv_ms)),
width_error = float("{0:.2f}".format(u_w_equiv_ms)),
scattering = float("{0:.5f}".format(scattering)),
scattering_error = float("{0:.5f}".format(u_scattering)),
dm = float(dm))
logger.info("Observation submitted to database")
return subbands