def test_find_t_sys_gain():
"""
Tests the find_t_sys_gain function
"""
print("find_t_sys_gain")
obsid_1 = 1223042480
pulsar_1 = "J2234+2114"
obsid_2 = 1222697776
pulsar_2 = "J2330-2005"
md_1 = md_dict[str(obsid_1)][0]
full_md_1 = md_dict[str(obsid_1)][1]
md_2 = md_dict[str(obsid_2)][0]
full_md_2 = md_dict[str(obsid_2)][1]
q_1 = psrqpy.QueryATNF(psrs=[pulsar_1], loadfromdb=data_load.ATNF_LOC)
q_2 = psrqpy.QueryATNF(psrs=[pulsar_2], loadfromdb=data_load.ATNF_LOC)
test_cases = []
test_cases.append((pulsar_1, obsid_1, 1223042887, q_1, md_1, full_md_1,\
325.16928459135534, 6.5034678040055613, 0.1405071004856564, 0.1065230048864121))
test_cases.append((pulsar_2, obsid_2, None, q_2, md_2, full_md_2,\
295.7376541472588, 5.914753082945176, 0.29127739909167133, 0.049367851504392074))
for pulsar, obsid, beg, query, metadata, full_meta,\
exp_t_sys, exp_t_sys_err, exp_gain, exp_gain_err in test_cases:
t_sys, t_sys_err, gain, gain_err = snfe.find_t_sys_gain(pulsar, obsid, beg=beg,\
query=query, obs_metadata=metadata, full_meta=full_meta)
assert_almost_equal(exp_t_sys, t_sys, decimal=4)
assert_almost_equal(exp_t_sys_err, t_sys_err, decimal=4)
assert_almost_equal(exp_gain, gain, decimal=4)
assert_almost_equal(exp_gain_err, gain_err, decimal=4)
def test_find_t_sys_gain():
"""
Tests the find_t_sys_gain function
"""
print("find_t_sys_gain")
obsid_1=1223042480
pulsar_1= "J2234+2114"
obsid_2=1222697776
pulsar_2= "J2330-2005"
md_1 = mwa_metadb_utils.get_common_obs_metadata(obsid_1) #this is to speed up the tests
md_2 = mwa_metadb_utils.get_common_obs_metadata(obsid_2)
q_1 = psrqpy.QueryATNF(psrs=[pulsar_1], loadfromdb=ATNF_LOC)
q_2 = psrqpy.QueryATNF(psrs=[pulsar_2], loadfromdb=ATNF_LOC)
test_cases = []
test_cases.append((pulsar_1, obsid_1, 1223042887, 600, q_1, md_1,\
325.17339020027805, 6.5034678040055613, 0.12547449408456718, 0.095633857616224477))
test_cases.append((pulsar_2, obsid_2, None, None, q_2, md_2,\
295.73944858721245, 5.9147889717442492, 0.29127750035795497, 0.049367868667752078))
for pulsar, obsid, beg, t_int, query, metadata,\
exp_t_sys, exp_t_sys_err, exp_gain, exp_gain_err in test_cases:
t_sys, t_sys_err, gain, gain_err = snfe.find_t_sys_gain(pulsar, obsid, beg=beg,
t_int=t_int, query=query, obs_metadata=metadata,
trcvr='database/MWA_Trcvr_tile_56.csv')
assert_almost_equal(exp_t_sys, t_sys, decimal=6)
assert_almost_equal(exp_t_sys_err, t_sys_err, decimal=6)
assert_almost_equal(exp_gain, gain, decimal=6)
assert_almost_equal(exp_gain_err, gain_err, decimal=6)
def multi_psr_snfe(pulsar_list, obsid,\
beg=None, end=None, obs_metadata=None, full_meta=None, plot_flux=False,\
query=None, min_z_power=0.3, trcvr=data_load.TRCVR_FILE):
if obs_metadata is None or full_meta is None:
logger.debug("Obtaining obs metadata")
obs_metadata, full_meta = mwa_metadb_utils.get_common_obs_metadata(
obsid, return_all=True)
obs_beg, obs_end = mwa_metadb_utils.obs_max_min(obsid, meta=full_meta)
if beg is None:
beg = obs_beg
if end is None:
end = obs_end
mega_query = psrqpy.QueryATNF(psrs=pulsar_list,
loadfromdb=data_load.ATNF_LOC).pandas
sn_dict = {}
for i, pulsar in enumerate(mega_query["PSRJ"]):
psr_query = {}
for key in mega_query.keys():
psr_query[key] = [mega_query[key][i]]
sn, sn_e = est_pulsar_sn(pulsar, obsid,\
beg=beg, end=end, obs_metadata=obs_metadata, full_meta=full_meta, plot_flux=plot_flux,\
query=psr_query, min_z_power=min_z_power, trcvr=trcvr)
sn_dict[pulsar] = [sn, sn_e]
return sn_dict
def get_psrcat_ra_dec(pulsar_list=None, max_dm=1000., include_dm=False):
"""
Uses PSRCAT to return a list of pulsar names, ras and decs. Not corrected for proper motion.
Removes pulsars without any RA or DEC recorded
If no pulsar_list given then returns all pulsar on the catalogue
If include_dm is True then also ouput DM
get_psrcat_ra_dec(pulsar_list = None)
Args:
pulsar_list: A space list of pulsar names eg: [J0534+2200, J0538+2817].
(default: uses all pulsars)
return [[Jname, RAJ, DecJ]]
"""
import psrqpy
params = ['JNAME', 'RAJ', 'DECJ', 'DM']
query = psrqpy.QueryATNF(params=params,
psrs=pulsar_list,
loadfromdb=ATNF_LOC).pandas
pulsar_ra_dec = []
for row in query.itertuples():
# Only record if under the max_dm
dm = row.DM
if not math.isnan(dm):
if float(dm) < max_dm:
if include_dm:
pulsar_ra_dec.append([row.JNAME, row.RAJ, row.DECJ, dm])
else:
pulsar_ra_dec.append([row.JNAME, row.RAJ, row.DECJ])
return pulsar_ra_dec
def get_psrcat_ra_dec(pulsar_list=None, max_dm=250., include_dm=False, query=None):
"""
Uses PSRCAT to return a list of pulsar names, ras and decs. Not corrected for proper motion.
Removes pulsars without any RA or DEC recorded
If no pulsar_list given then returns all pulsar on the catalogue
If include_dm is True then also ouput DM
get_psrcat_ra_dec(pulsar_list = None)
Args:
pulsar_list: A space list of pulsar names eg: [J0534+2200, J0538+2817].
(default: uses all pulsars)
return [[Jname, RAJ, DecJ]]
"""
import psrqpy
#params = ['JNAME', 'RAJ', 'DECJ', 'DM']
if query is None:
query = psrqpy.QueryATNF(params = ['PSRJ', 'RAJ', 'DECJ', 'DM'], psrs=pulsar_list, loadfromdb=data_load.ATNF_LOC).pandas
pulsar_ra_dec = []
for i, _ in enumerate(query["PSRJ"]):
# Only record if under the max_dm
dm = query["DM"][i]
if not math.isnan(dm):
if float(dm) < max_dm:
if include_dm:
pulsar_ra_dec.append([query["PSRJ"][i], query["RAJ"][i], query["DECJ"][i], dm])
else:
pulsar_ra_dec.append([query["PSRJ"][i], query["RAJ"][i], query["DECJ"][i]])
return pulsar_ra_dec
def find_RM_from_cat(pulsar):
"""
Gets rotation measure from prscat query. Returns None if not on catalogue
Parameters:
-----------
pulsar: str
The J-name of the pulsar
Returns:
--------
rm: float
The rotation measure
rm_err: float
The uncertainty in the rotation measure
"""
query = psrqpy.QueryATNF(params=["RM"], psrs=[pulsar],
loadfromdb=ATNF_LOC).pandas
rm = query["RM"][0]
rm_err = query["RM_ERR"][0]
if np.isnan(rm):
return None, None
elif np.isnan(rm_err):
rm_err = 0.15 * rm
return rm, rm_err
def bin_sampling_limit(pulsar, sampling_rate=1e-4, query=None):
"""Finds the sampling limit of the input pulsar in units of number of bins"""
if query is None:
query = psrqpy.QueryATNF(params=["P0"],
psrs=[pulsar],
loadfromdb=data_load.ATNF_LOC).pandas
query_index = list(query["JNAME"]).index(pulsar)
period = query["P0"][query_index]
bin_lim = math.ceil(period / sampling_rate) #round up the limit
return bin_lim
def bin_sampling_limit(pulsar, sampling_rate=1e-4):
#returns the minimum number of bins you can use for this pulsar based on MWA sampling rate
query = psrqpy.QueryATNF(params=["P0"], psrs=[pulsar],
loadfromdb=ATNF_LOC).pandas
period = query["P0"][0]
min_bins = int(period / sampling_rate +
1) #the +1 is to round the limit up every time
logger.debug("Bin limit: {0}".format(min_bins))
return min_bins
def is_binary(pulsar, query=None):
"""Checks the ATNF database to see if a pulsar is part of a binary system"""
if query is None:
query = psrqpy.QueryATNF(params=["BINARY"],
psrs=[pulsar],
loadfromdb=data_load.ATNF_LOC).pandas
query_index = list(query["JNAME"]).index(pulsar)
if isinstance(query["BINARY"][query_index], str):
return True
else:
return False
def fill(request):
f = open("database/static/list.txt")
lines = f.readlines()
pulsars = []
for line in lines:
psrs = line.strip().split(',')
for psr in psrs:
if not "READ" in psr and not "search" in psr and not "J0000" in psr and psr != '':
pulsars.append(psr.strip())
if "README.txt" in line:
break
f.close()
# TODO psrqpy
all = psrqpy.PSR_ALL_PARS
atrs = Pulsar.__dict__.keys()
attrs = []
for at in atrs:
if at.upper() in all:
attrs.append(at.upper())
q = psrqpy.QueryATNF(params=attrs, psrs=pulsars)
cat = q.catalogue
for ps in pulsars:
p = Pulsar.objects.filter(NAME=ps)
if len(p) == 0:
psr = Pulsar(NAME=ps)
res = cat.loc[cat["NAME"] == psr.NAME]
for atr in attrs:
new_vals = getattr(res, atr).values
if len(new_vals) > 0:
setattr(psr, atr, new_vals[0])
psr.save()
elif len(p) == 1:
# new values
psr = p[0]
res = cat.loc[cat["NAME"] == psr.NAME]
for atr in attrs:
val = getattr(psr, atr)
if val == None:
new_vals = getattr(res, atr).values
if len(new_vals) > 0:
setattr(psr, atr, new_vals[0])
psr.save()
return HttpResponse("Database changes <br /> {}".format(1))
def is_binary(pulsar):
"""
Checks the ATNF database to see if a pulsar is part of a binary system
Parameters:
-----------
pulsar: string
The J name of the pulsar
Returns:
--------
boolean
True if the pulsar is a binary. False otherwise
"""
query = psrqpy.QueryATNF(params=["BINARY"],
psrs=[pulsar],
loadfromdb=ATNF_LOC).pandas
if isinstance(query["BINARY"][0], str):
return True
else:
return False
def bin_sampling_limit(pulsar, sampling_rate=1e-4):
"""
Finds the sampling limit of the input pulsar in units of number of bins
Parameters:
-----------
puslar: string
The J name of the pulsar to check the sampling limit for
sampling_rate: float
OPTIONAL - the sampling rate of the instrument. Default=1e-4 (MWA VCS)
Returns:
bin_lim: int
The highest number of bins that can be logically folded on
"""
query = psrqpy.QueryATNF(params=["P0"], psrs=[pulsar],
loadfromdb=ATNF_LOC).pandas
period = query["P0"][0]
bin_lim = int(period / sampling_rate +
1) #the +1 is to round the limit up every time
logger.debug("Bin limit: {0}".format(bin_lim))
return bin_lim
def getSunDist(psrName, toa):
'''
Function to return the distance (in degrees) between a pulsar and the sun
Pulsar coordinates is queried from the ATNF catalog
Parameters
-----------
psrName : string, name of pulsar
toa : float, Time of Arrival in MJD
Return value
----------
Float: Distance of sun from pulsar in degrees
'''
q = psrqpy.QueryATNF(params=['RaJ', 'DecJ'], psrs=[psrName])
tab = q.table()
if len(tab) == 0:
print('No pulsar found with name %s in ATNF! Try again.' % psrName)
print('\n\n')
sys.exit(1)
c = SkyCoord(ra=tab['RAJ'][0],
dec=tab['DECJ'][0],
frame='icrs',
unit=(u.hourangle, u.deg))
t = Time(toa, format='mjd')
sun = get_sun(t)
#Apparently this only works in the sun.separation(c) form; NOT the c.separation(sun) form
#Has to do with conversions between ICRS and GCRS frames
sep = sun.separation(c)
return sep.degree
def create_cfgs_main(kwargs, psrs_pointing_dict):
"""
uses kwargs from observation_processing_pipeline.py
"""
metadata, full_meta = get_common_obs_metadata(kwargs["obsid"],
return_all=True)
query = psrqpy.QueryATNF(loadfromdb=data_load.ATNF_LOC).pandas
cfgs = []
# Make the fold times dictionary (it's done for all pulsars simultaneously for speed)
psr_list = list(psrs_pointing_dict.keys())
fold_times_dict = find_fold_times(psr_list,
kwargs["obsid"],
kwargs["beg"],
kwargs["end"],
metadata=metadata,
full_meta=full_meta,
query=query)
for psr in progress_bar(psr_list, "Initiating pulsar configs: "):
logger.info(psr)
pointing_list = psrs_pointing_dict[psr]
enter = fold_times_dict[psr]["enter"]
leave = fold_times_dict[psr]["leave"]
power = fold_times_dict[psr]["power"]
try:
cfgs.append(
initiate_cfg(kwargs,
psr,
pointing_list,
enter,
leave,
power,
query=query,
metadata=metadata))
except TypeError as e:
logger.info(e)
continue
return cfgs
logger.setLevel(loglevels[args.loglvl])
ch = logging.StreamHandler()
ch.setLevel(loglevels[args.loglvl])
formatter = logging.Formatter(
'%(asctime)s %(filename)s %(name)s %(lineno)-4d %(levelname)-9s :: %(message)s'
)
ch.setFormatter(formatter)
logger.addHandler(ch)
logger.propagate = False
if args.obsid == None or args.pulsar == None:
logger.error("Obsid and Pulsar name must be supplied. Exiting...")
sys.exit(1)
query = psrqpy.QueryATNF(psrs=[args.pulsar], loadfromdb=ATNF_LOC).pandas
#Decide what to use as ra and dec
if args.pointing == None:
raj = query["RAJ"][0]
decj = query["DECJ"][0]
else:
raj = args.pointing.split("_")[0]
decj = args.pointing.split("_")[1]
if args.loglvl == "DEBUG":
plot = True
else:
plot = False
SN, SN_err = est_pulsar_sn(args.pulsar, args.obsid,\
beg=args.beg, end=args.end, p_ra=raj, p_dec=decj, plot_flux=plot, query=query)
import pandas as pd
import numpy as np
import altair as alt
alt.renderers.enable('notebook')
alt.data_transformers.enable('json')
import psrqpy
from astropy.coordinates import Angle
from astropy import units as u
from astropy.coordinates import ICRS, Galactic, FK4, FK5, SkyCoord
catalogue = psrqpy.QueryATNF(params=[
'PSRJ', 'P0', 'P1', 'DM', 'DIST', 'F0', 'F1', 'RAJ', 'DecJ', 'Dist_DM',
'AGE', 'BSurf', 'Date'
])
catalogue = catalogue.table
catalogue = catalogue.to_pandas()
coordinates = catalogue['RAJ'] + ' ' + catalogue['DECJ']
coordinates = coordinates.replace(to_replace='None None',
value='0:00:00 0:00:00')
catalogue['log_P0'] = np.log10(catalogue['P0'].values)
catalogue['log_P1'] = np.log10(catalogue['P1'].values)
catalogue['log_F0'] = np.log10(catalogue['F0'].values)
catalogue['log_AGE'] = np.log10(catalogue['AGE'].values)
catalogue['log_DIST_DM'] = np.log10(catalogue['DIST_DM'].values)
catalogue['DIST_DM'] = catalogue['DIST_DM'].fillna(0)
catalogue['BSURF'] = catalogue['BSURF'].fillna(0)
c = SkyCoord(coordinates.values, unit=(u.hourangle, u.deg))
catalogue['ra'] = c.ra.degree
catalogue['dec'] = c.dec.degree
xx = np.array([-0.51, 51.2])
def source_beam_coverage_and_times(obsid,
pulsar,
p_ra=None,
p_dec=None,
obs_beg=None,
obs_end=None,
files_beg=None,
files_end=None,
min_z_power=0.3,
dt_input=100,
common_metadata=None,
query=None,
beam='analytic'):
"""Finds the normalised time that a pulsar is in the beam for a given obsid.
If pulsar is not in beam, returns None, None
Parameters
----------
obsid : `int`
The observation ID
pulsar : `str`
The pulsar's J name
p_ra, p_dec : `str`, optional
The target's right ascension and declination in sexidecimals.
If not supplied will use the values from the ANTF.
obs_beg, obs_end : `int`, optional
Beginning and end GPS time of the observation.
If not supplied will use :py:meth:`vcstools.metadb_utils.obs_max_min` to find it.
files_beg, files_end : `int`, optional
Beginning and end GPS time of the (fits of VCS) files.
If not supplied will assume the full observation is available.
min_z_power : `float`, optional
Zenith normalised power cut off. |br| Default: 0.3.
common_metadata : `list`, optional
The list of common metadata generated from :py:meth:`vcstools.metadb_utils.get_common_obs_metadata`
query : psrqpy object, optional
A previous psrqpy query. Can be supplied to prevent performing a new query.
beam : `str`, optional
The primary beam model to use out of [analytic, advanced, full_EE]. |br| Default: analytic.
Returns
-------
enter_files : `float`
A float between 0 and 1 that describes the normalised time that the pulsar enters the beam
exit_files : `float`
A float between 0 and 1 that describes the normalised time that the pulsar exits the beam
"""
# Perform required metadata calls
if query is None:
query = psrqpy.QueryATNF(psrs=pulsar,
loadfromdb=data_load.ATNF_LOC).pandas
if p_ra is None or p_dec is None:
# Get some basic pulsar and obs info info
query_id = list(query['PSRJ']).index(pulsar)
p_ra = query["RAJ"][query_id]
p_dec = query["DECJ"][query_id]
if not common_metadata:
common_metadata = get_common_obs_metadata(obsid)
if obs_beg is None or obs_end is None:
obs_beg, obs_end = obs_max_min(obsid)
obs_dur = obs_end - obs_beg + 1
if not files_beg:
files_beg = obs_beg
if not files_end:
files_end = obs_end
files_dur = files_end - files_beg + 1
beam_coverage = source_beam_coverage(
[obsid], [[pulsar, p_ra, p_dec]],
common_metadata_list=[common_metadata],
dt_input=dt_input,
beam=beam,
min_z_power=min_z_power)
if pulsar not in beam_coverage[obsid].keys():
# Not in beam exiting
return None, None, None, None, None, None, None, None, None
dect_beg_norm, dect_end_norm, _ = beam_coverage[obsid][pulsar]
# GPS times the source enters and exits beam
dect_beg = obs_beg + obs_dur * dect_beg_norm
dect_end = obs_beg + obs_dur * dect_end_norm
# Normalised time the source enters/exits the beam in the files (used for Presto commands)
files_beg_norm = (dect_beg - files_beg) / files_dur
files_end_norm = (dect_end - files_beg) / files_dur
if files_beg_norm > 1. or files_end_norm < 0.:
logger.debug(
"source {0} is not in the beam for the files on disk".format(
pulsar))
files_beg_norm = None
files_end_norm = None
else:
if files_beg_norm < 0.:
files_beg_norm = 0.
if files_end_norm > 1.:
files_end_norm = 1.
return dect_beg, dect_end, dect_beg_norm, dect_end_norm, files_beg_norm, files_end_norm, obs_beg, obs_end, obs_dur
def beamform_and_fold(obsid,
DI_dir,
cal_obs,
args,
psrbeg,
psrend,
product_dir='/group/mwaops/vcs',
mwa_search_version='master'):
#obsbeg, obsend, obsdur = file_maxmin.print_minmax(obsid)
#wrapping for find_pulsar_in_obs.py
names_ra_dec = np.array(fpio.grab_source_alog(max_dm=250))
obs_data, meta_data = fpio.find_sources_in_obs([obsid],
names_ra_dec,
dt_input=100)
channels = meta_data[-1][-1]
oap = get_obs_array_phase(obsid)
centrefreq = 1.28 * float(min(channels) + max(channels)) / 2.
fwhm = calc_ta_fwhm(centrefreq, array_phase=oap)
# Sort all the sources into 3 categories, pulsars which is for slow pulsars, vdif
# for fast pulsars that require vdif and sp for singple pulse searches (FRBs,
# RRATs and pulsars without ATNF periods)
pulsar_pointing_list = []
pulsar_name_list = []
vdif_pointing_list = []
vdif_name_list = []
sp_pointing_list = []
sp_name_list = []
for pulsar_line in obs_data[obsid]:
vdif_check = False
sp_check = False
PSRJ = pulsar_line[0]
if not (len(PSRJ) < 11 or PSRJ[-1] == 'A' or PSRJ[-2:] == 'aa'):
continue
for line in names_ra_dec:
if PSRJ == line[0]:
temp = [line]
#temp = fpio.get_psrcat_ra_dec(pulsar_list=[PSRJ])
temp = fpio.format_ra_dec(temp, ra_col=1, dec_col=2)
jname, raj, decj = temp[0]
#get pulsar period
period = psrqpy.QueryATNF(params=["P0"],
psrs=[jname],
loadfromdb=ATNF_LOC).pandas["P0"][0]
if math.isnan(period):
print(
"WARNING: Period not found in ephermeris for {0} so assuming "
"it's an RRAT".format(jname))
sp_check = True
period = 0.
elif float(period) < .05:
vdif_check = True
period = float(period) * 1000.
print(
"{0:12} RA: {1} Dec: {2} Period: {3:8.2f} (ms) Begin {4} End {5}".
format(PSRJ, raj, decj, period, psrbeg, psrend))
jname_temp_list = []
if PSRJ[-1] == 'A' or PSRJ[-2:] == 'aa':
#Got to find all the pulsar J names with other letters
vdif_check = True
for pulsar_l in pulsar_lines:
if pulsar_l.startswith(PSRJ[:-2]):
jname_temp_list.append(pulsar_l.split()[0])
else:
jname_temp_list.append(jname)
#convert to radians
coord = SkyCoord(raj, decj, unit=(u.hourangle, u.deg))
rar = coord.ra.radian #in radians
decr = coord.dec.radian
#make a grid around each pulsar
grid_sep = np.radians(fwhm * 0.6)
rads, decds = get_grid(rar, decr, grid_sep, 1)
#convert back to sexidecimals
coord = SkyCoord(rads, decds, unit=(u.deg, u.deg))
rajs = coord.ra.to_string(unit=u.hour, sep=':')
decjs = coord.dec.to_string(unit=u.degree, sep=':')
temp = []
for raj, decj in zip(rajs, decjs):
temp.append([raj, decj])
pointing_list_list = fpio.format_ra_dec(temp, ra_col=0, dec_col=1)
# sort the pointings into the right groups
for prd in pointing_list_list:
if vdif_check:
vdif_name_list.append(jname_temp_list)
vdif_pointing_list.append("{0} {1}".format(prd[0], prd[1]))
elif sp_check:
sp_name_list.append(jname_temp_list)
sp_pointing_list.append("{0} {1}".format(prd[0], prd[1]))
else:
pulsar_name_list.append(jname_temp_list)
pulsar_pointing_list.append("{0} {1}".format(prd[0], prd[1]))
print('SENDING OFF PULSAR SEARCHS')
# Send off pulsar search
relaunch_script = 'mwa_search_pipeline.py -o {0} -O {1} --DI_dir {2} -b {3} -e {4} --channels'.format(
obsid, cal_obs, DI_dir, psrbeg, psrend)
for ch in channels:
relaunch_script = "{0} {1}".format(relaunch_script, ch)
search_opts = search_pipe.search_options_class(
obsid,
cal_id=cal_obs,
begin=psrbeg,
end=psrend,
channels=channels,
args=args,
DI_dir=DI_dir,
relaunch_script=relaunch_script,
search_ver=mwa_search_version)
search_pipe.beamform(search_opts,
pulsar_pointing_list,
pulsar_list_list=pulsar_name_list)
print('SENDING OFF VDIF PULSAR SEARCHS')
#Send off vdif pulsar search
relaunch_script = "{0} --vdif".format(relaunch_script)
search_opts = search_pipe.search_options_class(
obsid,
cal_id=cal_obs,
begin=psrbeg,
end=psrend,
channels=channels,
args=args,
DI_dir=DI_dir,
relaunch_script=relaunch_script,
search_ver=mwa_search_version,
vdif=True)
search_pipe.beamform(search_opts,
vdif_pointing_list,
pulsar_list_list=vdif_name_list)
#Get the rest of the singple pulse search canidates
orig_names_ra_dec = fpio.grab_source_alog(source_type='RRATs',
max_dm=250,
include_dm=True)
# remove any RRATs without at least arc minute accuracy
names_ra_dec = np.array(
[s for s in orig_names_ra_dec if (len(s[1]) > 4 and len(s[2]) > 4)])
obs_data, meta_data = fpio.find_sources_in_obs([obsid],
names_ra_dec,
dt_input=100)
for pulsar_line in obs_data[obsid]:
jname = pulsar_line[0]
for line in names_ra_dec:
if jname == line[0]:
jname, raj, decj, dm = line
jname_temp_list = [jname]
#convert to radians
coord = SkyCoord(raj, decj, unit=(u.hourangle, u.deg))
rar = coord.ra.radian #in radians
decr = coord.dec.radian
#make a grid around each pulsar
grid_sep = np.radians(fwhm * 0.6)
rads, decds = get_grid(rar, decr, grid_sep, 1)
#convert back to sexidecimals
coord = SkyCoord(rads, decds, unit=(u.deg, u.deg))
rajs = coord.ra.to_string(unit=u.hour, sep=':')
decjs = coord.dec.to_string(unit=u.degree, sep=':')
temp = []
for raj, decj in zip(rajs, decjs):
temp.append([raj, decj])
pointing_list_list = fpio.format_ra_dec(temp, ra_col=0, dec_col=1)
# sort the pointings into the right groups
for prd in pointing_list_list:
sp_name_list.append(jname_temp_list)
sp_pointing_list.append("{0} {1}".format(prd[0], prd[1]))
print('SENDING OFF SINGLE PULSE SEARCHS')
# Send off pulsar search
relaunch_script = 'mwa_search_pipeline.py -o {0} -O {1} --DI_dir {2} -b {3} -e {4} --single_pulse --channels'.format(
obsid, cal_obs, DI_dir, psrbeg, psrend)
for ch in channels:
relaunch_script = "{0} {1}".format(relaunch_script, ch)
search_opts = search_pipe.search_options_class(
obsid,
cal_id=cal_obs,
begin=psrbeg,
end=psrend,
channels=channels,
args=args,
DI_dir=DI_dir,
relaunch_script=relaunch_script,
search_ver=mwa_search_version,
single_pulse=True)
search_pipe.beamform(search_opts,
sp_pointing_list,
pulsar_list_list=sp_name_list,
code_comment="Single pulse search")
return
est_pulsar_flux, est_pulsar_sn,\
flux_calc_radiometer_equation,\
flux_calc_flux_profile
import logging
logger = logging.getLogger(__name__)
#logger = setup_logger(logger, log_level="DEBUG")
#Global obsid information to speed things up
md_dict={}
obsid_list = [1222697776, 1226062160, 1225713560, 1117643248]
for obs in obsid_list:
md_dict[str(obs)] = get_common_obs_metadata(obs, return_all=True)
query = psrqpy.QueryATNF(loadfromdb=data_load.ATNF_LOC).pandas
# The five pulsars from 1276619416 that were also detected in imaging.
pulsars_to_test = []
# J1820-0427 545.428 mJy. A bright pulsar with two components and a scattering tail that is about 50% of the profile
J1820_0427_profile = np.array([0.00974835457, 0.00275491384, 0.00109578621, -0.00279699934, 0.000562140998, 0.00113477532, 0.00339133085, -0.0109063454, -0.00964722073, 0.00111819766, 0.00510179576, -0.00238241189, -0.0159297665, 0.00526543335, 0.00104425447, -0.00416604215, -0.00577553402, -0.0107780994, -0.0082326258, -0.0134817171, -0.00582210704, -0.0124964885, -0.00268844238, -0.00974819377, -0.016102884, -0.0141605262, -0.00882266424, -0.0111797553, -0.0167948192, -0.00793797119, -0.00794229791, -0.000902770299, -2.88211273e-05, -0.00844239888, -0.00128177167, -0.00585817926, 0.00461725154, 0.118554769, 0.631823206, 0.978832608, 1.0, 0.833592824, 0.657390729, 0.523552684, 0.423043522, 0.361585454, 0.317663881, 0.288038185, 0.253027957, 0.223348542, 0.190868669, 0.16998052, 0.150608232, 0.158389622, 0.168857566, 0.17383913, 0.194769962, 0.185245049, 0.18829777, 0.171631238, 0.13852924, 0.136540455, 0.128550629, 0.121552035, 0.126898161, 0.11610917, 0.0966390774, 0.0836497683, 0.0709359634, 0.0663007999, 0.062169121, 0.0804554427, 0.0738022707, 0.0652449194, 0.0631586179, 0.0528215401, 0.0435259772, 0.037796751, 0.0423237659, 0.0357590644, 0.0347282449, 0.0329981882, 0.0295279872, 0.0228235617, 0.0293093176, 0.0313360707, 0.0207441587, 0.02619471, 0.00945243598, 0.0276467383, 0.0149054666, 0.011583468, 0.00358243583, 0.010961684, 0.0240656226, -0.00276778182, 0.00860143302, 0.00782033821, 0.011622846, 0.00154440406, 0.00617944115, -0.000217641207, 0.0141621455, 0.0112963973, 0.0187439007, 0.00671303776, 0.014185432, 0.00511642883, 0.00926998444, 0.00543009184, 0.00484238691, 0.00771907348, 0.0186740412, -0.00596104829, -0.00456473254, -1.99246096e-05, 0.00802933345, 0.00264869039, 0.0164105337, -0.00607378612, 0.00422687429, 0.0106680503, -0.00803578427, -0.00855158784, -0.0111729006, -0.00683086519, -0.00433629136, -0.00145620176])
J1820_0427_profile_bool = [True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True]
J1820_0427_data = [0.00808072162938938, 11.269025292925557, 2786.22, 17.813186127136287, 4920.0, 597.3405435783852, 74.26521548924087, 194.85305766037393, 16.8787754588467, 570.440121819025, 5.127532489305003]
pulsars_to_test.append( ("J1820-0427", 545.428, J1820_0427_profile, J1820_0427_profile_bool, J1820_0427_data) )
# J1825-0935 176.222 mJy. A bright pulsar with a small interpulse
J1825_0935_profile = np.array([0.00314088125, 0.000556045128, 0.00105395013, 0.0045659471, 0.000470929106, -0.000409939544, -0.00412380941, 0.00288133825, 0.0103584648, -0.00706736449, -0.000329613318, 0.00548152716, 0.00668359412, -0.00481527652, 0.00774982755, -0.00279912297, -0.00403820169, -0.00168432796, 0.000365357593, 0.0031319756, 0.000397067214, 0.0107867029, 0.0191924172, 0.0207955352, 0.018085596, 0.0247475429, 0.0163618917, 0.0198945505, 0.0172405494, 0.0250506153, 0.0946288765, 0.499182263, 1.0, 0.593519933, 0.135216572, 0.0189654022, 0.00493651504, -0.000583671957, 0.00145633429, -0.00247092049, 0.00340683702, 0.00189445108, -0.00245343584, 0.000542236211, 0.00387792879, 0.0181227866, 0.00592787911, -0.00376045883, 0.0049415421, 0.00555503833, 0.00204712627, 0.00205753797, 0.0036205589, -0.000353569177, 0.00817348899, 0.01150161, 0.00514606606, 0.00020703299, 0.00484254839, -0.00216935931, 0.00545115511, 0.0109659712, -0.000176346712, 0.0054985373, 0.00441087186, 0.00814568892, 0.00871168261, 0.00776448921, 0.0023226238, -0.000963073342, -0.00339329842, 0.00568027449, 0.00280664424, -0.00246610319, 0.00131316992, 0.000193402873, -0.00158659617, -0.00183816535, -0.00666429952, 0.0033070112, 0.00128840946, -0.00409108888, 0.0073903315, 0.00100847286, 0.00455245796, 0.00993097978, -0.000629802544, -0.0088467171, 0.00566167921, 0.00443358643, 0.00194643739, 0.00724301346, 0.0081186625, 0.0155670915, 0.0196579249, 0.0117332429, 0.0249344743, 0.0400196965, 0.0639603915, 0.0317363231, 0.00753277035, 0.00222084149, 0.0037240295, 0.00248847523, -0.00204282341, 0.00200497056, 0.00673926304, 0.0134845754, -0.00201286054, 0.00573517662, 0.00362153543, 0.00730374381, 0.00889853625, 0.00706360068, 0.00481938682, 0.00830079167, -0.00203974254, 0.00745314557, 0.00618885252, -0.00471675388, 0.00923441341, 0.000585897973, 0.0116158121, 0.00490610861, -0.00228758785, 0.00194633768, 0.00473734135, 0.00743085737])
J1825_0935_profile_bool = [True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True]
J1825_0935_data = [0.0031191764012526266, 3.0603445862819347, 2785.346666666667, 17.89875807816685, 4920.0, 440.5273973148081, 152.4567114103757, 254.19451614153593, 28.828176263272688, 435.22926553433194, 5.202306430741997]
pulsars_to_test.append( ("J1825-0935", 176.222, J1825_0935_profile, J1825_0935_profile_bool, J1825_0935_data) )
# J1834-0010 107.99 mJy
J1834_0010_profile = np.array([-0.2517820717787388, 0.1073720480401459, -0.03562029417799154, 0.12123301331614068, 0.022520123232647064, -0.03921288035870826, -0.00942285788080348, -0.08900187838661028, -0.0597638186089579, 0.07171540061051168, -0.11003798832690412, -0.07754092698059321, -0.005151024442136903, -0.06281649427867554, -0.053737611989087365, 0.06781081865710001, -0.029738541531959966, 0.08740268176424729, -0.04827681586605577, -0.12847620106115867, -0.05090738062152244, -0.01104385579823273, -0.13709881451627448, 0.03722849783537372, -0.01159681782225459, 0.18298493341726202, 0.07629630076944585, -0.009082114320017978, 0.16891643616191482, -0.20750281089398878, 0.01447809793486095, -0.17549843339571278, -0.011874560911267427, 0.0837931415388911, -0.3776954883456491, -0.0685699285838, -0.023140790634872684, 0.21150727061938127, 0.04155030435439834, -0.01871578498396509, -0.03225650145130725, -0.14180002981110043, 0.15766137991183468, -0.3079289054653312, -0.2103799157152547, -0.26779795815540575, -0.16311319365160062, 0.04472990275769002, 0.2057219613319569, -0.3741992335303586, -0.1743795353683869, -0.13591797564496597, -0.2620446961474828, -0.25851303425725924, 0.1743837709637479, -0.010241846784127956, 0.06913220035577065, 0.2274924019455588, 0.38604218214629044, 0.5534242614965105, 0.39911902285770867, 0.6414705436855919, 0.7880298170311245, 0.9850721704515689, 0.907415237223024, 1.0, 0.5084029165643795, 0.46409488204695876, 0.42737076730468254, 0.5010080929680137, 0.09016464898052999, 0.17994845368449675, 0.08770692416286939, -0.029463512123872635, 0.3539655259719344, 0.06465133595209765, -0.3172996333698848, -0.1636517085552666, 0.0433149895231194, 0.08655943341493454, -0.22292771786909613, -0.2628809406155497, -0.219690339599241, 0.05958428200907011, 0.14130675723678124, -0.027028539151686015, -0.0395821477212221, 0.040630090845121654, -0.07026903743758411, -0.06573744873775388, 0.08410298359656589, -0.3382789713822445, 0.135255076475651, 0.020477417576418622, 0.25000363437752965, -0.11972255558421015, -0.048019343542813675, -0.20919632008872022, 0.03227351460714043, -0.430842283157619, 0.08651209475878921, -0.24400147474794415, 0.25250771575728637, -0.1136311603181063, -0.008050579588773909, -0.036413912318377414, 0.07707627297146324, -0.1493332942089684, -0.0786053790922069, 0.056969017245107494, 0.03837770552492264, -0.15263450859202404, 0.0032237911776078726, 0.22013401920733605, 0.056147779863301814, -0.014655162069905482, -0.03770183390049333, 0.30842117288428333, -0.2566691957541137, -0.06079811871028795, -0.21165482885864825, -0.10278832512290409, 0.021476525974317578, 0.044770108309688125, 0.018076697287914195, -0.11719866002626701, -0.050156578335739326, -0.04280315775692326])
def est_pulsar_sn(pulsar, obsid,\
beg=None, end=None, p_ra=None, p_dec=None, obs_metadata=None, plot_flux=False,\
query=None, o_enter=None, o_exit=None, trcvr="/group/mwaops/PULSAR/MWA_Trcvr_tile_56.csv"):
"""
Estimates the signal to noise ratio for a pulsar in a given observation using the radiometer equation
S/N = (s_mean * gain * sqrt(n_p * t_int * df * (period - W_50)/W_50)) / t_sys
Note that W_50 should be W_equiv but we can't figure that out so we're estimating
Parameters:
----------
pulsar: string
Name of the pulsar e.g. J2241-5236
obsid: int
Observation ID e.g. 1226406800
beg: int
OPTIONAL - beginning of the observing time
end: int
OPTIONAL - end of the observing time
p_ra: str
OPTIONAL - the target's right ascension
p_dec: str
OPTIONAL - the target's declination
obs_metadata: list
OPTIONAL - the array generated from mwa_metadb_utils.get_common_obs_metadata(obsid)
plot_flux: boolean
OPTIONAL - whether or not to produce a plot of the flux estimation. Default = False
o_enter: float
OPTIONAL - The normalised o_enter time of the pulsar's coverage in the beam (between 0 and 1)
o_exit: float
OPTIONAL - The normalised o_exit time of the pulsar's covreage in the beam (between 0 and 1)
Returns:
--------
sn: float
The expected signal to noise ratio for the given inputs
sn_err: float
The uncertainty in the signal to noise ratio
"""
#We will attain uncertainties for s_mean, gain, t_sys and W_50.
# other uncertainties are considered negligible
if query is None:
query = psrqpy.QueryATNF(psrs=pulsar, loadfromdb=ATNF_LOC).pandas
if p_ra is None or p_dec is None:
#Get some basic pulsar and obs info info
p_ra = query["RAJ"][0]
p_dec = query["DECJ"][0]
#get metadata if not supplied
if obs_metadata is None:
logger.debug("Obtaining obs metadata")
obs_metadata = mwa_metadb_utils.get_common_obs_metadata(obsid)
n_p = 2 #constant
df = 30.72e6 #(24*1.28e6)
#estimate flux
s_mean, s_mean_err = est_pulsar_flux(pulsar, obsid, plot_flux=plot_flux,\
metadata=obs_metadata, query=query)
#fluxes may be Nones. If so, return None
if s_mean is None and s_mean_err is None:
return None, None
#find integration time
if o_enter is not None and o_exit is not None:
t_int = o_exit - o_enter
if beg is not None and end is not None:
t_int = t_int * (end - beg)
else:
t_int = t_int * obs_metadata[3] #duration
else:
beg, end, t_int = find_times(obsid, pulsar, beg=beg, end=end)
if t_int <= 0.:
logger.warning(
"Pulsar not in beam for obs files or specificed beginning and end times"
)
return 0., 0.
#find system temp and gain
t_sys, t_sys_err, gain, gain_err = find_t_sys_gain(pulsar, obsid,\
beg=beg, p_ra=p_ra, p_dec=p_dec, query=query,\
obs_metadata=obs_metadata, trcvr=trcvr)
#Find W_50
W_50, W_50_err = find_pulsar_w50(pulsar, query=query)
#calculate SN
period = float(query["P0"][0])
SN = ((s_mean * gain) / t_sys) * np.sqrt(n_p * t_int * df *
(period - W_50) / W_50)
#Calculate SN uncertainty using variance formula. Assuming error from period, df and t_int is zero
dc_expr = np.sqrt((period - W_50) / W_50)
var_s_mean = (gain * np.sqrt(n_p * t_int * df)) * dc_expr / t_sys
var_gain = s_mean * np.sqrt(n_p * t_int * df) * dc_expr / t_sys
var_W_50 = s_mean * gain * np.sqrt(
n_p * t_int * df) / t_sys * (period / (-2. * W_50**2.)) * dc_expr**-1
var_t_sys = -s_mean * gain * np.sqrt(
n_p * t_int * df) * dc_expr / t_sys**2.
var_s_mean = var_s_mean**2. * s_mean_err**2.
var_gain = var_gain**2. * gain_err**2.
var_W_50 = var_W_50**2. * W_50_err**2.
var_t_sys = var_t_sys**2. * t_sys_err**2.
logger.debug("variance estimates for s_mean: {0}, gain: {1}, W_50: {2}, t_sys: {3}"\
.format(var_s_mean, var_gain, var_W_50, var_t_sys))
SN_err = np.sqrt(var_s_mean + var_gain + var_W_50 + var_t_sys)
logger.debug("S_mean: {0} +/- {1}".format(s_mean, s_mean_err))
logger.debug("Gain: {0} +/- {1}".format(gain, gain_err))
logger.debug("t_int: {0}".format(t_int))
logger.debug("df: {0}".format(df))
logger.debug("period: {0}".format(period))
logger.debug("W_50: {0} +/- {1}".format(W_50, W_50_err))
logger.debug("t_sys: {0} +/- {1}".format(t_sys, t_sys_err))
return SN, SN_err
def find_t_sys_gain(pulsar, obsid, beg=None, t_int=None, p_ra=None, p_dec=None,\
obs_metadata=None, query=None, trcvr="/group/mwaops/PULSAR/MWA_Trcvr_tile_56.csv"):
"""
Finds the system temperature and gain for an observation.
A function snippet originally written by Nick Swainston - adapted for general VCS use.
Parameters:
-----------
pulsar: str
the J name of the pulsar. e.g. J2241-5236
obsid: int
The observation ID. e.g. 1226406800
beg: int
The beginning of the observing time
t_int: float
The total time that the target is in the beam
p_ra: str
OPTIONAL - the target's right ascension
p_dec: str
OPTIONAL - the target's declination
obs_metadata: list
OPTIONAL - the array generated from mwa_metadb_utils.get_common_obs_metadata(obsid)
query: object
OPTIONAL - The return of the psrqpy function for this pulsar
trcvr: str
The location of the MWA receiver temp csv file. Default = '/group/mwaops/PULSAR/MWA_Trcvr_tile_56.csv'
Returns:
--------
t_sys: float
The system temperature
t_sys_err: float
The system temperature's uncertainty
gain: float
The system gain
gain_err: float
The gain's uncertainty
"""
#get ra and dec if not supplied
if p_ra is None or p_dec is None and query is None:
logger.debug("Obtaining pulsar RA and Dec from ATNF")
query = psrqpy.QueryATNF(psrs=[pulsar], loadfromdb=ATNF_LOC).pandas
p_ra = query["RAJ"][0]
p_dec = query["DECJ"][0]
elif query is not None:
query = psrqpy.QueryATNF(psrs=[pulsar], loadfromdb=ATNF_LOC).pandas
p_ra = query["RAJ"][0]
p_dec = query["DECJ"][0]
#get metadata if not supplied
if obs_metadata is None:
logger.debug("Obtaining obs metadata")
obs_metadata = mwa_metadb_utils.get_common_obs_metadata(obsid)
obsid, obs_ra, obs_dec, _, delays, centrefreq, channels = obs_metadata
#get beg if not supplied
if beg is None or t_int is None:
logger.debug("Calculating beginning time for pulsar coverage")
beg, _, t_int = find_times(obsid, pulsar, beg=beg)
#Find 'start_time' for fpio - it's usually about 7 seconds
#obs_start, _ = mwa_metadb_utils.obs_max_min(obsid)
start_time = beg - int(obsid)
#Get important info
trec_table = Table.read(trcvr, format="csv")
ntiles = 128 #TODO actually we excluded some tiles during beamforming, so we'll need to account for that here
beam_power = fpio.get_beam_power_over_time([obsid, obs_ra, obs_dec, t_int, delays,\
centrefreq, channels],\
np.array([[pulsar, p_ra, p_dec]]),\
dt=100, start_time=start_time)
beam_power = np.mean(beam_power)
# Usa a primary beam function to convolve the sky temperature with the primary beam
# (prints suppressed)
sys.stdout = open(os.devnull, 'w')
_, _, Tsky_XX, _, _, _, Tsky_YY, _ = pbtant.make_primarybeammap(
int(obsid), delays, centrefreq * 1e6, 'analytic', plottype='None')
sys.stdout = sys.__stdout__
#TODO can be inaccurate for coherent but is too difficult to simulate
t_sky = (Tsky_XX + Tsky_YY) / 2.
# Get T_sys by adding Trec and Tsky (other temperatures are assumed to be negligible
t_sys_table = t_sky + submit_to_database.get_Trec(trec_table, centrefreq)
t_sys = np.mean(t_sys_table)
t_sys_err = t_sys * 0.02 #TODO: figure out what t_sys error is
logger.debug("pul_ra: {} pul_dec: {}".format(p_ra, p_dec))
_, _, zas = mwa_metadb_utils.mwa_alt_az_za(obsid, ra=p_ra, dec=p_dec)
theta = np.radians(zas)
gain = submit_to_database.from_power_to_gain(beam_power,
centrefreq * 1e6,
ntiles,
coh=True)
logger.debug("beam_power: {} theta: {} pi: {}".format(
beam_power, theta, np.pi))
gain_err = gain * ((1. - beam_power) * 0.12 + 2. *
(theta / (0.5 * np.pi))**2. + 0.1)
# Removed the below error catch because couldn't find an obs that breaks it
#sometimes gain_err is a numpy array and sometimes it isnt so i have to to this...
#try:
# gain_err.shape
# gain_err = gain_err[0]
#except:
# pass
return t_sys, t_sys_err, gain, gain_err
def flux_from_atnf(pulsar, query=None):
"""Queries the ATNF database for flux and spectral index info on a particular pulsar at all frequencies
Parameters
----------
pulsar : `str`
The Jname of the pulsar.
query : psrqpy object, optional
A previous psrqpy.QueryATNF query. Can be supplied to prevent performing a new query.
Returns
-------
freq_all : `list`
All frequencies in Hz with flux values on ATNF.
flux_all : `list`
The flux values corresponding to the freq_all list in Jy.
flux_err_all : `list`
The uncertainty in the flux_all values.
spind : `float`
The spectral index from ATNF, will be None if not available.
spind_err : `float`
The ucnertainty in spind from ATNF, will be None if not available.
"""
if query is None:
query = psrqpy.QueryATNF(psrs=[pulsar],
loadfromdb=data_load.ATNF_LOC).pandas
query_id = list(query['PSRJ']).index(pulsar)
flux_queries = ["S40", "S50", "S60", "S80", "S100", "S150", "S200",\
"S300", "S400", "S600", "S700", "S800", "S900",\
"S1400", "S1600", "S2000", "S3000", "S4000", "S5000",\
"S6000", "S8000"]
freq_all = []
flux_all = []
flux_err_all = []
#Get all available data from dataframe and check for missing values
for flux_query in flux_queries:
flux = query[flux_query][query_id]
if not np.isnan(flux):
#sometimes error values don't exist, causing a key error in pandas
try:
flux_err = query[flux_query + "_ERR"][query_id]
if flux_err == 0.0:
logger.debug("{0} flux error for query: {1}, is zero. Assuming 20% uncertainty"\
.format(pulsar, flux_query))
flux_err = flux * 0.2
except KeyError:
logger.debug("{0} flux error value {1}, not available. assuming 20% uncertainty"\
.format(pulsar, flux_query))
flux_err = flux * 0.2
if np.isnan(flux_err):
logger.debug("{0} flux error value for {1} not available. assuming 20% uncertainty"\
.format(pulsar, flux_query))
flux_err = flux * 0.2
freq_all.append(int(flux_query.split()[0][1:]) *
1e6) #convert to Hz
flux_all.append(flux * 1e-3) #convert to Jy
flux_err_all.append(flux_err * 1e-3) #convert to Jy
#Also get spectral index if it exists
spind = query["SPINDX"][query_id]
spind_err = query["SPINDX_ERR"][query_id]
return freq_all, flux_all, flux_err_all, spind, spind_err
psrNames = []
psrData = []
for folder in folders:
if folder == "surveyOutput": continue
psrName = "J" + folder.split('J')[1]
psrNames.append(psrName)
psrPath = f"./{folder}"
for fil in os.listdir(folder):
if "udp_16130" in fil:
arbName = fil.replace("16130", "1613%d")
psrUdp = f"{psrPath}/{arbName}"
psrData.append((psrName, psrPath, psrUdp))
universe = psrqpy.QueryATNF()
pulsarData = {}
for psr in psrNames:
pulsar = universe.get_pulsar(psr)
dm, period, raj, decj = float(pulsar['DM']), float(pulsar['P0']), pulsar['RAJ'][0], pulsar['DECJ'][0]
print(psr, dm, period, raj, decj)
pulsarData[psr] = [dm, period, raj, decj]
with open("./batchProccess.sh", 'w+') as ref:
for name, folder, data in psrData:
def initiate_cfg(kwargs,
psr,
pointings,
enter,
leave,
power,
query=None,
metadata=None):
"""
Adds all available keys to the cfg dictionary and figures out some useful constants
Takes kwargs from observation_processing_pipeline
"""
cfg = {
"obs": {},
"source": {},
"completed": {},
"folds": {},
"run_ops": {},
"pol": {},
"files": {}
}
if query is None:
query = psrqpy.QueryATNF(loadfromdb=data_load.ATNF_LOC).pandas
if metadata is None:
metadata = get_common_obs_metadata(kwargs["obsid"])
query_index = list(query["JNAME"]).index(psr)
cfg["obs"]["ra"] = metadata[1]
cfg["obs"]["dec"] = metadata[2]
cfg["obs"]["dur"] = metadata[3]
cfg["obs"]["freq"] = metadata[5]
cfg["obs"]["id"] = kwargs["obsid"]
cfg["obs"]["cal"] = kwargs["calid"]
cfg["obs"]["beg"] = kwargs["beg"]
cfg["obs"]["end"] = kwargs["end"]
cfg["run_ops"]["loglvl"] = kwargs["loglvl"]
cfg["run_ops"]["mwa_search"] = kwargs["mwa_search"]
cfg["run_ops"]["vcstools"] = kwargs["vcstools"]
cfg["run_ops"]["label"] = kwargs["label"]
cfg["run_ops"]["thresh_chi"] = 3.5
cfg["run_ops"]["thresh_sn"] = 8.0
cfg["run_ops"]["good_chi"] = 4.0
cfg["run_ops"]["good_sn"] = 20.0
cfg["run_ops"]["vdif"] = None
cfg["run_ops"]["mask"] = None
cfg["files"]["file_precursor"] = file_precursor(kwargs, psr)
cfg["files"]["psr_dir"] = join(comp_config["base_data_dir"],
str(cfg["obs"]["id"]), "dpp",
cfg["files"]["file_precursor"])
cfg["files"]["batch_dir"] = join(comp_config['base_data_dir'],
cfg["obs"]["id"], "batch")
cfg["files"]["classify_dir"] = join(cfg["files"]["psr_dir"],
"classifier_ppp")
cfg["files"]["my_name"] = join(
cfg["files"]["psr_dir"], f"{cfg['files']['file_precursor']}_cfg.yaml")
cfg["files"]["logfile"] = join(cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}.log")
cfg["files"]["archive"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_archive.ar")
cfg["files"]["archive_ascii"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_archive.txt")
cfg["files"]["gfit_plot"] = join(
cfg["files"]["psr_dir"], f"{cfg['files']['file_precursor']}_gfit.png")
cfg["files"]["converted_fits"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_archive.fits")
# debased fits file needs to be the same as archive except for the extension
arch = cfg["files"]["archive"].split(".ar")[0]
cfg["files"]["debased_fits"] = f"{arch}.debase.gg"
# paswing file needs to be the same as debase except for the .gg extension
debase = cfg["files"]["debased_fits"].split(".gg")[0]
cfg["files"]["paswing"] = f"{debase}.paswing"
cfg["files"]["chigrid_initial_ps"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_chigrid_initial.ps")
cfg["files"]["paswing_initial_ps"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_paswing_initial.ps")
cfg["files"]["RVM_fit_initial"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_RVM_fit_initial.out")
cfg["files"]["chigrid_final_ps"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_chigrid_final.ps")
cfg["files"]["paswing_final_ps"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_paswing_final.ps")
cfg["files"]["RVM_fit_final"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_RVM_fit_final.out")
cfg["files"]["ppol_profile_ps"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_profile.ps")
cfg["files"]["ppol_polar_profile_ps"] = join(
cfg["files"]["psr_dir"], f"{cfg['files']['file_precursor']}_pol.ps")
cfg["source"]["enter_frac"] = None
cfg["source"]["exit_frac"] = None
cfg["source"]["power"] = None
cfg["source"]["name"] = psr
try:
cfg["source"]["enter_frac"] = float(enter)
cfg["source"]["exit_frac"] = float(leave)
cfg["source"]["power"] = float(power)
except TypeError as _: #If any of these are nones, the pulsar isn't in the beam for given beg, end
raise TypeError(f"{cfg['source']['name']} not in beam for given times")
cfg["source"]["sampling_limit"] = int(
bin_sampling_limit(cfg["source"]["name"], query=query))
cfg["source"]["ATNF_P"] = float(query["P0"][query_index])
cfg["source"]["ATNF_DM"] = float(query["DM"][query_index])
cfg["source"]["my_DM"] = None
cfg["source"]["my_P"] = None
cfg["source"]["my_Pdot"] = None
cfg["source"]["my_bins"] = None
cfg["source"]["my_pointing"] = None
cfg["source"]["my_component"] = None
cfg["source"]["gfit"] = None
cfg["source"]["binary"] = is_binary(cfg["source"]["name"], query=query)
cfg["source"]["seek"] = cfg["source"]["enter_frac"] * (cfg["obs"]["end"] -
cfg["obs"]["beg"])
cfg["source"]["total"] = (cfg["source"]["exit_frac"] -
cfg["source"]["enter_frac"]) * (
cfg["obs"]["end"] - cfg["obs"]["beg"])
if cfg["source"]["binary"]:
cfg["source"]["edited_eph_name"] = join(
cfg["files"]["psr_dir"], f"{cfg['files']['file_precursor']}.eph")
cfg["source"]["edited_eph"] = create_edited_eph(cfg["source"]["name"])
else:
cfg["source"]["edited_eph"] = None
cfg["source"]["edited_eph_name"] = None
init, post = required_bin_folds(cfg["source"]["name"], query=query)
for pointing in pointings:
cfg["folds"] = {pointing: {"init": {}, "post": {}}}
cfg["folds"][pointing]["classifier"] = 0
cfg["folds"][pointing]["dir"] = join(cfg["files"]["psr_dir"], pointing)
for _, i in enumerate(init):
cfg["folds"][pointing]["init"][str(i)] = {}
for _, i in enumerate(post):
cfg["folds"][pointing]["post"][str(i)] = {}
cfg["pol"]["RM"] = None
cfg["pol"]["RM_e"] = None
cfg["pol"]["alpha"] = None
cfg["pol"]["beta"] = None
cfg["pol"]["l0"] = None
cfg["pol"]["pa0"] = None
cfg["pol"]["chi"] = None
cfg["completed"] = {}
cfg["completed"]["init_folds"] = False
cfg["completed"]["classify"] = False
cfg["completed"]["post_folds"] = False
cfg["completed"]["upload"] = False
cfg["completed"]["debase"] = False
cfg["completed"]["RM"] = False
cfg["completed"]["RVM_initial"] = False
cfg["completed"]["RVM_final"] = False
return cfg
def flux_from_atnf(pulsar, query=None):
"""
Queries the ATNF database for flux and spectral index info on a particular pulsar at all frequencies
Parameters:
-----------
pulsar: string
The J name of the pulsar
query: object
OPTIONAL - The return from psrqpy.QueryATNF for this pulsar. Default: None
Returns:
--------
freq_all: list
All frequencies in Hz with flux values on ATNF
flux_all: list
The flux values corresponding to the freq_all list in Jy
flux_err_all: list
The uncertainty in the flux_all values
spind: float
The spectral index from ATNF, will be None if not available
spind_err: float
The ucnertainty in spind from ATNF, will be None if not available
"""
if query is None:
query = psrqpy.QueryATNF(psrs=[pulsar],
loadfromdb=data_load.ATNF_LOC).pandas
flux_queries = ["S40", "S50", "S60", "S80", "S100", "S150", "S200",\
"S300", "S400", "S600", "S700", "S800", "S900",\
"S1400", "S1600", "S2000", "S3000", "S4000", "S5000",\
"S6000", "S8000"]
freq_all = []
flux_all = []
flux_err_all = []
#Get all available data from dataframe and check for missing values
for flux_query in flux_queries:
flux = query[flux_query][0]
if not np.isnan(flux):
#sometimes error values don't exist, causing a key error in pandas
try:
flux_err = query[flux_query + "_ERR"][0]
if flux_err == 0.0:
logger.warning("{0} flux error for query: {1}, is zero. Assuming 20% uncertainty"\
.format(pulsar, flux_query))
flux_err = flux * 0.2
except KeyError:
logger.warning("{0} flux error value {1}, not available. assuming 20% uncertainty"\
.format(pulsar, flux_query))
flux_err = flux * 0.2
if np.isnan(flux_err):
logger.warning("{0} flux error value for {1} not available. assuming 20% uncertainty"\
.format(pulsar, flux_query))
flux_err = flux * 0.2
freq_all.append(int(flux_query.split()[0][1:]) *
1e6) #convert to Hz
flux_all.append(flux * 1e-3) #convert to Jy
flux_err_all.append(flux_err * 1e-3) #convert to Jy
#Also get spectral index if it exists
spind = query["SPINDX"][0]
spind_err = query["SPINDX_ERR"][0]
return freq_all, flux_all, flux_err_all, spind, spind_err
logger.setLevel(loglevels[args.loglvl])
ch = logging.StreamHandler()
ch.setLevel(loglevels[args.loglvl])
formatter = logging.Formatter(
'%(asctime)s %(filename)s %(name)s %(lineno)-4d %(levelname)-9s :: %(message)s'
)
ch.setFormatter(formatter)
logger.addHandler(ch)
logger.propagate = False
if args.mode == "SNFE":
if not args.obsid or not args.pulsar:
logger.error("Obsid and Pulsar name must be supplied. Exiting...")
sys.exit(1)
pulsar = args.pulsar[0]
query = psrqpy.QueryATNF(psrs=[pulsar],
loadfromdb=data_load.ATNF_LOC).pandas
#Decide what to use as ra and dec
if args.pointing is None:
raj = None
decj = None
else:
raj = args.pointing.split("_")[0]
decj = args.pointing.split("_")[1]
SN, SN_err = est_pulsar_sn(pulsar, args.obsid,\
beg=args.beg, end=args.end, p_ra=raj, p_dec=decj, plot_flux=args.plot_est, query=query, min_z_power=args.min_z_power)
elif args.mode == "ATNF":
if not args.pulsar:
logger.error("Pulsar name must be supplied. Exiting...")
sys.exit(1)
ATNF_spectral_data_plot(args.pulsar)
def find_pulsars_in_fov(obsid,
psrbeg,
psrend,
fwhm=None,
search_radius=0.02,
meta_data=None,
full_meta=None,
no_known_pulsars=False,
no_search_cands=False):
"""
Find all pulsars in the field of view and return all the pointings sorted into vdif and normal lists:
Parameters:
-----------
obsid: int
The observation ID
psrbeg: int
The begining of the observation you are processing in GPS time
psrend: int
The end of the observation you are processing in GPS time
fwhm: float
OPTIONAL - The FWHM of the beam in degrees.
Default None: Value will be estimated
search_radius: float
OPTIONAL - The radius to search (create beams within) in degrees to account for ionosphere.
Default: 0.02 degrees
meta_data: list
OPTIONAL - Comon observation metadata in the format from vcstools.metadb_utils.get_common_obs_metadata
Default None: Will perform the metadata call
full_meta: dict
OPTIONAL - Full observation metadata in the format from vcstools.metadb_utils.getmeta
Default None: Will perform the metadata call
no_known_pulsars: bool
OPTIONAL - Will return no known pulsars
Default: False
no_search_cands: bool
OPTIONAL - Will return no search candidates
Default: False
Returns:
--------
list of lists:
[pulsar_name_list,
pulsar_pointing_list,
vdif_name_list,
vdif_pointing_list,
sp_name_list,
sp_pointing_list]
"""
if not meta_data or not full_meta:
meta_data, full_meta = get_common_obs_metadata(obsid, return_all=True)
channels = meta_data[-1]
if fwhm is None:
# Estimate FWHM
oap = get_obs_array_phase(obsid)
centrefreq = 1.28 * float(min(channels) + max(channels)) / 2.
fwhm = calc_ta_fwhm(centrefreq, array_phase=oap)
# Find all pulsars in beam at at least 0.3 and 0.1 of zenith normlaized power
names_ra_dec = np.array(grab_source_alog(max_dm=250))
pow_dict, _ = find_pulsars_power(obsid,
powers=[0.3, 0.1],
names_ra_dec=names_ra_dec)
obs_psrs = pow_dict[0.3][obsid]
# Find pulsars with power between 0.3 and 0.1 and calculate their SN
psrs_list_03 = [x[0] for x in obs_psrs]
psrs_list_01 = [x[0] for x in pow_dict[0.1][obsid]]
psrs_03_01 = psrs_list_01
for psr in psrs_list_03:
if psr in psrs_list_01:
psrs_03_01.remove(psr)
sn_dict_01 = snfu.multi_psr_snfe(psrs_03_01,
obsid,
beg=psrbeg,
end=psrend,
min_z_power=0.1,
obs_metadata=meta_data,
full_meta=full_meta)
# Include all bright pulsars in beam at at least 0.1 of zenith normalized power
for psr in psrs_03_01:
sn, sn_err, _, _ = sn_dict_01[psr]
if sn is not None and sn_err is not None:
if sn - sn_err >= 10.:
for psr_list in pow_dict[0.1][obsid]:
if psr in psr_list:
obs_psrs.append(psr_list)
#get all the pulsars periods
pulsar_list = []
for o in obs_psrs:
pulsar_list.append(o[0])
period_query = psrqpy.QueryATNF(params=["PSRJ", "P0"],
psrs=pulsar_list,
loadfromdb=data_load.ATNF_LOC).pandas
# Sort all the sources into 3 categories, pulsars which is for slow pulsars, vdif
# for fast pulsars that require vdif and sp for singple pulse searches (FRBs,
# RRATs and pulsars without ATNF periods)
pulsar_pointing_list = []
pulsar_name_list = []
vdif_pointing_list = []
vdif_name_list = []
pulsar_search_pointing_list = []
pulsar_search_name_list = []
sp_pointing_list = []
sp_name_list = []
for pi, pulsar_line in enumerate(obs_psrs):
vdif_check = False
sp_check = False
PSRJ = pulsar_line[0]
if not (len(PSRJ) < 11 or PSRJ[-1] == 'A' or PSRJ[-2:] == 'aa'):
continue
for line in names_ra_dec:
if PSRJ == line[0]:
temp = [line]
temp = format_ra_dec(temp, ra_col=1, dec_col=2)
jname, raj, decj = temp[0]
#get pulsar period
period = period_query[period_query['PSRJ'] ==
PSRJ].reset_index()["P0"][0]
if math.isnan(period):
logger.warn("Period not found in ephermeris for {0} so assuming "
"it's an RRAT".format(jname))
sp_check = True
period = 0.
elif float(period) < .05:
vdif_check = True
period = float(period) * 1000.
logger.debug(
"{0:12} RA: {1} Dec: {2} Period: {3:8.2f} (ms) Begin {4} End {5}".
format(PSRJ, raj, decj, period, psrbeg, psrend))
jname_temp_list = []
if PSRJ[-1] == 'A' or PSRJ[-2:] == 'aa':
#Got to find all the pulsar J names with other letters
vdif_check = True
for pulsar_l in obs_psrs:
pulsar_name = pulsar_l[0]
if pulsar_name.startswith(PSRJ[:-2]):
jname_temp_list.append(pulsar_name)
else:
jname_temp_list.append(jname)
# grid the pointings to fill 2 arcminute raduis to account for ionosphere shift
pointing_list_list = get_pointings_required(raj, decj, fwhm,
search_radius)
# sort the pointings into the right groups
for prd in pointing_list_list:
if vdif_check:
vdif_name_list.append(jname_temp_list)
vdif_pointing_list.append("{0}_{1}".format(prd[0], prd[1]))
elif sp_check:
sp_name_list.append(jname_temp_list)
sp_pointing_list.append("{0}_{1}".format(prd[0], prd[1]))
else:
pulsar_name_list.append(jname_temp_list)
pulsar_pointing_list.append("{0}_{1}".format(prd[0], prd[1]))
#Get the rest of the singple pulse search canidates
#-----------------------------------------------------------------------------------------------------------
temp = get_sources_in_fov(obsid, 'RRATs', fwhm)
sp_name_list = sp_name_list + temp[0]
sp_pointing_list = sp_pointing_list + temp[1]
# Find all of the FRB candidates
#-----------------------------------------------------------------------------------------------------------
temp = get_sources_in_fov(obsid, 'FRB', fwhm)
sp_name_list = sp_name_list + temp[0]
sp_pointing_list = sp_pointing_list + temp[1]
# Find all of the Fermi candidates
#-----------------------------------------------------------------------------------------------------------
fermi_list = get_sources_in_fov(obsid, 'Fermi', fwhm)
logger.info(f"{obsid} Fermi candidates: {fermi_list}")
pulsar_search_name_list = pulsar_search_name_list + fermi_list[0]
pulsar_search_pointing_list = pulsar_search_pointing_list + fermi_list[1]
# Find all of the points of interest candidates
#-----------------------------------------------------------------------------------------------
poi_list = get_sources_in_fov(obsid, 'POI', fwhm)
logger.info(f"{obsid} Points of interest: {poi_list}")
pulsar_search_name_list = pulsar_search_name_list + poi_list[0]
pulsar_search_pointing_list = pulsar_search_pointing_list + poi_list[1]
# Sometimes we get redundant RRATs that are found in RRAT and ANTF catalogues so they need to be removed
sp_name_list = list(dict.fromkeys([":".join(s) for s in sp_name_list]))
sp_pointing_list = list(dict.fromkeys(sp_pointing_list))
# Changing the format of the names list to make it easier to format
pulsar_name_list = [":".join(s) for s in pulsar_name_list]
vdif_name_list = [":".join(s) for s in vdif_name_list]
pulsar_search_name_list = [":".join(s) for s in pulsar_search_name_list]
if no_known_pulsars:
# Return empty list for all known pulsar categories
pulsar_name_list = []
pulsar_pointing_list = []
vdif_name_list = []
vdif_pointing_list = []
if no_search_cands:
# Return empty list for all search candidate categories
pulsar_search_name_list = []
pulsar_search_pointing_list = []
sp_name_list = []
sp_pointing_list = []
return [
pulsar_name_list, pulsar_pointing_list, vdif_name_list,
vdif_pointing_list, pulsar_search_name_list,
pulsar_search_pointing_list, sp_name_list, sp_pointing_list
]
def est_pulsar_flux(pulsar, obsid, plot_flux=False, metadata=None, query=None):
"""
Estimates a pulsar's flux from archival data by assuming a power law relation between flux and frequency. Frist tries to attain a apectral index from the ATNF database. If this fails, try to work out a spectrla index. If this fails, uses an index of -1.4 with uncertainty of 1.
Parameters:
-----------
pulsar: string
The puslar's name. e.g. 'J2241-5236'
obsid: int
The observation ID
plot_flux: boolean
OPTIONAL - Whether or not to make a plot of the flux estimation. Default = False
metadata: list
OPTIONAL - The metadata call for this obsid
query: object
OPTIONAL - The return from psrqpy.QueryATNF for this pulsar
Returns:
-------
flux: float
The estimated flux in Jy
flux_err: float
The estimated flux's uncertainty in Jy
"""
if metadata is None:
logger.debug("obtaining mean freq from obs metadata")
metadata = mwa_metadb_utils.get_common_obs_metadata(obsid)
f_mean = metadata[5] * 1e6
if query is None:
query = psrqpy.QueryATNF(psrs=[pulsar], loadfromdb=ATNF_LOC).pandas
flux_queries = ["S40", "S50", "S60", "S80", "S100", "S150", "S200",\
"S300", "S400", "S600", "S700", "S800", "S900",\
"S1400", "S1600", "S2000", "S3000", "S4000", "S5000",\
"S6000", "S8000"]
freq_all = []
flux_all = []
flux_err_all = []
#Get all available data from dataframe and check for missing values
for flux_query in flux_queries:
flux = query[flux_query][0]
if not np.isnan(flux):
#sometimes error values don't exist, causing a key error in pandas
try:
flux_err = query[flux_query + "_ERR"][0]
if flux_err == 0.0:
logger.warning(
"Flux error for query: {0}, pulsar {1}, is zero. Assuming 20% uncertainty"
.format(query[flux_query][0], pulsar))
flux_err = flux * 0.2
except KeyError:
logger.warning(
"flux error value for {0}, pulsar {1}, not available. assuming 20% uncertainty"
.format(query[flux_query][0], pulsar))
flux_err = flux * 0.2
if np.isnan(flux_err):
logger.warning(
"flux error value for {0}, pulsar {1}, not available. assuming 20% uncertainty"
.format(query[flux_query][0], pulsar))
flux_err = flux * 0.2
freq_all.append(int(flux_query.split()[0][1:]) *
1e6) #convert to Hz
flux_all.append(flux * 1e-3) #convert to Jy
flux_err_all.append(flux_err * 1e-3) #convert to Jy
#Also get spectral index if it exists
spind = query["SPINDX"][0]
spind_err = query["SPINDX_ERR"][0]
logger.debug("Freqs: {0}".format(freq_all))
logger.debug("Fluxes: {0}".format(flux_all))
logger.debug("Flux Errors: {0}".format(flux_err_all))
logger.info(
"There are {0} flux values available on the ATNF database for {1}".
format(len(flux_all), pulsar))
#Attempt to estimate flux
if len(flux_all) > 1:
logger.info("Fitting power law to archive data")
for i, _ in enumerate(flux_all):
flux_all[i] = flux_all[i]
flux_err_all[i] = flux_err_all[i]
#apply spectral index bounds if an error already exists
initial_spind = -1.5
spind_bounds = None
if not np.isnan(spind):
initial_spind = spind
if not np.isnan(spind_err):
#if spind_error exists on cat, use it to create 5 sigma bounds for fitting
spind_bounds = [spind - spind_err * 5, spind + spind_err * 5]
spind, c, covar_matrix = fit_plaw_psr(freq_all,
flux_all,
alpha_initial=initial_spind,
alpha_bound=spind_bounds)
logger.info("Derived spectral index: {0} +/- {1}".format(
spind, covar_matrix.item(0)))
#plot if in debug mode
if plot_flux == True:
plot_flux_estimation(freq_all,
flux_all,
flux_err_all,
pulsar,
obsid,
alpha=spind,
c=c)
#flux calc.
flux_est = np.exp(spind * np.log(f_mean) + c)
#calculate error from covariance matrix
a_mat = np.matrix([np.log(f_mean), 1])
log_flux_err = np.sqrt(a_mat * covar_matrix * a_mat.T)
#to find the error, we take the average log error in linear space
b = np.exp(log_flux_err)
flux_est_err = flux_est / 2. * (b - (1 / b))
flux_est_err = flux_est_err.item(0)
#Do something different if there is only one flux value in archive
elif len(flux_all) == 1:
logger.warning("Only a single flux value available on the archive")
if not np.isnan(spind) and np.isnan(spind_err):
logger.warning(
"Spectral index error not available. Assuming 20% error")
spind_err = spind * 0.2
if np.isnan(spind):
logger.warning(
"Insufficient archival data to estimate spectral index. Using alpha=-1.4 +/- 1.0 as per Bates2013"
)
spind = -1.4
spind_err = 1.
#formula for flux:
#S_1 = nu_1^a * nu_2 ^-a * S_2
nu_1 = f_mean #in Hz
nu_2 = freq_all[0] #in Hz
s_2 = flux_all[0] #in Jy
s_2_err = flux_err_all[0]
a = spind
a_err = spind_err
logger.debug("nu1 {0}".format(nu_1))
logger.debug("nu2 {0}".format(nu_2))
logger.debug("s2 {0}".format(s_2))
logger.info(
"calculating flux using spectral index: {0} and error: {1}".format(
spind, spind_err))
flux_est = nu_1**a * nu_2**(-a) * s_2
#variance formula error est
s_2_var = nu_1**a * nu_2**(-a)
a_var = s_2 * nu_1**a * nu_2**(-a) * (np.log(nu_1) - np.log(nu_2))
a_var = a_var**2 * a_err**2
s_2_var = s_2_var**2 * s_2_err**2
flux_est_err = np.sqrt(s_2_var + a_var)
elif len(flux_all) < 1:
logger.warning(
"No flux values on archive for {0}. Cannot estimate flux. Will return Nones"
.format(pulsar))
return None, None
logger.info("Source flux estimate at {0} MHz: {1} +/- {2} Jy".format(
f_mean / 1e6, flux_est, flux_est_err))
return flux_est, flux_est_err
def find_pulsars_in_fov(obsid, psrbeg, psrend):
"""
Find all pulsars in the field of view and return all the pointings sorted into vdif and normal lists:
-----------
obsid: int
The observation ID
psrbeg: int
The begining of the observation you are processing in GPS time
psrend: int
The end of the observation you are processing in GPS time
Returns:
--------
list of lists:
[pulsar_name_list,
pulsar_pointing_list,
vdif_name_list,
vdif_pointing_list,
sp_name_list,
sp_pointing_list]
"""
#Find all pulsars in beam at at least 0.3 of zenith normlaized power
names_ra_dec = np.array(fpio.grab_source_alog(max_dm=250))
pow_dict, meta_data = find_pulsars_power(obsid,
powers=[0.3, 0.1],
names_ra_dec=names_ra_dec)
channels = meta_data[-1][-1]
obs_psrs = pow_dict[0.3][obsid]
psrs_list_03 = [x[0] for x in obs_psrs]
#Include all bright pulsars in beam at at least 0.1 of zenith normalized power
psrs_01 = [x[0] for x in pow_dict[0.1][obsid]]
sn_dict_01 = snfe.multi_psr_snfe(psrs_01,
obsid,
beg=psrbeg,
end=psrend,
min_z_power=0.1)
for psr in pow_dict[0.1][obsid]:
if psr[0] not in psrs_list_03:
sn, sn_err = sn_dict_01[psr[0]]
if sn is not None and sn_err is not None:
if sn - sn_err >= 10.:
obs_psrs.append(psr)
#get all the pulsars periods
pulsar_list = []
for o in obs_psrs:
pulsar_list.append(o[0])
periods = psrqpy.QueryATNF(params=["P0"],
psrs=pulsar_list,
loadfromdb=ATNF_LOC).pandas["P0"]
oap = get_obs_array_phase(obsid)
centrefreq = 1.28 * float(min(channels) + max(channels)) / 2.
fwhm = calc_ta_fwhm(centrefreq, array_phase=oap)
# Sort all the sources into 3 categories, pulsars which is for slow pulsars, vdif
# for fast pulsars that require vdif and sp for singple pulse searches (FRBs,
# RRATs and pulsars without ATNF periods)
pulsar_pointing_list = []
pulsar_name_list = []
vdif_pointing_list = []
vdif_name_list = []
pulsar_search_pointing_list = []
pulsar_search_name_list = []
sp_pointing_list = []
sp_name_list = []
for pi, pulsar_line in enumerate(obs_psrs):
vdif_check = False
sp_check = False
PSRJ = pulsar_line[0]
#See if pulsar is in beam for times
#enter, exit = snfe.pulsar_beam_coverage(obsid, PSRJ, beg=psrbeg, end=psrend)
#if enter is None or exit is None:
# print("{0} is not in beam for time range. Not beamforming".format(PSRJ))
# continue
#TODO: uncomment this section when sn_flux_est is pulled to vcstools master
if not (len(PSRJ) < 11 or PSRJ[-1] == 'A' or PSRJ[-2:] == 'aa'):
continue
for line in names_ra_dec:
if PSRJ == line[0]:
temp = [line]
#temp = fpio.get_psrcat_ra_dec(pulsar_list=[PSRJ])
temp = fpio.format_ra_dec(temp, ra_col=1, dec_col=2)
jname, raj, decj = temp[0]
#get pulsar period
period = periods[pi]
if math.isnan(period):
print(
"WARNING: Period not found in ephermeris for {0} so assuming "
"it's an RRAT".format(jname))
sp_check = True
period = 0.
elif float(period) < .05:
vdif_check = True
period = float(period) * 1000.
print(
"{0:12} RA: {1} Dec: {2} Period: {3:8.2f} (ms) Begin {4} End {5}".
format(PSRJ, raj, decj, period, psrbeg, psrend))
jname_temp_list = []
if PSRJ[-1] == 'A' or PSRJ[-2:] == 'aa':
#Got to find all the pulsar J names with other letters
vdif_check = True
for pulsar_l in obs_psrs:
pulsar_name = pulsar_l[0]
if pulsar_name.startswith(PSRJ[:-2]):
jname_temp_list.append(pulsar_name)
else:
jname_temp_list.append(jname)
# grid the pointings to fill 2 arcminute raduis to account for ionosphere shift
pointing_list_list = get_pointings_required(raj, decj, fwhm, 2. / 60.)
# sort the pointings into the right groups
for prd in pointing_list_list:
if vdif_check:
vdif_name_list.append(jname_temp_list)
vdif_pointing_list.append("{0}_{1}".format(prd[0], prd[1]))
elif sp_check:
sp_name_list.append(jname_temp_list)
sp_pointing_list.append("{0}_{1}".format(prd[0], prd[1]))
else:
pulsar_name_list.append(jname_temp_list)
pulsar_pointing_list.append("{0}_{1}".format(prd[0], prd[1]))
#Get the rest of the singple pulse search canidates
#-----------------------------------------------------------------------------------------------------------
temp = get_sources_in_fov(obsid, 'RRATs', fwhm)
sp_name_list = sp_name_list + temp[0]
sp_pointing_list = sp_pointing_list + temp[1]
# Find all of the FRB candidates
#-----------------------------------------------------------------------------------------------------------
temp = get_sources_in_fov(obsid, 'FRB', fwhm)
sp_name_list = sp_name_list + temp[0]
sp_pointing_list = sp_pointing_list + temp[1]
# Find all of the Fermi candidates
#-----------------------------------------------------------------------------------------------------------
fermi_list = get_sources_in_fov(obsid, 'Fermi', fwhm)
pulsar_search_name_list = pulsar_search_name_list + fermi_list[0]
pulsar_search_pointing_list = pulsar_search_pointing_list + fermi_list[1]
# Find all of the points of interest candidates
#-----------------------------------------------------------------------------------------------
poi_list = get_sources_in_fov(obsid, 'POI', fwhm)
pulsar_search_name_list = pulsar_search_name_list + poi_list[0]
pulsar_search_pointing_list = pulsar_search_pointing_list + poi_list[1]
# Sometimes we get redundant RRATs that are found in RRAT and ANTF catalogues so they need to be removed
sp_name_list = list(dict.fromkeys([";".join(s) for s in sp_name_list]))
sp_pointing_list = list(dict.fromkeys(sp_pointing_list))
# Changing the format of the names list to make it easier to format
return [[";".join(s) for s in pulsar_name_list], pulsar_pointing_list,
[";".join(s) for s in vdif_name_list], vdif_pointing_list,
[";".join(s) for s in pulsar_search_name_list],
pulsar_search_pointing_list, sp_name_list, sp_pointing_list]
def find_pulsar_w50(pulsar, query=None):
"""
Attempts to find a pulsar's W50 from the ATNF catalogue. If unavailable, will estimate
Paramters:
---------
pulsar: string
The J-name of the pulsar. e.g. 'J2241-5236'
query: object
OPTIONAL - The query for 'pulsar' returned from psrqpy.QueryATNF
Returns:
--------
w50: float
The pulsar's W50
w50_err: float
The W50's uncertainty
"""
if query is None:
#returns W_50 and error for a pulsar from the ATNF archive IN SECONDS
logger.debug("Accessing ATNF database")
query = psrqpy.QueryATNF(psrs=[pulsar], loadfromdb=ATNF_LOC).pandas
W_50 = query["W50"][0]
W_50_err = query["W50_ERR"][0]
if np.isnan(W_50):
logger.warning("W_50 is not on archive")
W_50 = np.nan
W_50_err = np.nan
else:
#convert to seconds
W_50 = W_50 / 1000.
if np.isnan(W_50_err) and not np.isnan(W_50):
logger.warning(
"W_50 error not on archive for {0}. returning standard 5% error".
format(pulsar))
W_50_err = W_50 * 0.05
else:
#convert to seconds
W_50_err = W_50_err / 1000.
if np.isnan(W_50):
logger.warning(
"Applying estimated W_50 for {0}. Uncertainty will be inflated".
format(pulsar))
#Rankin1993 - W = x*P^0.5 where x=4.8+/-0.5 degrees of rotation at 1GHz
#We will nflate this error due to differing frequencies and pulsar behaviour. W_50_err=1. degrees
coeff = 4.8
coeff_err = 2.
period = float(query["P0"][0])
#This estimation is worse for msps, add extra uncetainty if period < 50ms
if period < 0.05:
coeff_err = 4.
#calculate
W_50 = coeff * period**0.5
W_50_err = coeff_err * period**0.5 #error from variance calculation
#W_50 is now in degrees of rotation phase. Convert to seconds
W_50 = (W_50 / 360.) * period
W_50_err = (W_50_err / 360.) * period
if np.isnan(W_50):
W_50 = None
if np.isnan(W_50_err):
W_50_err = None
return W_50, W_50_err
