def test_get_obs_array_phase():
"""Test FWHM calculation."""
for obsid, expect_ans in [(1117101752, 'P1'),
(1252177744, 'P2C'),
(1247832024, 'P2E'),
(1188439520, 'OTH')]:
ans = get_obs_array_phase(obsid)
if ans != expect_ans:
raise AssertionError()
def find_pulsars_in_fov(obsid,
psrbeg,
psrend,
fwhm=None,
search_radius=0.02,
meta_data=None,
full_meta=None,
no_known_pulsars=False,
no_search_cands=False):
"""
Find all pulsars in the field of view and return all the pointings sorted into vdif and normal lists:
Parameters:
-----------
obsid: int
The observation ID
psrbeg: int
The begining of the observation you are processing in GPS time
psrend: int
The end of the observation you are processing in GPS time
fwhm: float
OPTIONAL - The FWHM of the beam in degrees.
Default None: Value will be estimated
search_radius: float
OPTIONAL - The radius to search (create beams within) in degrees to account for ionosphere.
Default: 0.02 degrees
meta_data: list
OPTIONAL - Comon observation metadata in the format from vcstools.metadb_utils.get_common_obs_metadata
Default None: Will perform the metadata call
full_meta: dict
OPTIONAL - Full observation metadata in the format from vcstools.metadb_utils.getmeta
Default None: Will perform the metadata call
no_known_pulsars: bool
OPTIONAL - Will return no known pulsars
Default: False
no_search_cands: bool
OPTIONAL - Will return no search candidates
Default: False
Returns:
--------
list of lists:
[pulsar_name_list,
pulsar_pointing_list,
vdif_name_list,
vdif_pointing_list,
sp_name_list,
sp_pointing_list]
"""
if not meta_data or not full_meta:
meta_data, full_meta = get_common_obs_metadata(obsid, return_all=True)
channels = meta_data[-1]
if fwhm is None:
# Estimate FWHM
oap = get_obs_array_phase(obsid)
centrefreq = 1.28 * float(min(channels) + max(channels)) / 2.
fwhm = calc_ta_fwhm(centrefreq, array_phase=oap)
# Find all pulsars in beam at at least 0.3 and 0.1 of zenith normlaized power
names_ra_dec = np.array(grab_source_alog(max_dm=250))
pow_dict, _ = find_pulsars_power(obsid,
powers=[0.3, 0.1],
names_ra_dec=names_ra_dec)
obs_psrs = pow_dict[0.3][obsid]
# Find pulsars with power between 0.3 and 0.1 and calculate their SN
psrs_list_03 = [x[0] for x in obs_psrs]
psrs_list_01 = [x[0] for x in pow_dict[0.1][obsid]]
psrs_03_01 = psrs_list_01
for psr in psrs_list_03:
if psr in psrs_list_01:
psrs_03_01.remove(psr)
sn_dict_01 = snfu.multi_psr_snfe(psrs_03_01,
obsid,
beg=psrbeg,
end=psrend,
min_z_power=0.1,
obs_metadata=meta_data,
full_meta=full_meta)
# Include all bright pulsars in beam at at least 0.1 of zenith normalized power
for psr in psrs_03_01:
sn, sn_err, _, _ = sn_dict_01[psr]
if sn is not None and sn_err is not None:
if sn - sn_err >= 10.:
for psr_list in pow_dict[0.1][obsid]:
if psr in psr_list:
obs_psrs.append(psr_list)
#get all the pulsars periods
pulsar_list = []
for o in obs_psrs:
pulsar_list.append(o[0])
period_query = psrqpy.QueryATNF(params=["PSRJ", "P0"],
psrs=pulsar_list,
loadfromdb=data_load.ATNF_LOC).pandas
# Sort all the sources into 3 categories, pulsars which is for slow pulsars, vdif
# for fast pulsars that require vdif and sp for singple pulse searches (FRBs,
# RRATs and pulsars without ATNF periods)
pulsar_pointing_list = []
pulsar_name_list = []
vdif_pointing_list = []
vdif_name_list = []
pulsar_search_pointing_list = []
pulsar_search_name_list = []
sp_pointing_list = []
sp_name_list = []
for pi, pulsar_line in enumerate(obs_psrs):
vdif_check = False
sp_check = False
PSRJ = pulsar_line[0]
if not (len(PSRJ) < 11 or PSRJ[-1] == 'A' or PSRJ[-2:] == 'aa'):
continue
for line in names_ra_dec:
if PSRJ == line[0]:
temp = [line]
temp = format_ra_dec(temp, ra_col=1, dec_col=2)
jname, raj, decj = temp[0]
#get pulsar period
period = period_query[period_query['PSRJ'] ==
PSRJ].reset_index()["P0"][0]
if math.isnan(period):
logger.warn("Period not found in ephermeris for {0} so assuming "
"it's an RRAT".format(jname))
sp_check = True
period = 0.
elif float(period) < .05:
vdif_check = True
period = float(period) * 1000.
logger.debug(
"{0:12} RA: {1} Dec: {2} Period: {3:8.2f} (ms) Begin {4} End {5}".
format(PSRJ, raj, decj, period, psrbeg, psrend))
jname_temp_list = []
if PSRJ[-1] == 'A' or PSRJ[-2:] == 'aa':
#Got to find all the pulsar J names with other letters
vdif_check = True
for pulsar_l in obs_psrs:
pulsar_name = pulsar_l[0]
if pulsar_name.startswith(PSRJ[:-2]):
jname_temp_list.append(pulsar_name)
else:
jname_temp_list.append(jname)
# grid the pointings to fill 2 arcminute raduis to account for ionosphere shift
pointing_list_list = get_pointings_required(raj, decj, fwhm,
search_radius)
# sort the pointings into the right groups
for prd in pointing_list_list:
if vdif_check:
vdif_name_list.append(jname_temp_list)
vdif_pointing_list.append("{0}_{1}".format(prd[0], prd[1]))
elif sp_check:
sp_name_list.append(jname_temp_list)
sp_pointing_list.append("{0}_{1}".format(prd[0], prd[1]))
else:
pulsar_name_list.append(jname_temp_list)
pulsar_pointing_list.append("{0}_{1}".format(prd[0], prd[1]))
#Get the rest of the singple pulse search canidates
#-----------------------------------------------------------------------------------------------------------
temp = get_sources_in_fov(obsid, 'RRATs', fwhm)
sp_name_list = sp_name_list + temp[0]
sp_pointing_list = sp_pointing_list + temp[1]
# Find all of the FRB candidates
#-----------------------------------------------------------------------------------------------------------
temp = get_sources_in_fov(obsid, 'FRB', fwhm)
sp_name_list = sp_name_list + temp[0]
sp_pointing_list = sp_pointing_list + temp[1]
# Find all of the Fermi candidates
#-----------------------------------------------------------------------------------------------------------
fermi_list = get_sources_in_fov(obsid, 'Fermi', fwhm)
logger.info(f"{obsid} Fermi candidates: {fermi_list}")
pulsar_search_name_list = pulsar_search_name_list + fermi_list[0]
pulsar_search_pointing_list = pulsar_search_pointing_list + fermi_list[1]
# Find all of the points of interest candidates
#-----------------------------------------------------------------------------------------------
poi_list = get_sources_in_fov(obsid, 'POI', fwhm)
logger.info(f"{obsid} Points of interest: {poi_list}")
pulsar_search_name_list = pulsar_search_name_list + poi_list[0]
pulsar_search_pointing_list = pulsar_search_pointing_list + poi_list[1]
# Sometimes we get redundant RRATs that are found in RRAT and ANTF catalogues so they need to be removed
sp_name_list = list(dict.fromkeys([":".join(s) for s in sp_name_list]))
sp_pointing_list = list(dict.fromkeys(sp_pointing_list))
# Changing the format of the names list to make it easier to format
pulsar_name_list = [":".join(s) for s in pulsar_name_list]
vdif_name_list = [":".join(s) for s in vdif_name_list]
pulsar_search_name_list = [":".join(s) for s in pulsar_search_name_list]
if no_known_pulsars:
# Return empty list for all known pulsar categories
pulsar_name_list = []
pulsar_pointing_list = []
vdif_name_list = []
vdif_pointing_list = []
if no_search_cands:
# Return empty list for all search candidate categories
pulsar_search_name_list = []
pulsar_search_pointing_list = []
sp_name_list = []
sp_pointing_list = []
return [
pulsar_name_list, pulsar_pointing_list, vdif_name_list,
vdif_pointing_list, pulsar_search_name_list,
pulsar_search_pointing_list, sp_name_list, sp_pointing_list
]
print(parser.print_help())
sys.exit(1)
# Handle defaults
if args.out_name is None:
if args.multibeam_file:
args.out_name = args.multibeam_file.split(".")[0]
if args.bestprof_files:
args.out_name = args.bestprof_files[0].split(".")[0]
# Work out max baseline (diameter) in meters
if args.telescope == "ASKAP":
max_baseline = 12.
elif args.telescope == "MWA":
from vcstools.metadb_utils import get_obs_array_phase
obsid = args.out_name = args.bestprof_files[0].split("_")[0]
array_phase = get_obs_array_phase(obsid)
if array_phase == 'P1':
# True max_baseline is 2800 but due to the minimal amount of long baselines
# the following is more realisitic
max_baseline = 2000.
if array_phase == 'P2C':
# True max_baseline is 700.
max_baseline = 327.
elif array_phase == 'P2E':
max_baseline = 4818.
# Parse input files
if args.multibeam_file:
# Load multi_beam_global file
col_name = ['beam', 'xpos', 'ypos', 'flux', 'ferr', 'freq', 'sefd']
dtype = {'beam': 'string', 'xpos': 'f8', 'ypos': 'f8', 'flux': 'f4',
def launch_pabeam_sim(obsid,
pointing,
begin,
duration,
source_name="noname",
metafits_file=None,
flagged_tiles=None,
delays=None,
efficiency=1,
vcstools_version='master',
args=None,
common_metadata=None,
output_dir=None):
"""Submit a job to run the pabeam code to estimate the system equivelent
flux density and a dependent job to resume the submit_to_databse.py code if `args` is given.
Parameters
----------
obsid : `int`
The MWA observation ID.
pointing : `str`
The pointing of the simulation in the format HH:MM:SS.SS_DD:MM:SS.SS.
begin : `int`
The begining of the simulation in GPS time.
duration : `int`
The duration of the simulation in seconds (used to calculate the end of the simulation).
source_name : `str`, optional
The name of the source to be used to label output files. |br| Default: "noname".
metafits_file : `str`, optional
The location of the metafits file. If none given will assume the default location.
flagged_tiles : `list`, optional
A list of the flagged tiles. If none given will assume no tiles were flagged.
efficiency : `float`, optional
Frequency and pointing dependent array efficiency. |br| Default: 1.
vcstools_version : `str`, optional
VCSTools version to load in the job.
args : `dict`, optional
The argument parse dictionary from submit_to_database.py.
If supplied will launch a dependedn job with submit_to_databse.py to complete the script.
common_metadata : `list`, optional
The list of common metadata generated from :py:meth:`vcstools.metadb_utils.get_common_obs_metadata`.
output_dir : `str`
The output directory of the simulation results. By default will put it in the VCS directory under <obsid>/sefd_simulations.
Examples
--------
A simple example:
>>>launch_pabeam_sim(1206977296, "12:49:12_+27:12:00", 1206977300, 600, source_name="SEFD_test", output_dir=".")
"""
# Load computer dependant config file
comp_config = load_config_file()
# Ensure metafits file is there
data_dir = "{}{}".format(comp_config['base_data_dir'], obsid)
ensure_metafits(data_dir, obsid, "{0}_metafits_ppds.fits".format(obsid))
# Perform metadata calls
if common_metadata is None:
common_metadata = get_common_obs_metadata(obsid)
# Get frequencies
centre_freq = common_metadata[5] * 10e5
low_freq = common_metadata[6][0] * 1.28 * 10e5
high_freq = common_metadata[6][-1] * 1.28 * 10e5
sim_freqs = [str(low_freq), str(centre_freq), str(high_freq)]
# Calculate required pixel res and cores/mem
array_phase = get_obs_array_phase(obsid)
fwhm = calc_ta_fwhm(high_freq / 10e5, array_phase=array_phase) #degrees
phi_res = theta_res = fwhm / 3
if phi_res < 0.015:
# Going any smaller causes memory errors
phi_res = theta_res = 0.015
npixels = 360. // phi_res + 90. // theta_res
cores_required = npixels * len(sim_freqs) // 600
nodes_required = cores_required // 24 + 1
# Make directories
batch_dir = "{}/batch".format(data_dir)
if output_dir is None:
sefd_dir = "{}/sefd_simulations".format(data_dir)
else:
sefd_dir = output_dir
if not os.path.exists(batch_dir):
mdir(batch_dir, "Batch", gid=comp_config['gid'])
if not os.path.exists(sefd_dir):
mdir(sefd_dir, "SEFD", gid=comp_config['gid'])
# Parse defaults
if metafits_file is None:
metafits_file = "{0}{1}/{1}_metafits_ppds.fits".format(
comp_config['base_data_dir'], obsid)
# Get delays if none given
if delays is None:
delays = get_common_obs_metadata(obsid)[4][0]
print(delays)
print(' '.join(np.array(delays, dtype=str)))
# Set up pabeam command
command = 'srun --export=all -u -n {} pabeam.py'.format(
int(nodes_required * 24))
command += ' -o {}'.format(obsid)
command += ' -b {}'.format(begin)
command += ' -d {}'.format(int(duration))
command += ' -s {}'.format(
int(duration // 4 - 1)) # force 4 time steps to get reasonable std
command += ' -e {}'.format(efficiency)
command += ' --metafits {}'.format(metafits_file)
command += ' -p {}'.format(pointing)
command += ' --grid_res {:.3f} {:.3f}'.format(theta_res, phi_res)
command += ' --delays {}'.format(' '.join(np.array(delays, dtype=str)))
command += ' --out_dir {}'.format(sefd_dir)
command += ' --out_name {}'.format(source_name)
command += ' --freq {}'.format(" ".join(sim_freqs))
if flagged_tiles is not None:
logger.debug("flagged_tiles: {}".format(flagged_tiles))
command += ' --flagged_tiles {}'.format(' '.join(flagged_tiles))
# Set up and launch job
batch_file_name = 'pabeam_{}_{}_{}'.format(obsid, source_name, pointing)
job_id = submit_slurm(batch_file_name, [command],
batch_dir=batch_dir,
slurm_kwargs={
"time": datetime.timedelta(seconds=10 * 60 * 60),
"nodes": int(nodes_required)
},
module_list=['hyperbeam-python'],
queue='cpuq',
cpu_threads=24,
mem=12288,
vcstools_version=vcstools_version)
if args:
# Set up dependant submit_to_database.py job
submit_args = vars(args)
# Add sefd_file argument
submit_args['sefd_file'] = "{}/{}*stats".format(sefd_dir, source_name)
command_str = "submit_to_database.py"
for key, val in submit_args.items():
if val:
if val == True:
command_str += " --{}".format(key)
else:
command_str += " --{} {}".format(key, val)
batch_file_name = 'submit_to_database_{}_{}_{}'.format(
obsid, source_name, pointing)
job_id_dependant = submit_slurm(
batch_file_name, [command_str],
batch_dir=batch_dir,
slurm_kwargs={"time": datetime.timedelta(seconds=1 * 60 * 60)},
queue='cpuq',
vcstools_version=vcstools_version,
depend=[job_id])
return job_id, job_id_dependant
help=
'Manualy give the declination FWHM in degrees instead of it estimating it from the array phase and frequency'
)
args = parser.parse_args()
# Set up plots
fig = plt.figure()
plt.rc("font", size=8)
fig.add_subplot(111)
ax = plt.axes()
ax.axis('equal')
# Get the fwhm of the observation
meta_data = get_common_obs_metadata(args.obsid)
channels = meta_data[-1]
oap = get_obs_array_phase(args.obsid)
if oap == "OTH":
#Assume it's phase 2 extended array
oap = "P2E"
centrefreq = 1.28 * float(min(channels) + max(channels)) / 2.
fwhm = calc_ta_fwhm(centrefreq, array_phase=oap)
print("Observation ID: {}".format(args.obsid))
print("FWHM: {} deg".format(fwhm))
detections = []
if args.bestprof_dir:
for bestprof_file in glob.glob("{}/*bestprof".format(
args.bestprof_dir)):
with open(bestprof_file, "r") as bestprof:
lines = bestprof.readlines()
ra, dec = lines[0].split("=")[-1].split("_")[1:3]