def upload_file_to_db(obsid, pulsar, filepath, filetype, subbands=None, coh=True):
"""
Uploads a file to the pulsar database
Parameters
----------
obsid : `str`
The observation ID
pulsar : `str`
The name of the pulsar
filepath : `str`
The file path of the file to upload
filetype : `int`
The type of file to upload: 1: Archive, 2: Timeseries, 3: Diagnostics, 4: Calibration Solution, 5: Bestprof
subbands : `int`
OPTINOAL - The number of frequency subbands in an observation. If None, will calculate it.
coh : `boolean`
OPTINOAL - Whether this is a coherent detection or not. Default: True
"""
if not subbands:
subbands = get_subbands(get_common_obs_metadata(obsid))
web_address, auth = get_db_auth_addr()
client.detection_file_upload(web_address, auth,
observationid = str(obsid),
pulsar = pulsar,
filetype = int(filetype),
coherent = coh,
subband = subbands,
filepath = filepath)
def field_of_view(obsid, common_metadata=None, dur=None):
"""Will find the field-of-view of the observation (including the drift) in square degrees.
Parameters
----------
obsid : `int`
The MWA observation ID.
common_metadata : `list`, optional
The list of common metadata generated from :py:meth:`vcstools.metadb_utils.get_common_obs_metadata`
dur : `int`, optional
Duration of observation to calculate for in seconds.
By default will use the entire observation duration.
Returns
-------
area : `float`
The field-of-view of the observation in square degrees.
"""
if common_metadata is None:
common_metadata = get_common_obs_metadata(obsid)
if dur is None:
dt = 296
else:
dt = 100
# Change the dur to the inpur dur
obsid, ra, dec, _, delays, centrefreq, channels = common_metadata
common_metadata = [obsid, ra, dec, dur, delays, centrefreq, channels]
# Make a pixel for each degree on the sky
names_ra_dec = []
for ra in range(0, 360):
for dec in range(-90, 90):
names_ra_dec.append(["sky_pos", ra + 0.5, dec + 0.5])
# Get tile beam power for all pixels
sky_powers = get_beam_power_over_time(names_ra_dec,
common_metadata=common_metadata,
degrees=True,
dt=dt)
# Find the maximum power over all time
max_sky_powers = []
for pixel_power in sky_powers:
temp_power = 0.
for time_power in pixel_power:
if time_power[0] > temp_power:
temp_power = time_power[0]
max_sky_powers.append(temp_power)
# Find all pixels greater than the half power point and sum their area
half_power_point = max(max_sky_powers) / 2
i = 0
area_sum = 0
for ra in range(0, 360):
for dec in range(-90, 90):
if max_sky_powers[i] > half_power_point:
area_sum = area_sum + pixel_area(ra, ra + 1, dec, dec + 1)
i = i + 1
return area_sum
def get_bestprof_data(bestprof_files):
from vcstools.metadb_utils import get_common_obs_metadata
col_name = ['beam', 'xpos', 'ypos', 'flux', 'ferr', 'freq', 'sefd']
#dtype = ['string', 'f8', 'f8', 'f4', 'f4', 'f4', 'f4']
dtype = {'beam': 'string', 'xpos': 'f8', 'ypos': 'f8', 'flux': 'f4',
'ferr': 'f4', 'freq': 'f4', 'sefd': 'f4'}
#beam_data = pd.DataFrame([], columns=col_name, dtype=dtype)
beam_data = pd.DataFrame([], columns=col_name)
obsid = bestprof_files[0].split("_")[0]
freq = get_common_obs_metadata(obsid)[5]/1000 #GHz
sefd = 1 # This is wrong but doesn't make a difference
for i, bestprof_name in enumerate(bestprof_files):
# File name should be in the format obsid_ra_dec*.bestprof
raj, decj = bestprof_name.split("_")[1:3]
c = SkyCoord( raj, decj, frame='icrs', unit=(u.hourangle,u.deg))
rad = c.ra.deg
decd = c.dec.deg
# Get bestprof data from file
with open(bestprof_name,"r") as bestprof:
lines = bestprof.readlines()
sigma = float(lines[13].split("~")[-1].split(" sigma")[0])
# Assume 10% uncertainty
# TODO make this more robust
sigma_err = sigma * 0.3
beam_data = beam_data.append(pd.DataFrame([["{:02d}".format(i), rad, decd, sigma, sigma_err, freq, sefd]], columns=col_name), ignore_index=True)
for c_dtype in dtype.keys():
if dtype[c_dtype] == 'string':
beam_data[c_dtype] = beam_data[c_dtype].astype(str)
else:
beam_data[c_dtype] = pd.to_numeric(beam_data[c_dtype])#, downcast=dtype[c_dtype])
return beam_data
def find_beg_end(obsid, base_path=comp_config["base_data_dir"]):
"""
looks through the comined files of the obsid to find the beginning and end gps times
Parameters:
-----------
obsid: int
The observation ID
base_path: string
OPTIONAL - The system's base working pat
Returns:
--------
beg: int
The beginning time for on-disk files
end: int
The end time for on-disk files
"""
#TODO have some sort of check to look for gaps
if glob.glob("{0}/{1}/combined/{1}*_ics.dat".format(base_path, obsid)):
combined_files = glob.glob("{0}/{1}/combined/{1}*_ics.dat".format(
base_path, obsid))
else:
meta_data = get_common_obs_metadata(obsid)
channels = meta_data[-1]
combined_files = glob.glob("{0}/{1}/combined/{1}*_ch{2}.dat".\
format(base_path, obsid, channels[-1]))
comb_times = []
for comb in combined_files:
comb_times.append(int(comb.split("_")[1]))
beg = min(comb_times)
end = max(comb_times)
return beg, end
def test_get_common_obs_metadata():
"""Test the get_common_obs_metadata function."""
expect = [1117101752, 141.662872328864, 10.6540169083522, 4800,
[[21, 19, 17, 15, 16, 14, 12, 10, 11, 9, 7, 5, 6, 4, 2, 0],
[21, 19, 17, 15, 16, 14, 12, 10, 11, 9, 7, 5, 6, 4, 2, 0]],
184.96, [133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144,
145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156]]
ans = get_common_obs_metadata(1117101752)
if ans != expect:
raise AssertionError
def create_cfgs_main(kwargs, psrs_pointing_dict):
"""
uses kwargs from observation_processing_pipeline.py
"""
metadata, full_meta = get_common_obs_metadata(kwargs["obsid"],
return_all=True)
query = psrqpy.QueryATNF(loadfromdb=data_load.ATNF_LOC).pandas
cfgs = []
# Make the fold times dictionary (it's done for all pulsars simultaneously for speed)
psr_list = list(psrs_pointing_dict.keys())
fold_times_dict = find_fold_times(psr_list,
kwargs["obsid"],
kwargs["beg"],
kwargs["end"],
metadata=metadata,
full_meta=full_meta,
query=query)
for psr in progress_bar(psr_list, "Initiating pulsar configs: "):
logger.info(psr)
pointing_list = psrs_pointing_dict[psr]
enter = fold_times_dict[psr]["enter"]
leave = fold_times_dict[psr]["leave"]
power = fold_times_dict[psr]["power"]
try:
cfgs.append(
initiate_cfg(kwargs,
psr,
pointing_list,
enter,
leave,
power,
query=query,
metadata=metadata))
except TypeError as e:
logger.info(e)
continue
return cfgs
def initiate_cfg(kwargs,
psr,
pointings,
enter,
leave,
power,
query=None,
metadata=None):
"""
Adds all available keys to the cfg dictionary and figures out some useful constants
Takes kwargs from observation_processing_pipeline
"""
cfg = {
"obs": {},
"source": {},
"completed": {},
"folds": {},
"run_ops": {},
"pol": {},
"files": {}
}
if query is None:
query = psrqpy.QueryATNF(loadfromdb=data_load.ATNF_LOC).pandas
if metadata is None:
metadata = get_common_obs_metadata(kwargs["obsid"])
query_index = list(query["JNAME"]).index(psr)
cfg["obs"]["ra"] = metadata[1]
cfg["obs"]["dec"] = metadata[2]
cfg["obs"]["dur"] = metadata[3]
cfg["obs"]["freq"] = metadata[5]
cfg["obs"]["id"] = kwargs["obsid"]
cfg["obs"]["cal"] = kwargs["calid"]
cfg["obs"]["beg"] = kwargs["beg"]
cfg["obs"]["end"] = kwargs["end"]
cfg["run_ops"]["loglvl"] = kwargs["loglvl"]
cfg["run_ops"]["mwa_search"] = kwargs["mwa_search"]
cfg["run_ops"]["vcstools"] = kwargs["vcstools"]
cfg["run_ops"]["label"] = kwargs["label"]
cfg["run_ops"]["thresh_chi"] = 3.5
cfg["run_ops"]["thresh_sn"] = 8.0
cfg["run_ops"]["good_chi"] = 4.0
cfg["run_ops"]["good_sn"] = 20.0
cfg["run_ops"]["vdif"] = None
cfg["run_ops"]["mask"] = None
cfg["files"]["file_precursor"] = file_precursor(kwargs, psr)
cfg["files"]["psr_dir"] = join(comp_config["base_data_dir"],
str(cfg["obs"]["id"]), "dpp",
cfg["files"]["file_precursor"])
cfg["files"]["batch_dir"] = join(comp_config['base_data_dir'],
cfg["obs"]["id"], "batch")
cfg["files"]["classify_dir"] = join(cfg["files"]["psr_dir"],
"classifier_ppp")
cfg["files"]["my_name"] = join(
cfg["files"]["psr_dir"], f"{cfg['files']['file_precursor']}_cfg.yaml")
cfg["files"]["logfile"] = join(cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}.log")
cfg["files"]["archive"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_archive.ar")
cfg["files"]["archive_ascii"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_archive.txt")
cfg["files"]["gfit_plot"] = join(
cfg["files"]["psr_dir"], f"{cfg['files']['file_precursor']}_gfit.png")
cfg["files"]["converted_fits"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_archive.fits")
# debased fits file needs to be the same as archive except for the extension
arch = cfg["files"]["archive"].split(".ar")[0]
cfg["files"]["debased_fits"] = f"{arch}.debase.gg"
# paswing file needs to be the same as debase except for the .gg extension
debase = cfg["files"]["debased_fits"].split(".gg")[0]
cfg["files"]["paswing"] = f"{debase}.paswing"
cfg["files"]["chigrid_initial_ps"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_chigrid_initial.ps")
cfg["files"]["paswing_initial_ps"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_paswing_initial.ps")
cfg["files"]["RVM_fit_initial"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_RVM_fit_initial.out")
cfg["files"]["chigrid_final_ps"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_chigrid_final.ps")
cfg["files"]["paswing_final_ps"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_paswing_final.ps")
cfg["files"]["RVM_fit_final"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_RVM_fit_final.out")
cfg["files"]["ppol_profile_ps"] = join(
cfg["files"]["psr_dir"],
f"{cfg['files']['file_precursor']}_profile.ps")
cfg["files"]["ppol_polar_profile_ps"] = join(
cfg["files"]["psr_dir"], f"{cfg['files']['file_precursor']}_pol.ps")
cfg["source"]["enter_frac"] = None
cfg["source"]["exit_frac"] = None
cfg["source"]["power"] = None
cfg["source"]["name"] = psr
try:
cfg["source"]["enter_frac"] = float(enter)
cfg["source"]["exit_frac"] = float(leave)
cfg["source"]["power"] = float(power)
except TypeError as _: #If any of these are nones, the pulsar isn't in the beam for given beg, end
raise TypeError(f"{cfg['source']['name']} not in beam for given times")
cfg["source"]["sampling_limit"] = int(
bin_sampling_limit(cfg["source"]["name"], query=query))
cfg["source"]["ATNF_P"] = float(query["P0"][query_index])
cfg["source"]["ATNF_DM"] = float(query["DM"][query_index])
cfg["source"]["my_DM"] = None
cfg["source"]["my_P"] = None
cfg["source"]["my_Pdot"] = None
cfg["source"]["my_bins"] = None
cfg["source"]["my_pointing"] = None
cfg["source"]["my_component"] = None
cfg["source"]["gfit"] = None
cfg["source"]["binary"] = is_binary(cfg["source"]["name"], query=query)
cfg["source"]["seek"] = cfg["source"]["enter_frac"] * (cfg["obs"]["end"] -
cfg["obs"]["beg"])
cfg["source"]["total"] = (cfg["source"]["exit_frac"] -
cfg["source"]["enter_frac"]) * (
cfg["obs"]["end"] - cfg["obs"]["beg"])
if cfg["source"]["binary"]:
cfg["source"]["edited_eph_name"] = join(
cfg["files"]["psr_dir"], f"{cfg['files']['file_precursor']}.eph")
cfg["source"]["edited_eph"] = create_edited_eph(cfg["source"]["name"])
else:
cfg["source"]["edited_eph"] = None
cfg["source"]["edited_eph_name"] = None
init, post = required_bin_folds(cfg["source"]["name"], query=query)
for pointing in pointings:
cfg["folds"] = {pointing: {"init": {}, "post": {}}}
cfg["folds"][pointing]["classifier"] = 0
cfg["folds"][pointing]["dir"] = join(cfg["files"]["psr_dir"], pointing)
for _, i in enumerate(init):
cfg["folds"][pointing]["init"][str(i)] = {}
for _, i in enumerate(post):
cfg["folds"][pointing]["post"][str(i)] = {}
cfg["pol"]["RM"] = None
cfg["pol"]["RM_e"] = None
cfg["pol"]["alpha"] = None
cfg["pol"]["beta"] = None
cfg["pol"]["l0"] = None
cfg["pol"]["pa0"] = None
cfg["pol"]["chi"] = None
cfg["completed"] = {}
cfg["completed"]["init_folds"] = False
cfg["completed"]["classify"] = False
cfg["completed"]["post_folds"] = False
cfg["completed"]["upload"] = False
cfg["completed"]["debase"] = False
cfg["completed"]["RM"] = False
cfg["completed"]["RVM_initial"] = False
cfg["completed"]["RVM_final"] = False
return cfg
def plotSkyMap(obsfile,
targetfile,
oname,
show_psrcat=False,
show_mwa_sky=False,
show_mwa_unique=False):
fig = plt.figure()
ax = fig.add_subplot(111, projection="mollweide")
mwa_dec_lim = 30
mwa_only_dec_lim = -50
if show_mwa_sky:
# Make a patch that is transformable through axis projection
# and highlight the visible sky for the MWA
Path = mpath.Path
ra_start = -np.pi
ra_end = np.pi
dec_start = -np.pi / 2
dec_end = np.radians(mwa_dec_lim)
path_data = [
(Path.MOVETO, (ra_start, dec_start)),
(Path.LINETO, (ra_start, dec_end)),
(Path.LINETO, (ra_end, dec_end)),
(Path.LINETO, (ra_end, dec_start)),
(Path.CLOSEPOLY, (ra_end, dec_start)),
]
codes, verts = zip(*path_data)
path = mpath.Path(verts, codes)
patch = mpatches.PathPatch(path,
lw=0,
ec="lightskyblue",
fc="lightskyblue",
label="All MWA sky")
ax.add_patch(patch)
if show_mwa_unique:
# Make a patch that is transformable through axis projection
# and highlight the part of the sky ONLY visible to the MWA
ra_start = -np.pi
ra_end = np.pi
dec_start = -np.pi / 2
dec_end = np.radians(mwa_only_dec_lim)
path_data = [
(Path.MOVETO, (ra_start, dec_start)),
(Path.LINETO, (ra_start, dec_end)),
(Path.LINETO, (ra_end, dec_end)),
(Path.LINETO, (ra_end, dec_start)),
(Path.CLOSEPOLY, (ra_end, dec_start)),
]
codes, verts = zip(*path_data)
path = mpath.Path(verts, codes)
patch = mpatches.PathPatch(path,
lw=0,
ec="lightgreen",
fc="lightgreen",
label="Unique MWA sky")
ax.add_patch(patch)
if show_psrcat:
# Retrieve the local installed PSRCAT catalogue and plot those pulsar positions
cmd = 'psrcat -o short_csv -nocand -nonumber -c "Jname RAJ DECJ" | sed "2d" > psrcat.csv'
subprocess.call(cmd, shell=True)
psrcat_coords = read_data("psrcat.csv", delim=";")
os.remove("psrcat.csv")
# Create a mask for pulsar in and out of the declination limit of the MWA
maskGood = psrcat_coords.dec.wrap_at(180 * u.deg).deg < mwa_dec_lim
maskBad = psrcat_coords.dec.wrap_at(180 * u.deg).deg >= mwa_dec_lim
psrcat_ra_good = -psrcat_coords.ra.wrap_at(
180 *
u.deg).rad[maskGood] # negative because RA increases to the West
psrcat_dec_good = psrcat_coords.dec.wrap_at(180 * u.deg).rad[maskGood]
psrcat_ra_bad = -psrcat_coords.ra.wrap_at(
180 *
u.deg).rad[maskBad] # negative because RA increases to the West
psrcat_dec_bad = psrcat_coords.dec.wrap_at(180 * u.deg).rad[maskBad]
# Now plot the pulsar locations
ax.scatter(psrcat_ra_good,
psrcat_dec_good,
0.01,
marker="x",
color="0.4",
zorder=1.4)
ax.scatter(psrcat_ra_bad,
psrcat_dec_bad,
0.01,
marker="x",
color="0.8",
zorder=1.4)
# Calculate beam patterns and plot contours
levels = np.arange(0.25, 1., 0.05)
cmap = plt.get_cmap("cubehelix_r")
obsids = read_data(obsfile, coords=False)["OBSID"]
for obsid in obsids:
#print "Accessing database for observation: {0}".format(obsid)
logger.info("Accessing database for observation: {0}".format(obsid))
# TODO: I think this process is now implemented in a function in mwa_metadb_utils, need to double check
beam_meta_data = getmeta(service='obs', params={'obs_id': obsid})
ra = beam_meta_data[u'metadata'][u'ra_pointing']
dec = beam_meta_data[u'metadata'][u'dec_pointing']
duration = beam_meta_data[u'stoptime'] - beam_meta_data[
u'starttime'] #gps time
delays = beam_meta_data[u'rfstreams'][u'0'][u'xdelays']
minfreq = float(min(
beam_meta_data[u'rfstreams'][u"0"][u'frequencies']))
maxfreq = float(max(
beam_meta_data[u'rfstreams'][u"0"][u'frequencies']))
centrefreq = 1.28e6 * (minfreq + (maxfreq - minfreq) / 2) #in MHz
channels = beam_meta_data[u'rfstreams'][u"0"][u'frequencies']
beam_meta_data = get_common_obs_metadata(obsid)
# Create a meshgrid over which to iterate
Dec = []
RA = []
map_dec_range = np.arange(-89, 90, 3)
map_ra_range = np.arange(0, 360, 3)
for i in map_dec_range:
for j in map_ra_range:
Dec.append(i)
RA.append(j)
RADecIndex = np.arange(len(RA))
names_ra_dec = np.column_stack((RADecIndex, RA, Dec))
#print "Creating beam patterns..."
time_intervals = 600 # seconds
powout = get_beam_power_over_time(names_ra_dec,
common_metadata=beam_meta_data,
dt=time_intervals,
degrees=True)
z = []
x = []
y = []
for c in range(len(RA)):
temppower = 0.
for t in range(powout.shape[1]):
power_ra = powout[c, t, 0]
if power_ra > temppower:
temppower = power_ra
z.append(temppower)
if RA[c] > 180:
x.append(-RA[c] / 180. * np.pi + 2 * np.pi)
else:
x.append(-RA[c] / 180. * np.pi)
y.append(Dec[c] / 180. * np.pi)
nx = np.array(x)
ny = np.array(y)
nz = np.array(z)
#print "Plotting beam pattern contours..."
logger.info("Plotting beam pattern contours...")
# Set vmin and vmax to ensure the beam patterns are on the same color scale
# and plot the beam pattern contours on the map
#c = ax.tricontour(wrapped_ra, wrapped_dec, final_pattern_power, levels=levels, cmap=cmap, vmin=levels.min(), vmax=levels.max(), zorder=1.3)
c = ax.tricontour(nx,
ny,
nz,
levels=levels,
cmap=cmap,
vmin=levels.min(),
vmax=levels.max(),
zorder=1.3)
# Make a figure color scale based on the contour sets (which all have the same max/min values
fig.colorbar(c, fraction=0.02, pad=0.03, label="Zenith normalised power")
# Plot the target positions, using Astropy to wrap at correct RA
target_coords = read_data(targetfile)
wrapped_target_ra = -target_coords.ra.wrap_at(
180 * u.deg).rad # negative because RA increases to the West
wrapped_target_dec = target_coords.dec.wrap_at(180 * u.deg).rad
#print "Plotting target source positions..."
logger.info("Plotting target source positions...")
ax.scatter(wrapped_target_ra,
wrapped_target_dec,
10,
marker="x",
color="red",
zorder=1.6,
label="Target sources")
xtick_labels = [
"10h", "8h", "6h", "4h", "2h", "0h", "22h", "20h", "18h", "16h", "14h"
]
ax.set_xticklabels(xtick_labels, color="0.2")
ax.set_xlabel("Right Ascension")
ax.set_ylabel("Declination")
ax.grid(True, color="k", lw=0.5, ls=":")
# Place upper-right corner of legend at specified Axis coordinates
ax.legend(loc="upper right",
bbox_to_anchor=(1.02, 0.08),
numpoints=1,
borderaxespad=0.,
fontsize=6)
plt.savefig(oname, format="eps", bbox_inches="tight")
args.begin = Time(obs_begin, format='isot', scale='utc').gps
if not args.duration:
# Use observation duration time from metafits
args.duration = int(obs_duration)
sim_end = args.begin + args.duration -1
# Make a range of times to test
times_range = np.arange(args.begin, sim_end, args.step)
# Centre the range
times_range = times_range + (sim_end - times_range[-1]) // 2
logger.info("running for {} time steps: {}".format(len(times_range), times_range))
# Convert to astropy format
times = Time(times_range, format='gps')
if args.freq is None:
# By default use the all coarse channel centre frequencies
args.freq = np.array(get_common_obs_metadata(1275751816)[-1])*1.28*1e6
else:
args.freq = np.array(args.freq)
# get the tile locations from the metafits
logger.info("getting tile locations from metafits file")
xpos, ypos, zpos, cable_delays = getTileLocations(args.metafits, flags=flags)
logger.debug("xpos: {}".format(xpos))
logger.debug("ypos: {}".format(ypos))
logger.debug("zpos: {}".format(zpos))
#if args.coplanar:
# zpos = np.zeros_like(xpos)
# small calculations and data gathering from arguments is fine and won't run into trouble by having multiple processes do it simultaneously
ra, dec = args.target.split("_")
def plot_beam_pattern(obsid, obsfreq, obstime, ra, dec, cutoff=0.1):
# extra imports from MWA_Tools to access database and beam models
from mwa_pb import primary_beam as pb
_az = np.linspace(0, 360, 3600)
_za = np.linspace(0, 90, 900)
az, za = np.meshgrid(_az, _za)
## TARGET TRACKING ##
times = Time(obstime, format='gps')
target = SkyCoord(ra, dec, unit=(u.hourangle, u.deg))
location = EarthLocation(lat=-26.7033 * u.deg,
lon=116.671 * u.deg,
height=377.827 * u.m)
altaz = target.transform_to(AltAz(obstime=times, location=location))
targetAZ = altaz.az.deg
targetZA = 90 - altaz.alt.deg
colours = cm.viridis(np.linspace(1, 0, len(targetAZ)))
## MWA BEAM CALCULATIONS ##
delays = get_common_obs_metadata(obsid)[
4] # x-pol and y-pol delays for obsid
_, ptAZ, ptZA = mwa_alt_az_za(obsid)
#ptAZ, _, ptZA = dbq.get_beam_pointing(obsid) # Az and ZA in degrees for obsid
logger.info("obs pointing: ({0}, {1})".format(ptAZ, ptZA))
xp, yp = pb.MWA_Tile_analytic(np.radians(za),
np.radians(az),
obsfreq * 1e6,
delays=delays,
zenithnorm=True) #interp=True)
pattern = np.sqrt((xp + yp) / 2.0).real # sum to get "total intensity"
logger.debug("Pattern: {0}".format(pattern))
pmax = pattern.max()
hpp = 0.5 * pmax # half-power point
logger.info("tile pattern maximum: {0:.3f}".format(pmax))
logger.info("tile pattern half-max: {0:.3f}".format(hpp))
pattern[np.where(pattern < cutoff)] = 0 # ignore everything below cutoff
# figure out the fwhm
fwhm_idx = np.where((pattern > 0.498 * pmax) & (pattern < 0.502 * pmax))
fwhm_az_idx = fwhm_idx[1]
fwhm_za_idx = fwhm_idx[0]
# collapse beam pattern along axes
pattern_ZAcol = pattern.mean(axis=0)
pattern_AZcol = pattern.mean(axis=1)
# figure out beam pattern value at target tracking points
track_lines = []
logger.info("beam power at target track points:")
for ta, tz in zip(targetAZ, targetZA):
xp, yp = pb.MWA_Tile_analytic(np.radians([[tz]]),
np.radians([[ta]]),
obsfreq * 1e6,
delays=delays,
zenithnorm=True) #interp=False
bp = (xp + yp) / 2
track_lines.append(bp[0])
logger.info("({0:.2f},{1:.2f}) = {2:.3f}".format(ta, tz, bp[0][0]))
## PLOTTING ##
fig = plt.figure(figsize=(10, 8))
gs = gridspec.GridSpec(4, 4)
axP = plt.subplot(gs[1:, 0:3])
axAZ = plt.subplot(gs[0, :3])
axZA = plt.subplot(gs[1:, 3])
axtxt = plt.subplot(gs[0, 3])
# info text in right-hand corner axis
axtxt.axis('off')
infostr = """Obs ID: {0}
Frequency: {1:.2f}MHz
Beam Pmax: {2:.3f}
Beam half-Pmax: {3:.3f}
""".format(obsid, obsfreq, pmax, hpp)
axtxt.text(0.01, 0.5, infostr, verticalalignment='center')
logger.debug("az: {0}, za: {1}, pattern: {2}, pmax: {3}".format(
az, za, pattern, pmax))
# plot the actual beam patter over sky
p = axP.contourf(az,
za,
pattern,
100,
cmap=plt.get_cmap('gist_yarg'),
vmax=pmax) # plot beam contours
axP.scatter(_az[fwhm_az_idx], _za[fwhm_za_idx], marker='.', s=1,
color='r') # plot the fwhm border
axP.plot(ptAZ, ptZA, marker="+", ms=8, ls="",
color='C0') # plot the tile beam pointing
for ta, tz, c in zip(targetAZ, targetZA, colours):
axP.plot(ta, tz, marker="x", ms=8, ls="",
color=c) # plot the target track through the beam
axP.set_xlim(0, 360)
axP.set_ylim(0, 90)
axP.set_xticks(np.arange(0, 361, 60))
axP.set_xlabel("Azimuth (deg)")
axP.set_ylabel("Zenith angle (deg)")
# setup and configure colourbar
cbar_ax = fig.add_axes([0.122, -0.01, 0.58, 0.03])
cbar = plt.colorbar(p,
cax=cbar_ax,
orientation='horizontal',
label="Zenith normalised power")
cbar.ax.plot(hpp / pmax, [0.5], 'r.')
for l, c in zip(track_lines, colours):
cbar.ax.plot([l / pmax, l / pmax], [0, 1], color=c, lw=1.5)
# plot collapsed beam patterns:
axAZ.plot(_az, pattern_ZAcol, color='k') # collapsed along ZA
for ta, c in zip(targetAZ, colours): # draw tracking points
axAZ.axvline(ta, color=c)
axAZ.set_xlim(0, 360)
axAZ.set_xticks(np.arange(0, 361, 60))
axAZ.set_xticklabels([])
axAZ.set_yscale('log')
axZA.plot(pattern_AZcol, _za, color='k') # collapsed along AZ
for tz, c in zip(targetZA, colours): # draw tracking points
axZA.axhline(tz, color=c)
axZA.set_ylim(0, 90)
axZA.set_yticklabels([])
axZA.set_xscale('log')
plt.savefig("{0}_{1:.2f}MHz_flattile.png".format(obsid, obsfreq),
bbox_inches='tight')
print("getting obs metadata from obs_meta.csv")
ob, ra, dec, time, delays, centrefreq, channels = omf
ob = int(ob)
time = int(time)
delaysx = list(
map(int, delays[2:-2].split("], [")[0].split(",")))
delaysy = list(
map(int, delays[2:-2].split("], [")[1].split(",")))
delays = [delaysx, delaysx]
centrefreq = float(centrefreq)
channels = list(map(int, channels[1:-1].split(",")))
obs_foun_check = True
if not obs_foun_check:
print("Getting metadata for {}".format(ob))
ob, ra, dec, time, delays,centrefreq, channels =\
get_common_obs_metadata(ob)
with open('obs_meta.csv', 'a') as csvfile:
spamwriter = csv.writer(csvfile)
spamwriter.writerow(
[ob, ra, dec, time, delays, centrefreq, channels])
cord = [ob, ra, dec, time, delays, centrefreq, channels]
ra_list.append(ra)
dec_list.append(dec)
delays_list.append(delays)
#Loop over observations and calc beam power
for i, ob in enumerate(observations):
print("Calculating obs {0}/{1}".format(i + 1, len(observations)))
ra = ra_list[i]
dec = dec_list[i]
def est_pulsar_flux(pulsar, obsid, plot_flux=False, common_metadata=None, query=None):
"""Estimates a pulsar's flux from archival data by assuming a power law relation between flux and frequency
Parameters
----------
pulsar : `str`
The Jname of the pulsar.
obsid : `int`
The MWA observation ID
plot_flux : `boolean`, optional
Whether or not to make a plot of the flux estimation. |br| Default: `False`
common_metadata : `list`, optional
The list of common metadata generated from :py:meth:`vcstools.metadb_utils.get_common_obs_metadata`
query : psrqpy object, optional
A previous psrqpy.QueryATNF query. Can be supplied to prevent performing a new query.
Returns
-------
flux : `float`
The estimated flux in Jy.
flux_err : `float`
The estimated flux's uncertainty in Jy.
"""
if query is None:
query = psrqpy.QueryATNF(psrs=[pulsar], loadfromdb=data_load.ATNF_LOC).pandas
query_id = list(query['PSRJ']).index(pulsar)
if common_metadata is None:
logger.debug("Obtaining mean freq from obs metadata")
common_metadata = get_common_obs_metadata(obsid)
f_mean = common_metadata[5]*1e6
#freq_all, flux_all, flux_err_all, spind, spind_err = flux_from_atnf(pulsar, query=query)
freq_all, flux_all, flux_err_all, ref_all = flux_from_atnf(pulsar, query=query)
# convert to Hz
freq_all = [f*1e6 for f in freq_all]
# convert to Jy
flux_all = [f*1e-3 for f in flux_all]
flux_err_all = [f*1e-3 for f in flux_err_all]
spind = query["SPINDX"][query_id]
spind_err = query["SPINDX_ERR"][query_id]
logger.debug("Freqs: {0}".format(freq_all))
logger.debug("Fluxes: {0}".format(flux_all))
logger.debug("Flux Errors: {0}".format(flux_err_all))
logger.debug("{0} there are {1} flux values available on the ATNF database"\
.format(pulsar, len(flux_all)))
# Check if there is enough data to estimate the flux
if len(flux_all) == 0:
logger.debug("{} no flux values on archive. Cannot estimate flux. Will return Nones".format(pulsar))
return None, None
elif ( len(flux_all) == 1 ) and ( ( not spind ) or ( not spind_err ) ):
logger.debug("{} has only a single flux value and no spectral index. Cannot estimate flux. Will return Nones".format(pulsar))
return None, None
if ( not spind ) or ( not spind_err ):
# If no spind on ATNF fit our own
spind, spind_err, K, covar_mat = find_spind(pulsar, freq_all, flux_all, flux_err_all)
else:
K = covar_mat = None
if K and covar_mat is not None:
# Use the spind power law we fit
flux_est, flux_est_err = flux_from_plaw(f_mean, K, spind, covar_mat)
elif spind and spind_err and flux_all:
# Use ATNF spind
flux_est, flux_est_err = flux_from_spind(f_mean, freq_all[0], flux_all[0], flux_err_all[0],\
spind, spind_err)
logger.debug("Finished estimating flux")
if plot_flux == True:
plot_flux_estimation(pulsar, freq_all, flux_all, flux_err_all, spind,
my_nu=f_mean, my_S=flux_est, my_S_e=flux_est_err, obsid=obsid,
a_err=spind_err, K=K, covar_mat=covar_mat)
return flux_est, flux_est_err
def launch_pabeam_sim(obsid,
pointing,
begin,
duration,
source_name="noname",
metafits_file=None,
flagged_tiles=None,
delays=None,
efficiency=1,
vcstools_version='master',
args=None,
common_metadata=None,
output_dir=None):
"""Submit a job to run the pabeam code to estimate the system equivelent
flux density and a dependent job to resume the submit_to_databse.py code if `args` is given.
Parameters
----------
obsid : `int`
The MWA observation ID.
pointing : `str`
The pointing of the simulation in the format HH:MM:SS.SS_DD:MM:SS.SS.
begin : `int`
The begining of the simulation in GPS time.
duration : `int`
The duration of the simulation in seconds (used to calculate the end of the simulation).
source_name : `str`, optional
The name of the source to be used to label output files. |br| Default: "noname".
metafits_file : `str`, optional
The location of the metafits file. If none given will assume the default location.
flagged_tiles : `list`, optional
A list of the flagged tiles. If none given will assume no tiles were flagged.
efficiency : `float`, optional
Frequency and pointing dependent array efficiency. |br| Default: 1.
vcstools_version : `str`, optional
VCSTools version to load in the job.
args : `dict`, optional
The argument parse dictionary from submit_to_database.py.
If supplied will launch a dependedn job with submit_to_databse.py to complete the script.
common_metadata : `list`, optional
The list of common metadata generated from :py:meth:`vcstools.metadb_utils.get_common_obs_metadata`.
output_dir : `str`
The output directory of the simulation results. By default will put it in the VCS directory under <obsid>/sefd_simulations.
Examples
--------
A simple example:
>>>launch_pabeam_sim(1206977296, "12:49:12_+27:12:00", 1206977300, 600, source_name="SEFD_test", output_dir=".")
"""
# Load computer dependant config file
comp_config = load_config_file()
# Ensure metafits file is there
data_dir = "{}{}".format(comp_config['base_data_dir'], obsid)
ensure_metafits(data_dir, obsid, "{0}_metafits_ppds.fits".format(obsid))
# Perform metadata calls
if common_metadata is None:
common_metadata = get_common_obs_metadata(obsid)
# Get frequencies
centre_freq = common_metadata[5] * 10e5
low_freq = common_metadata[6][0] * 1.28 * 10e5
high_freq = common_metadata[6][-1] * 1.28 * 10e5
sim_freqs = [str(low_freq), str(centre_freq), str(high_freq)]
# Calculate required pixel res and cores/mem
array_phase = get_obs_array_phase(obsid)
fwhm = calc_ta_fwhm(high_freq / 10e5, array_phase=array_phase) #degrees
phi_res = theta_res = fwhm / 3
if phi_res < 0.015:
# Going any smaller causes memory errors
phi_res = theta_res = 0.015
npixels = 360. // phi_res + 90. // theta_res
cores_required = npixels * len(sim_freqs) // 600
nodes_required = cores_required // 24 + 1
# Make directories
batch_dir = "{}/batch".format(data_dir)
if output_dir is None:
sefd_dir = "{}/sefd_simulations".format(data_dir)
else:
sefd_dir = output_dir
if not os.path.exists(batch_dir):
mdir(batch_dir, "Batch", gid=comp_config['gid'])
if not os.path.exists(sefd_dir):
mdir(sefd_dir, "SEFD", gid=comp_config['gid'])
# Parse defaults
if metafits_file is None:
metafits_file = "{0}{1}/{1}_metafits_ppds.fits".format(
comp_config['base_data_dir'], obsid)
# Get delays if none given
if delays is None:
delays = get_common_obs_metadata(obsid)[4][0]
print(delays)
print(' '.join(np.array(delays, dtype=str)))
# Set up pabeam command
command = 'srun --export=all -u -n {} pabeam.py'.format(
int(nodes_required * 24))
command += ' -o {}'.format(obsid)
command += ' -b {}'.format(begin)
command += ' -d {}'.format(int(duration))
command += ' -s {}'.format(
int(duration // 4 - 1)) # force 4 time steps to get reasonable std
command += ' -e {}'.format(efficiency)
command += ' --metafits {}'.format(metafits_file)
command += ' -p {}'.format(pointing)
command += ' --grid_res {:.3f} {:.3f}'.format(theta_res, phi_res)
command += ' --delays {}'.format(' '.join(np.array(delays, dtype=str)))
command += ' --out_dir {}'.format(sefd_dir)
command += ' --out_name {}'.format(source_name)
command += ' --freq {}'.format(" ".join(sim_freqs))
if flagged_tiles is not None:
logger.debug("flagged_tiles: {}".format(flagged_tiles))
command += ' --flagged_tiles {}'.format(' '.join(flagged_tiles))
# Set up and launch job
batch_file_name = 'pabeam_{}_{}_{}'.format(obsid, source_name, pointing)
job_id = submit_slurm(batch_file_name, [command],
batch_dir=batch_dir,
slurm_kwargs={
"time": datetime.timedelta(seconds=10 * 60 * 60),
"nodes": int(nodes_required)
},
module_list=['hyperbeam-python'],
queue='cpuq',
cpu_threads=24,
mem=12288,
vcstools_version=vcstools_version)
if args:
# Set up dependant submit_to_database.py job
submit_args = vars(args)
# Add sefd_file argument
submit_args['sefd_file'] = "{}/{}*stats".format(sefd_dir, source_name)
command_str = "submit_to_database.py"
for key, val in submit_args.items():
if val:
if val == True:
command_str += " --{}".format(key)
else:
command_str += " --{} {}".format(key, val)
batch_file_name = 'submit_to_database_{}_{}_{}'.format(
obsid, source_name, pointing)
job_id_dependant = submit_slurm(
batch_file_name, [command_str],
batch_dir=batch_dir,
slurm_kwargs={"time": datetime.timedelta(seconds=1 * 60 * 60)},
queue='cpuq',
vcstools_version=vcstools_version,
depend=[job_id])
return job_id, job_id_dependant
def find_sources_in_obs(obsid_list,
names_ra_dec,
obs_for_source=False,
dt_input=300,
beam='analytic',
min_z_power=0.3,
cal_check=False,
all_volt=False,
degrees_check=False,
metadata_list=None):
"""Either creates text files for each MWA obs ID of each source within it or a text
file for each source with each MWA obs is that the source is in.
Parameters
----------
obsid_list : `list`
List of MWA observation IDs.
names_ra_dec : `list`
An array in the format [[source_name, RAJ, DecJ]]
obs_for_source : `boolean`, optional
If `True` creates a text file for each source with each MWA observation that the source is in.
If `False` creates text files for each MWA obs ID of each source within it. |br| Default: `False`.
dt_input : `int`, optional
The time interval in seconds of how often powers are calculated. |br| Default: 300.
beam : `str`, optional
The primary beam model to use out of [analytic, advanced, full_EE]. |br| Default: analytic.
min_z_power : `float`, optional
Zenith normalised power cut off. |br| Default: 0.3.
cal_check : `boolean`, optional
Checks the MWA pulsar database if there is a calibration suitable for the observation ID.
all_volt : `boolean`, optional
Included observations with missing or incorrect voltage files. |br| Default: `False`.
degrees_check : `boolean`, optional
If true assumes RAJ and DecJ are in degrees. |br| Default: `False`.
metadata_list : `list`
List of the outputs of vcstools.metadb_utils.get_common_obs_metadata.
If not provided, will make the metadata calls to find the data. |br| Default: `None`.
Returns
-------
output_data : `dict`
The format of output_data is dependant on obs_for_source.
|br| If obs_for_source is `True` :
|br| output_data = {jname:[[obsid, duration, enter, exit, max_power],
|br| [obsid, duration, enter, exit, max_power]]}
|br| If obs_for_source is `False` :
|br| ouput_data = {obsid:[[jname, enter, exit, max_power],
|br| [jname, enter, exit, max_power]]}
obsid_meta : `list`
A list of the output of get_common_obs_metadata for each obsid
"""
import urllib.request
#prepares metadata calls and calculates power
powers = []
#powers[obsid][source][time][freq]
common_metadata_list = []
obsid_to_remove = []
# Loop over observations to check if there are VCS files
for i, obsid in enumerate(obsid_list):
# Perform the file meta data call over 10 only 10 seconds as that is suffient test
try:
files_meta_data = getmeta(service='data_files',
params={
'obs_id': obsid,
'nocache': 1,
'mintime': int(obsid) + 10,
'maxtime': int(obsid) + 20
})
except urllib.error.HTTPError as err:
files_meta_data = None
if files_meta_data is None:
logger.warning(
"No file metadata data found for obsid {}. Skipping".format(
obsid))
obsid_to_remove.append(obsid)
continue
# Check raw voltage files
raw_available = False
raw_deleted = False
for file_name in files_meta_data.keys():
if file_name.endswith('dat'):
deleted = files_meta_data[file_name]['deleted']
if deleted:
raw_deleted = True
else:
raw_available = True
# Check combined voltage tar files
comb_available = False
comb_deleted = False
for file_name in files_meta_data.keys():
if file_name.endswith('tar'):
deleted = files_meta_data[file_name]['deleted']
if deleted:
comb_deleted = True
else:
comb_available = True
if raw_available or comb_available or all_volt:
if metadata_list:
common_metadata, _ = metadata_list[i]
else:
# No metadata supplied so make the metadata call
common_metadata = get_common_obs_metadata(obsid)
common_metadata_list.append(common_metadata)
elif raw_deleted and comb_deleted:
logger.warning(
'Raw and combined voltage files deleted for {}'.format(obsid))
obsid_to_remove.append(obsid)
elif raw_deleted:
logger.warning('Raw voltage files deleted for {}'.format(obsid))
obsid_to_remove.append(obsid)
elif comb_deleted:
logger.warning(
'Combined voltage files deleted for {}'.format(obsid))
obsid_to_remove.append(obsid)
else:
logger.warning(
'No raw or combined voltage files for {}'.format(obsid))
obsid_to_remove.append(obsid)
for otr in obsid_to_remove:
obsid_list.remove(otr)
# Calculate the power for all sources and obsids and find when they enter and exit the beam
beam_coverage = source_beam_coverage(
obsid_list,
names_ra_dec,
common_metadata_list=common_metadata_list,
dt_input=dt_input,
beam=beam,
min_z_power=min_z_power)
#chooses whether to list the source in each obs or the obs for each source
output_data = {}
if obs_for_source:
for source_name in np.array(names_ra_dec)[:, 0]:
source_data = []
for on, obsid in enumerate(obsid_list):
if source_name in beam_coverage[obsid].keys:
# Source was in the beam so include it
_, _, _, duration, _, centre_freq, channels = common_metadata_list[
on]
enter_beam_norm, exit_beam_norm, max_power = beam_coverage[
obsid][source_name]
source_data.append([
obsid, duration, enter_beam_norm, exit_beam_norm,
max_power, centre_freq, bandwidth
])
# For each source make a dictionary key that contains a list of
# lists of the data for each obsid
output_data[source_name] = source_data
else:
#output a list of sorces for each obs
for on, obsid in enumerate(obsid_list):
_, _, _, duration, _, centre_freq, channels = common_metadata_list[
on]
obsid_data = []
for source_name in np.array(names_ra_dec)[:, 0]:
if source_name in beam_coverage[obsid].keys():
enter_beam_norm, exit_beam_norm, max_power = beam_coverage[
obsid][source_name]
obsid_data.append([
source_name, enter_beam_norm, exit_beam_norm, max_power
])
# For each obsid make a dictionary key that contains a list of
# lists of the data for each source/pulsar
output_data[obsid] = obsid_data
return output_data, common_metadata_list
def est_pulsar_sn(pulsar, obsid,
p_ra=None, p_dec=None,
dect_beg=None, dect_end=None,
obs_beg=None, obs_end=None,
common_metadata=None, full_metadata=None,
query=None, plot_flux=False,
min_z_power=0.3, trcvr=data_load.TRCVR_FILE):
"""Estimates the signal to noise ratio for a pulsar in a given observation using the radiometer equation
.. math:: S/N = \frac{s_{mean} gain}{t_{sys}} \sqrt{n_p t_{int} df \frac{period - W_{50}}{W_{50}}
Note that W_50 should be W_equiv but we can't figure that out so we're estimating
Parameters
----------
pulsar : `str`
The Jname of the pulsar.
obsid : `int`
The MWA Observation ID.
p_ra, p_dec : `str`, optional
The target's right ascension and declination in sexidecimals. If not supplied will use the values from the ANTF.
dect_beg, dect_end : `int`, optional
The beg and end GPS time of the detection to calculate over.
If not supplied will estimate beam enter and exit.
obs_beg, obs_end : `int`, optional
Beginning and end GPS time of the observation.
If not supplied will use :py:meth:`vcstools.metadb_utils.obs_max_min` to find it.
common_metadata : `list`, optional
The list of common metadata generated from :py:meth:`vcstools.metadb_utils.get_common_obs_metadata`
full_metadata : `dict`, optional
The dictionary of metadata generated from :py:meth:`vcstools.metadb_utils.getmeta`
query : psrqpy object, optional
A previous psrqpy.QueryATNF query. Can be supplied to prevent performing a new query.
plot_flux : `boolean`
OPTIONAL - whether or not to produce a plot of the flux estimation. |br| Default: `False`.
min_z_power : `float`, optional
Zenith normalised power cut off. |br| Default: 0.3.
trcvr : `str`
The location of the MWA receiver temp csv file. |br| Default: <vcstools_data_dir>/MWA_Trcvr_tile_56.csv
Returns
-------
sn : `float`
The expected signal to noise ratio for the given inputs.
sn_err : `float`
The uncertainty in the signal to noise ratio.
"""
# We will attain uncertainties for s_mean, gain, t_sys and W_50.
# other uncertainties are considered negligible
if query is None:
query = psrqpy.QueryATNF(psrs=pulsar, loadfromdb=data_load.ATNF_LOC).pandas
query_id = list(query['PSRJ']).index(pulsar)
if p_ra is None or p_dec is None:
# Get some basic pulsar and obs info info
p_ra = query["RAJ"][query_id]
p_dec = query["DECJ"][query_id]
# Get metadata if not supplied
if not common_metadata or not full_metadata:
logger.debug("Obtaining obs metadata")
common_metadata, full_metadata = get_common_obs_metadata(obsid, return_all=True, full_metadata=full_metadata)
n_p = 2 #constant
df = 30.72e6 #(24*1.28e6)
# Estimate flux
s_mean, s_mean_err = est_pulsar_flux(pulsar, obsid, plot_flux=plot_flux,
common_metadata=common_metadata, query=query)
# Fluxes may be Nones. If so, return None
if s_mean is None and s_mean_err is None:
return None, None, None, None
if not dect_beg or not dect_end:
# Estimate integration time from when the source enters and exits the beam
dect_beg, dect_end = source_beam_coverage_and_times(obsid, pulsar,
p_ra=p_ra, p_dec=p_dec,
obs_beg=obs_beg, obs_end=obs_end,
min_z_power=min_z_power,
common_metadata=common_metadata,
query=query)[:2]
t_int = dect_end - dect_beg + 1
if t_int<=0.:
logger.warning("{} not in beam for obs files or specificed beginning and end times"\
.format(pulsar))
return None, None, None, None
# Find system temp and gain
t_sys, t_sys_err, gain, gain_err = find_t_sys_gain(pulsar, obsid,
p_ra=p_ra, p_dec=p_dec,
dect_beg=dect_beg, dect_end=dect_end,
obs_beg=obs_beg, obs_end=obs_end,
common_metadata=common_metadata, full_metadata=full_metadata,
trcvr=trcvr, min_z_power=min_z_power, query=query)
#Find W_50
W_50, W_50_err = find_pulsar_w50(pulsar, query=query)
# Calculate SN
period = float(query["P0"][query_id])
SN = ((s_mean * gain)/t_sys) * np.sqrt(n_p * t_int * df * (period - W_50)/W_50)
#Calculate SN uncertainty using variance formula. Assuming error from period, df and t_int is zero
dc_expr = np.sqrt((period-W_50)/W_50)
var_s_mean = (gain * np.sqrt(n_p * t_int * df)) * dc_expr / t_sys
var_gain = s_mean * np.sqrt(n_p * t_int * df) * dc_expr / t_sys
var_W_50 = s_mean * gain * np.sqrt(n_p * t_int * df)/t_sys * (period/(-2.*W_50**2.)) * dc_expr**-1
var_t_sys = -s_mean * gain * np.sqrt(n_p * t_int * df) * dc_expr / t_sys**2.
var_s_mean = var_s_mean**2. * s_mean_err**2.
var_gain = var_gain**2. * gain_err**2.
var_W_50 = var_W_50**2. * W_50_err**2.
var_t_sys = var_t_sys**2. * t_sys_err**2.
logger.debug("variance estimates for s_mean: {0}, gain: {1}, W_50: {2}, t_sys: {3}"\
.format(var_s_mean, var_gain, var_W_50, var_t_sys))
SN_err = np.sqrt(var_s_mean + var_gain + var_W_50 + var_t_sys)
logger.debug("Gain: {0} +/- {1}".format(gain, gain_err))
logger.debug("t_int: {0}".format(t_int))
logger.debug("df: {0}".format(df))
logger.debug("period: {0}".format(period))
logger.debug("W_50: {0} +/- {1}".format(W_50, W_50_err))
logger.debug("t_sys: {0} +/- {1}".format(t_sys, t_sys_err))
return SN, SN_err, s_mean, s_mean_err
def source_beam_coverage_and_times(obsid,
pulsar,
p_ra=None,
p_dec=None,
obs_beg=None,
obs_end=None,
files_beg=None,
files_end=None,
min_z_power=0.3,
dt_input=100,
common_metadata=None,
query=None,
beam='analytic'):
"""Finds the normalised time that a pulsar is in the beam for a given obsid.
If pulsar is not in beam, returns None, None
Parameters
----------
obsid : `int`
The observation ID
pulsar : `str`
The pulsar's J name
p_ra, p_dec : `str`, optional
The target's right ascension and declination in sexidecimals.
If not supplied will use the values from the ANTF.
obs_beg, obs_end : `int`, optional
Beginning and end GPS time of the observation.
If not supplied will use :py:meth:`vcstools.metadb_utils.obs_max_min` to find it.
files_beg, files_end : `int`, optional
Beginning and end GPS time of the (fits of VCS) files.
If not supplied will assume the full observation is available.
min_z_power : `float`, optional
Zenith normalised power cut off. |br| Default: 0.3.
common_metadata : `list`, optional
The list of common metadata generated from :py:meth:`vcstools.metadb_utils.get_common_obs_metadata`
query : psrqpy object, optional
A previous psrqpy query. Can be supplied to prevent performing a new query.
beam : `str`, optional
The primary beam model to use out of [analytic, advanced, full_EE]. |br| Default: analytic.
Returns
-------
enter_files : `float`
A float between 0 and 1 that describes the normalised time that the pulsar enters the beam
exit_files : `float`
A float between 0 and 1 that describes the normalised time that the pulsar exits the beam
"""
# Perform required metadata calls
if query is None:
query = psrqpy.QueryATNF(psrs=pulsar,
loadfromdb=data_load.ATNF_LOC).pandas
if p_ra is None or p_dec is None:
# Get some basic pulsar and obs info info
query_id = list(query['PSRJ']).index(pulsar)
p_ra = query["RAJ"][query_id]
p_dec = query["DECJ"][query_id]
if not common_metadata:
common_metadata = get_common_obs_metadata(obsid)
if obs_beg is None or obs_end is None:
obs_beg, obs_end = obs_max_min(obsid)
obs_dur = obs_end - obs_beg + 1
if not files_beg:
files_beg = obs_beg
if not files_end:
files_end = obs_end
files_dur = files_end - files_beg + 1
beam_coverage = source_beam_coverage(
[obsid], [[pulsar, p_ra, p_dec]],
common_metadata_list=[common_metadata],
dt_input=dt_input,
beam=beam,
min_z_power=min_z_power)
if pulsar not in beam_coverage[obsid].keys():
# Not in beam exiting
return None, None, None, None, None, None, None, None, None
dect_beg_norm, dect_end_norm, _ = beam_coverage[obsid][pulsar]
# GPS times the source enters and exits beam
dect_beg = obs_beg + obs_dur * dect_beg_norm
dect_end = obs_beg + obs_dur * dect_end_norm
# Normalised time the source enters/exits the beam in the files (used for Presto commands)
files_beg_norm = (dect_beg - files_beg) / files_dur
files_end_norm = (dect_end - files_beg) / files_dur
if files_beg_norm > 1. or files_end_norm < 0.:
logger.debug(
"source {0} is not in the beam for the files on disk".format(
pulsar))
files_beg_norm = None
files_end_norm = None
else:
if files_beg_norm < 0.:
files_beg_norm = 0.
if files_end_norm > 1.:
files_end_norm = 1.
return dect_beg, dect_end, dect_beg_norm, dect_end_norm, files_beg_norm, files_end_norm, obs_beg, obs_end, obs_dur
def source_beam_coverage(obs_list,
names_ra_dec,
common_metadata_list=None,
dt_input=300,
beam='analytic',
min_z_power=0.3):
"""For a list of MWA observations and sources will find if the
sources are in the beam and when they enter and exit.
Parameters
----------
obs_list : `list`
A list of MWA Observation IDs.
names_ra_dec : `list`
An array in the format [[source_name, RAJ, DecJ]]
common_metadata_list : `list` of `list`, optional
A list of lists where each list is the common metadata generated from
:py:meth:`vcstools.metadb_utils.get_common_obs_metadata` for each obs in the obs_list.
dt_input : `int`, optional
The time interval in seconds of how often powers are calculated. |br| Default: 300.
beam : `str`, optional
The primary beam model to use out of [analytic, advanced, full_EE, hyperbeam]. |br| Default: analytic.
min_z_power : `float`, optional
Zenith normalised power cut off. |br| Default: 0.3.
Returns
-------
beam_coverage : `dict`
A dictionary where the first key is the observation ID and the second is the pulsar names like so: |br|
beam_coverage[obsid][name] = [dect_beg_norm, dect_end_norm, np.amax(source_ob_power)]
dect_beg_norm : `float`
Fraction of the observation when the source enters the beam.
dect_end_norm : `float`
Fraction of the observation when the source enters and exits the beam respectively.
"""
beam_coverage = {}
for on, obsid in enumerate(obs_list):
beam_coverage[obsid] = {}
if common_metadata_list:
common_metadata = common_metadata_list[on]
else:
# No metadata supplied so make the metadata call
common_metadata = get_common_obs_metadata(obsid)
#common_metadata = obsid,ra_obs,dec_obs,time_obs,delays,centrefreq,channels
obs_dur = common_metadata[3]
if dt_input * 4 > obs_dur:
# If the observation time is very short then a smaller dt time is required
# to get enough points to fit a curve
dt = int(obs_dur / 4.)
else:
dt = dt_input
logger.debug("obsid: {0}, time_obs {1} s, dt {2} s".format(
obsid, obs_dur, dt))
logger.debug("names_ra_dec: {}".format(names_ra_dec))
powers = get_beam_power_over_time(names_ra_dec,
common_metadata=common_metadata,
dt=dt,
centeronly=True,
option=beam)
for source_ob_power, name in zip(powers, np.array(names_ra_dec)[:, 0]):
if max(source_ob_power) > min_z_power:
logger.debug(
"Running beam_enter_exit on obsid: {}".format(obsid))
dect_beg_norm, dect_end_norm = beam_enter_exit(
source_ob_power, obs_dur, dt=dt, min_z_power=min_z_power)
beam_coverage[obsid][name] = [
dect_beg_norm, dect_end_norm,
np.amax(source_ob_power)
]
return beam_coverage
def get_beam_power_over_time(names_ra_dec,
common_metadata=None,
dt=296,
centeronly=True,
verbose=False,
option='analytic',
degrees=False,
start_time=0):
"""Calculates the zenith normalised power for each source over time.
Parameters
----------
names_ra_dec : `list`
An array in the format [[source_name, RAJ, DecJ]]
common_metadata : `list`, optional
The list of common metadata generated from :py:meth:`vcstools.metadb_utils.get_common_obs_metadata`
dt : `int`, optional
The time interval of how often powers are calculated. |br| Default: 296.
centeronly : `boolean`, optional
Only calculates for the centre frequency. |br| Default: `True`.
verbose : `boolean`, optional
If `True` will not supress the output from mwa_pb. |br| Default: `False`.
option : `str`, optional
The primary beam model to use out of [analytic, advanced, full_EE, hyperbeam]. |br| Default: analytic.
degrees : `boolean`, optional
If true assumes RAJ and DecJ are in degrees. |br| Default: `False`.
start_time : `int`, optional
The time in seconds from the begining of the observation to start calculating at. |br| Default: 0.
Returns
-------
Powers : `numpy.array`, (len(names_ra_dec), ntimes, nfreqs)
The zenith normalised power for each source over time.
"""
if common_metadata is None:
common_metadata = get_common_obs_metadata(obsid)
obsid, _, _, time, delays, centrefreq, channels = common_metadata
names_ra_dec = np.array(names_ra_dec)
amps = [1.0] * 16
logger.debug("Calculating beam power for OBS ID: {0}".format(obsid))
if option == 'hyperbeam':
if "mwa_hyperbeam" not in sys.modules:
logger.error(
"mwa_hyperbeam not installed so can not use hyperbeam to create a beam model. Exiting"
)
sys.exit(1)
beam = mwa_hyperbeam.FEEBeam(config.h5file)
# Work out time steps to calculate over
starttimes = np.arange(start_time, time + start_time, dt)
stoptimes = starttimes + dt
stoptimes[stoptimes > time] = time
ntimes = len(starttimes)
midtimes = float(obsid) + 0.5 * (starttimes + stoptimes)
# Work out frequency steps
if centeronly:
if centrefreq > 1e6:
logger.warning(
"centrefreq is greater than 1e6, assuming input with units of Hz."
)
frequencies = np.array([centrefreq])
else:
frequencies = np.array([centrefreq]) * 1e6
nfreqs = 1
else:
# in Hz
frequencies = np.array(channels) * 1.28e6
nfreqs = len(channels)
# Set up np power array
PowersX = np.zeros((len(names_ra_dec), ntimes, nfreqs))
PowersY = np.zeros((len(names_ra_dec), ntimes, nfreqs))
# Convert RA and Dec to desired units
if degrees:
RAs = np.array(names_ra_dec[:, 1], dtype=float)
Decs = np.array(names_ra_dec[:, 2], dtype=float)
else:
RAs, Decs = sex2deg(names_ra_dec[:, 1], names_ra_dec[:, 2])
# Then check if they're valid
if len(RAs) == 0:
sys.stderr.write('Must supply >=1 source positions\n')
return None
if not len(RAs) == len(Decs):
sys.stderr.write('Must supply equal numbers of RAs and Decs\n')
return None
if verbose is False:
#Supress print statements of the primary beam model functions
sys.stdout = open(os.devnull, 'w')
for itime in range(ntimes):
# this differ's from the previous ephem_utils method by 0.1 degrees
_, Azs, Zas = mwa_alt_az_za(midtimes[itime],
ra=RAs,
dec=Decs,
degrees=True)
# go from altitude to zenith angle
theta = np.radians(Zas)
phi = np.radians(Azs)
for ifreq in range(nfreqs):
#Decide on beam model
if option == 'analytic':
rX, rY = primary_beam.MWA_Tile_analytic(
theta,
phi,
freq=frequencies[ifreq],
delays=delays,
zenithnorm=True,
power=True)
elif option == 'advanced':
rX, rY = primary_beam.MWA_Tile_advanced(
theta,
phi,
freq=frequencies[ifreq],
delays=delays,
zenithnorm=True,
power=True)
elif option == 'full_EE':
rX, rY = primary_beam.MWA_Tile_full_EE(theta,
phi,
freq=frequencies[ifreq],
delays=delays,
zenithnorm=True,
power=True)
elif option == 'hyperbeam':
jones = beam.calc_jones_array(phi, theta,
int(frequencies[ifreq]),
delays[0], amps, True)
jones = jones.reshape(1, len(phi), 2, 2)
vis = primary_beam.mwa_tile.makeUnpolInstrumentalResponse(
jones, jones)
rX, rY = (vis[:, :, 0, 0].real, vis[:, :, 1, 1].real)
PowersX[:, itime, ifreq] = rX
PowersY[:, itime, ifreq] = rY
if verbose is False:
sys.stdout = sys.__stdout__
Powers = 0.5 * (PowersX + PowersY)
return Powers
except ImportError as ie:
logger.error(
"Couldn't import version.py - have you installed vcstools?")
logger.error("ImportError: {0}".format(ie))
sys.exit(0)
if args.out_dir:
out_dir = args.out_dir
else:
out_dir = os.path.join(comp_config['base_data_dir'], str(args.obsid),
"cal", f"{args.cal1}_{args.cal2}", "rts")
cal_base_dir = os.path.join(comp_config['base_data_dir'], str(args.obsid),
"cal")
logger.debug(cal_base_dir)
obs_chans = get_common_obs_metadata(args.obsid)[6]
obs_chans_reordered = order_chans(obs_chans)
cal1_chans = get_common_obs_metadata(args.cal1)[6]
cal1_chans_reordered = order_chans(cal1_chans)
cal2_chans = get_common_obs_metadata(args.cal2)[6]
cal2_chans_reordered = order_chans(cal2_chans)
logger.debug(obs_chans)
logger.debug(obs_chans_reordered)
logger.debug(cal1_chans)
logger.debug(cal1_chans_reordered)
logger.debug(cal2_chans)
logger.debug(cal2_chans_reordered)
# Check input calbration obs have all the frequency channels we need
for fc in obs_chans:
def find_pulsars_in_fov(obsid,
psrbeg,
psrend,
fwhm=None,
search_radius=0.02,
meta_data=None,
full_meta=None,
no_known_pulsars=False,
no_search_cands=False):
"""
Find all pulsars in the field of view and return all the pointings sorted into vdif and normal lists:
Parameters:
-----------
obsid: int
The observation ID
psrbeg: int
The begining of the observation you are processing in GPS time
psrend: int
The end of the observation you are processing in GPS time
fwhm: float
OPTIONAL - The FWHM of the beam in degrees.
Default None: Value will be estimated
search_radius: float
OPTIONAL - The radius to search (create beams within) in degrees to account for ionosphere.
Default: 0.02 degrees
meta_data: list
OPTIONAL - Comon observation metadata in the format from vcstools.metadb_utils.get_common_obs_metadata
Default None: Will perform the metadata call
full_meta: dict
OPTIONAL - Full observation metadata in the format from vcstools.metadb_utils.getmeta
Default None: Will perform the metadata call
no_known_pulsars: bool
OPTIONAL - Will return no known pulsars
Default: False
no_search_cands: bool
OPTIONAL - Will return no search candidates
Default: False
Returns:
--------
list of lists:
[pulsar_name_list,
pulsar_pointing_list,
vdif_name_list,
vdif_pointing_list,
sp_name_list,
sp_pointing_list]
"""
if not meta_data or not full_meta:
meta_data, full_meta = get_common_obs_metadata(obsid, return_all=True)
channels = meta_data[-1]
if fwhm is None:
# Estimate FWHM
oap = get_obs_array_phase(obsid)
centrefreq = 1.28 * float(min(channels) + max(channels)) / 2.
fwhm = calc_ta_fwhm(centrefreq, array_phase=oap)
# Find all pulsars in beam at at least 0.3 and 0.1 of zenith normlaized power
names_ra_dec = np.array(grab_source_alog(max_dm=250))
pow_dict, _ = find_pulsars_power(obsid,
powers=[0.3, 0.1],
names_ra_dec=names_ra_dec)
obs_psrs = pow_dict[0.3][obsid]
# Find pulsars with power between 0.3 and 0.1 and calculate their SN
psrs_list_03 = [x[0] for x in obs_psrs]
psrs_list_01 = [x[0] for x in pow_dict[0.1][obsid]]
psrs_03_01 = psrs_list_01
for psr in psrs_list_03:
if psr in psrs_list_01:
psrs_03_01.remove(psr)
sn_dict_01 = snfu.multi_psr_snfe(psrs_03_01,
obsid,
beg=psrbeg,
end=psrend,
min_z_power=0.1,
obs_metadata=meta_data,
full_meta=full_meta)
# Include all bright pulsars in beam at at least 0.1 of zenith normalized power
for psr in psrs_03_01:
sn, sn_err, _, _ = sn_dict_01[psr]
if sn is not None and sn_err is not None:
if sn - sn_err >= 10.:
for psr_list in pow_dict[0.1][obsid]:
if psr in psr_list:
obs_psrs.append(psr_list)
#get all the pulsars periods
pulsar_list = []
for o in obs_psrs:
pulsar_list.append(o[0])
period_query = psrqpy.QueryATNF(params=["PSRJ", "P0"],
psrs=pulsar_list,
loadfromdb=data_load.ATNF_LOC).pandas
# Sort all the sources into 3 categories, pulsars which is for slow pulsars, vdif
# for fast pulsars that require vdif and sp for singple pulse searches (FRBs,
# RRATs and pulsars without ATNF periods)
pulsar_pointing_list = []
pulsar_name_list = []
vdif_pointing_list = []
vdif_name_list = []
pulsar_search_pointing_list = []
pulsar_search_name_list = []
sp_pointing_list = []
sp_name_list = []
for pi, pulsar_line in enumerate(obs_psrs):
vdif_check = False
sp_check = False
PSRJ = pulsar_line[0]
if not (len(PSRJ) < 11 or PSRJ[-1] == 'A' or PSRJ[-2:] == 'aa'):
continue
for line in names_ra_dec:
if PSRJ == line[0]:
temp = [line]
temp = format_ra_dec(temp, ra_col=1, dec_col=2)
jname, raj, decj = temp[0]
#get pulsar period
period = period_query[period_query['PSRJ'] ==
PSRJ].reset_index()["P0"][0]
if math.isnan(period):
logger.warn("Period not found in ephermeris for {0} so assuming "
"it's an RRAT".format(jname))
sp_check = True
period = 0.
elif float(period) < .05:
vdif_check = True
period = float(period) * 1000.
logger.debug(
"{0:12} RA: {1} Dec: {2} Period: {3:8.2f} (ms) Begin {4} End {5}".
format(PSRJ, raj, decj, period, psrbeg, psrend))
jname_temp_list = []
if PSRJ[-1] == 'A' or PSRJ[-2:] == 'aa':
#Got to find all the pulsar J names with other letters
vdif_check = True
for pulsar_l in obs_psrs:
pulsar_name = pulsar_l[0]
if pulsar_name.startswith(PSRJ[:-2]):
jname_temp_list.append(pulsar_name)
else:
jname_temp_list.append(jname)
# grid the pointings to fill 2 arcminute raduis to account for ionosphere shift
pointing_list_list = get_pointings_required(raj, decj, fwhm,
search_radius)
# sort the pointings into the right groups
for prd in pointing_list_list:
if vdif_check:
vdif_name_list.append(jname_temp_list)
vdif_pointing_list.append("{0}_{1}".format(prd[0], prd[1]))
elif sp_check:
sp_name_list.append(jname_temp_list)
sp_pointing_list.append("{0}_{1}".format(prd[0], prd[1]))
else:
pulsar_name_list.append(jname_temp_list)
pulsar_pointing_list.append("{0}_{1}".format(prd[0], prd[1]))
#Get the rest of the singple pulse search canidates
#-----------------------------------------------------------------------------------------------------------
temp = get_sources_in_fov(obsid, 'RRATs', fwhm)
sp_name_list = sp_name_list + temp[0]
sp_pointing_list = sp_pointing_list + temp[1]
# Find all of the FRB candidates
#-----------------------------------------------------------------------------------------------------------
temp = get_sources_in_fov(obsid, 'FRB', fwhm)
sp_name_list = sp_name_list + temp[0]
sp_pointing_list = sp_pointing_list + temp[1]
# Find all of the Fermi candidates
#-----------------------------------------------------------------------------------------------------------
fermi_list = get_sources_in_fov(obsid, 'Fermi', fwhm)
logger.info(f"{obsid} Fermi candidates: {fermi_list}")
pulsar_search_name_list = pulsar_search_name_list + fermi_list[0]
pulsar_search_pointing_list = pulsar_search_pointing_list + fermi_list[1]
# Find all of the points of interest candidates
#-----------------------------------------------------------------------------------------------
poi_list = get_sources_in_fov(obsid, 'POI', fwhm)
logger.info(f"{obsid} Points of interest: {poi_list}")
pulsar_search_name_list = pulsar_search_name_list + poi_list[0]
pulsar_search_pointing_list = pulsar_search_pointing_list + poi_list[1]
# Sometimes we get redundant RRATs that are found in RRAT and ANTF catalogues so they need to be removed
sp_name_list = list(dict.fromkeys([":".join(s) for s in sp_name_list]))
sp_pointing_list = list(dict.fromkeys(sp_pointing_list))
# Changing the format of the names list to make it easier to format
pulsar_name_list = [":".join(s) for s in pulsar_name_list]
vdif_name_list = [":".join(s) for s in vdif_name_list]
pulsar_search_name_list = [":".join(s) for s in pulsar_search_name_list]
if no_known_pulsars:
# Return empty list for all known pulsar categories
pulsar_name_list = []
pulsar_pointing_list = []
vdif_name_list = []
vdif_pointing_list = []
if no_search_cands:
# Return empty list for all search candidate categories
pulsar_search_name_list = []
pulsar_search_pointing_list = []
sp_name_list = []
sp_pointing_list = []
return [
pulsar_name_list, pulsar_pointing_list, vdif_name_list,
vdif_pointing_list, pulsar_search_name_list,
pulsar_search_pointing_list, sp_name_list, sp_pointing_list
]
from vcstools.general_utils import setup_logger
from vcstools.radiometer_equation import find_t_sys_gain, find_pulsar_w50,\
est_pulsar_flux, est_pulsar_sn,\
flux_calc_radiometer_equation,\
flux_calc_flux_profile
import logging
logger = logging.getLogger(__name__)
#logger = setup_logger(logger, log_level="DEBUG")
#Global obsid information to speed things up
md_dict={}
obsid_list = [1222697776, 1226062160, 1225713560, 1117643248]
for obs in obsid_list:
md_dict[str(obs)] = get_common_obs_metadata(obs, return_all=True)
query = psrqpy.QueryATNF(loadfromdb=data_load.ATNF_LOC).pandas
# The five pulsars from 1276619416 that were also detected in imaging.
pulsars_to_test = []
# J1820-0427 545.428 mJy. A bright pulsar with two components and a scattering tail that is about 50% of the profile
J1820_0427_profile = np.array([0.00974835457, 0.00275491384, 0.00109578621, -0.00279699934, 0.000562140998, 0.00113477532, 0.00339133085, -0.0109063454, -0.00964722073, 0.00111819766, 0.00510179576, -0.00238241189, -0.0159297665, 0.00526543335, 0.00104425447, -0.00416604215, -0.00577553402, -0.0107780994, -0.0082326258, -0.0134817171, -0.00582210704, -0.0124964885, -0.00268844238, -0.00974819377, -0.016102884, -0.0141605262, -0.00882266424, -0.0111797553, -0.0167948192, -0.00793797119, -0.00794229791, -0.000902770299, -2.88211273e-05, -0.00844239888, -0.00128177167, -0.00585817926, 0.00461725154, 0.118554769, 0.631823206, 0.978832608, 1.0, 0.833592824, 0.657390729, 0.523552684, 0.423043522, 0.361585454, 0.317663881, 0.288038185, 0.253027957, 0.223348542, 0.190868669, 0.16998052, 0.150608232, 0.158389622, 0.168857566, 0.17383913, 0.194769962, 0.185245049, 0.18829777, 0.171631238, 0.13852924, 0.136540455, 0.128550629, 0.121552035, 0.126898161, 0.11610917, 0.0966390774, 0.0836497683, 0.0709359634, 0.0663007999, 0.062169121, 0.0804554427, 0.0738022707, 0.0652449194, 0.0631586179, 0.0528215401, 0.0435259772, 0.037796751, 0.0423237659, 0.0357590644, 0.0347282449, 0.0329981882, 0.0295279872, 0.0228235617, 0.0293093176, 0.0313360707, 0.0207441587, 0.02619471, 0.00945243598, 0.0276467383, 0.0149054666, 0.011583468, 0.00358243583, 0.010961684, 0.0240656226, -0.00276778182, 0.00860143302, 0.00782033821, 0.011622846, 0.00154440406, 0.00617944115, -0.000217641207, 0.0141621455, 0.0112963973, 0.0187439007, 0.00671303776, 0.014185432, 0.00511642883, 0.00926998444, 0.00543009184, 0.00484238691, 0.00771907348, 0.0186740412, -0.00596104829, -0.00456473254, -1.99246096e-05, 0.00802933345, 0.00264869039, 0.0164105337, -0.00607378612, 0.00422687429, 0.0106680503, -0.00803578427, -0.00855158784, -0.0111729006, -0.00683086519, -0.00433629136, -0.00145620176])
J1820_0427_profile_bool = [True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True]
J1820_0427_data = [0.00808072162938938, 11.269025292925557, 2786.22, 17.813186127136287, 4920.0, 597.3405435783852, 74.26521548924087, 194.85305766037393, 16.8787754588467, 570.440121819025, 5.127532489305003]
pulsars_to_test.append( ("J1820-0427", 545.428, J1820_0427_profile, J1820_0427_profile_bool, J1820_0427_data) )
# J1825-0935 176.222 mJy. A bright pulsar with a small interpulse
J1825_0935_profile = np.array([0.00314088125, 0.000556045128, 0.00105395013, 0.0045659471, 0.000470929106, -0.000409939544, -0.00412380941, 0.00288133825, 0.0103584648, -0.00706736449, -0.000329613318, 0.00548152716, 0.00668359412, -0.00481527652, 0.00774982755, -0.00279912297, -0.00403820169, -0.00168432796, 0.000365357593, 0.0031319756, 0.000397067214, 0.0107867029, 0.0191924172, 0.0207955352, 0.018085596, 0.0247475429, 0.0163618917, 0.0198945505, 0.0172405494, 0.0250506153, 0.0946288765, 0.499182263, 1.0, 0.593519933, 0.135216572, 0.0189654022, 0.00493651504, -0.000583671957, 0.00145633429, -0.00247092049, 0.00340683702, 0.00189445108, -0.00245343584, 0.000542236211, 0.00387792879, 0.0181227866, 0.00592787911, -0.00376045883, 0.0049415421, 0.00555503833, 0.00204712627, 0.00205753797, 0.0036205589, -0.000353569177, 0.00817348899, 0.01150161, 0.00514606606, 0.00020703299, 0.00484254839, -0.00216935931, 0.00545115511, 0.0109659712, -0.000176346712, 0.0054985373, 0.00441087186, 0.00814568892, 0.00871168261, 0.00776448921, 0.0023226238, -0.000963073342, -0.00339329842, 0.00568027449, 0.00280664424, -0.00246610319, 0.00131316992, 0.000193402873, -0.00158659617, -0.00183816535, -0.00666429952, 0.0033070112, 0.00128840946, -0.00409108888, 0.0073903315, 0.00100847286, 0.00455245796, 0.00993097978, -0.000629802544, -0.0088467171, 0.00566167921, 0.00443358643, 0.00194643739, 0.00724301346, 0.0081186625, 0.0155670915, 0.0196579249, 0.0117332429, 0.0249344743, 0.0400196965, 0.0639603915, 0.0317363231, 0.00753277035, 0.00222084149, 0.0037240295, 0.00248847523, -0.00204282341, 0.00200497056, 0.00673926304, 0.0134845754, -0.00201286054, 0.00573517662, 0.00362153543, 0.00730374381, 0.00889853625, 0.00706360068, 0.00481938682, 0.00830079167, -0.00203974254, 0.00745314557, 0.00618885252, -0.00471675388, 0.00923441341, 0.000585897973, 0.0116158121, 0.00490610861, -0.00228758785, 0.00194633768, 0.00473734135, 0.00743085737])
J1825_0935_profile_bool = [True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True]
J1825_0935_data = [0.0031191764012526266, 3.0603445862819347, 2785.346666666667, 17.89875807816685, 4920.0, 440.5273973148081, 152.4567114103757, 254.19451614153593, 28.828176263272688, 435.22926553433194, 5.202306430741997]
pulsars_to_test.append( ("J1825-0935", 176.222, J1825_0935_profile, J1825_0935_profile_bool, J1825_0935_data) )
def analyise_and_flux_cal(pulsar, bestprof_data,
flagged_tiles=None,
calid=None,
common_metadata=None,
trcvr=data_load.TRCVR_FILE,
simple_sefd=False, sefd_file=None,
vcstools_version='master',
args=None,
flux_method="radiometer"):
"""Analyise a pulse profile and calculates its flux density
Parameters
----------
pulsar : `str`
The pulsar's Jname
bestprof_data: list
The output list from the function :py:meth:`vcstools.prof_utils.get_from_bestprof`
Optional parameters:
-------------------
flagged_tiles : `str`
The location of the flagged_tiles.txt file. If it's in the default location you can just supply the calid.
calid : `int`
The calibration ID of the detection. This is used to find the flagged_tiles.txt file.
common_metadata: list
The output of mwa_metadb_utils.get_common_obs_metadata(). If not supplied it will be downloaded.
trcvr : `str`
The file location of antena temperatures.
simple_sefd : `boolean`
If True perfom just a simple SEFD calculation instead of simulating the phased array beam response over the sky. Default: False.
sefd_file : `str`
The location of the pabeam.py's simulation of the phased array beam response over the sky output file. If not supplied will launch a pabeam.py simulation.
vcstools_version : `str`
The version of vcstools to use for the pabeam.py simulation. Default: master.
args: Namespace
The args from argparse to be used for job resubmission. Default: None.
Returns
-------
det_kwargs: dict
det_kwargs["flux"]: The mean flux density of the pulsar in mJy
det_kwargs["flux_error"]: The flux desnity error in mJy
det_kwargs["width"]: The equivalent width of the pulsar in ms
det_kwargs["width_error"]: The error of the equivalent width in ms
det_kwargs["scattering"]: The scattering width in s
det_kwargs["scattering_error"]: The error of the scattering in s
"""
# Load computer dependant config file
comp_config = load_config_file()
#unpack the bestprof_data
obsid, prof_psr, _, period, _, sigma, beg, t_int, profile, num_bins, pointing = bestprof_data
period=float(period)
num_bins=int(num_bins)
# Perform metadata calls
if common_metadata is None:
common_metadata = get_common_obs_metadata(obsid)
obsid, ra, dec, dura, [xdelays, ydelays], centrefreq, channels = common_metadata
# assume full bandwidth of 30.72 MHz
bandwidth = 30.72e6
# Find pulsar ra and dec
_, pul_ra, pul_dec = get_psrcat_ra_dec(pulsar_list=[pulsar])[0]
# Work out flagged tiles from calbration directory
if not flagged_tiles:
if calid:
flagged_file = os.path.join(comp_config['base_data_dir'], obsid, "cal", calid, "rts", "flagged_tiles.txt")
if os.path.exists(flagged_file):
with open(flagged_file, "r") as ftf:
flagged_tiles = []
reader = csv.reader(ftf)
for row in reader:
flagged_tiles.append(row)
flagged_tiles = np.array(flagged_tiles).flatten()
else:
logger.warn("No flagged_tiles.txt file found so assuming no tiles have been flagged")
flagged_tiles = []
else:
logger.warn("No flagged_tiles or calid provided so assuming no tiles have been flagged")
flagged_tiles = []
# Calc SEFD from the T_sys and gain
if simple_sefd:
t_sys, _, gain, u_gain = find_t_sys_gain(pulsar, obsid,
common_metadata=common_metadata,
beg=beg, end=(t_int + beg - 1))
sefd = tsys / gain
else:
if sefd_file is None:
launch_pabeam_sim(obsid, pointing, beg, t_int,
source_name=pulsar,
vcstools_version=vcstools_version,
flagged_tiles=flagged_tiles,
delays=xdelays,
args=args,
common_metadata=common_metadata)
sys.exit(0)
else:
sefd_freq_time, sefd, u_sefd = read_sefd_file(sefd_file)
#estimate S/N
try:
g_fitter = gfit(profile)
g_fitter.plot_name = f"{obsid}_{pulsar}_{num_bins}_bins_gaussian_fit.png"
g_fitter.component_plot_name = f"{obsid}_{pulsar}_{num_bins}_bins_gaussian_components.png"
g_fitter.auto_fit()
g_fitter.plot_fit()
prof_dict = g_fitter.fit_dict
except (prof_utils.ProfileLengthError, prof_utils.NoFitError):
logger.info("Profile couldn't be fit. Using old style of profile analysis")
prof_dict = prof_utils.auto_analyse_pulse_prof(profile, period)
if not prof_dict:
logger.warn("Profile could not be analysed using any methods")
det_kwargs = {}
det_kwargs["flux"] = None
det_kwargs["flux_error"] = None
det_kwargs["width"] = None
det_kwargs["width_error"] = None
det_kwargs["scattering"] = None
det_kwargs["scattering_error"] = None
return det_kwargs, None, None
# Unpack dictionary
sn = prof_dict["sn"]
u_sn = prof_dict["sn_e"]
profile = prof_dict["profile"]
on_pulse_bool = prof_dict["on_pulse_bool"]
noise_std = prof_dict["noise_std"]
noise_mean = prof_dict["noise_mean"]
w_equiv_phase = prof_dict["Weq"]
u_w_equiv_phase = prof_dict["Weq_e"]
w_equiv_bins = w_equiv_phase * num_bins
u_w_equiv_bins = w_equiv_phase * num_bins
w_equiv_ms = period * w_equiv_phase
u_w_equiv_ms = period * u_w_equiv_phase
scattering = prof_dict["Wscat"]*period/1000 #convert to seconds
u_scattering = prof_dict["Wscat_e"]*period/1000
scattered = prof_dict["scattered"]
logger.info("Profile scattered? {0}".format(scattered))
logger.info("S/N: {0} +/- {1}".format(sn, u_sn))
#logger.debug("Gain {0} K/Jy".format(gain))
logger.debug("Equivalent width in ms: {0}".format(w_equiv_ms))
#logger.debug("T_sys: {0} K".format(t_sys))
logger.debug("Detection time: {0}".format(t_int))
logger.debug("Number of bins: {0}".format(num_bins))
# Renormalise around the noise mean
noise = []
noise_i = []
on_pulse = []
on_pulse_i = []
for p, b, i in zip(profile, on_pulse_bool, range(len(profile))):
if not b:
noise.append(p)
noise_i.append(i)
else:
on_pulse.append(p)
on_pulse_i.append(i)
plt.scatter(on_pulse_i, on_pulse, color="blue")
plt.scatter(noise_i, noise, color="red")
plt.savefig("test.png")
noise_mean = np.mean(noise)
print(f"Noise mean: {noise_mean}")
profile = (profile - noise_mean) / max(profile - noise_mean)
logger.debug(list(profile))
logger.debug(on_pulse_bool)
logger.debug(noise_std, w_equiv_bins, sefd, u_sefd, t_int)
# Final calc of the mean flux density in mJy
if flux_method == "radiometer":
S_mean, u_S_mean = flux_calc_radiometer_equation(profile, on_pulse_bool,
noise_std, w_equiv_bins,
sefd, u_sefd, t_int, bandwidth=bandwidth)
elif flux_method == "flux_profile":
S_mean, u_S_mean = flux_calc_flux_profile(profile,
noise_std,
sefd, u_sefd, t_int, bandwidth=bandwidth)
logger.info('Smean {0:.3f} +/- {1:.3f} mJy'.format(S_mean, u_S_mean))
#prevent TypeError caused by trying to format Nones given to fluxes for highly scattered pulsars
S_mean = float("{0:.3f}".format(S_mean))
u_S_mean = float("{0:.3f}".format(u_S_mean))
# Plot flux comparisons for ANTF
freq_all, flux_all, flux_err_all, _ = flux_from_atnf(pulsar)
logger.debug("Freqs: {0}".format(freq_all))
logger.debug("Fluxes: {0}".format(flux_all))
logger.debug("Flux Errors: {0}".format(flux_err_all))
logger.debug("{0} there are {1} flux values available on the ATNF database"\
.format(pulsar, len(flux_all)))
# Check if there is enough data to estimate the flux
#if len(flux_all) == 0:
# logger.debug("{} no flux values on archive. Cannot estimate flux.".format(pulsar))
#elif ( len(flux_all) == 1 ) and ( ( not spind ) or ( not spind_err ) ):
# logger.debug("{} has only a single flux value and no spectral index. Cannot estimate flux. Will return Nones".format(pulsar))
#else:
# spind, spind_err, K, covar_mat = find_spind(pulsar, freq_all, flux_all, flux_err_all)
# plot_flux_estimation(pulsar, freq_all, flux_all, flux_err_all, spind,
# my_nu=centrefreq, my_S=S_mean, my_S_e=u_S_mean, obsid=obsid,
# a_err=spind_err, K=K, covar_mat=covar_mat)
#format data for uploading
w_equiv_ms = float("{0:.2f}".format(w_equiv_ms))
u_w_equiv_ms = float("{0:.2f}".format(u_w_equiv_ms))
scattering = float("{0:.5f}".format(scattering))
u_scattering = float("{0:.5f}".format(u_scattering))
det_kwargs = {}
det_kwargs["flux"] = S_mean
det_kwargs["flux_error"] = u_S_mean
det_kwargs["width"] = w_equiv_ms
det_kwargs["width_error"] = u_w_equiv_ms
det_kwargs["scattering"] = scattering
det_kwargs["scattering_error"] = u_scattering
return det_kwargs, sn, u_sn
default="master",
help="VCSTools version to load in jobs (i.e. on the queues) ")
optional_options.add_argument("-L",
"--loglvl",
type=str,
help="Logger verbosity level. Default: INFO",
choices=loglevels.keys(),
default="INFO")
args = parser.parse_args()
# set up the logger for stand-alone execution
logger = setup_logger(logger, log_level=loglevels[args.loglvl])
# get metadata
common_metadata = get_common_obs_metadata(args.obsid)
# Option parsing
if not args.obsid:
logger.error(
"Observation ID required, please put in with -o or --obsid")
sys.exit(0)
if args.all and (args.begin or args.end):
logger.error("Please specify EITHER (-b,-e) OR -a")
sys.exit(0)
elif args.all:
args.begin, args.end = obs_max_min(args.obsid)
if args.begin and args.end:
if args.begin > args.end:
logger.error("Starting time is after end time")
sys.exit(0)
def find_t_sys_gain(pulsar, obsid,
p_ra=None, p_dec=None,
dect_beg=None, dect_end=None,
obs_beg=None, obs_end=None,
common_metadata=None, full_metadata=None,
query=None, min_z_power=0.3, trcvr=data_load.TRCVR_FILE):
"""Finds the system temperature and gain for an observation.
Parameters
----------
pulsar : `str`
The Jname of the pulsar.
obsid : `int`
The MWA Observation ID.
p_ra, p_dec : `str`, optional
The target's right ascension and declination in sexidecimals. If not supplied will use the values from the ANTF.
dect_beg, dect_end : `int`, optional
The beg and end GPS time of the detection to calculate over.
If not supplied will estimate beam enter and exit.
obs_beg, obs_end : `int`, optional
Beginning and end GPS time of the observation.
If not supplied will use :py:meth:`vcstools.metadb_utils.obs_max_min` to find it.
common_metadata : `list`, optional
The list of common metadata generated from :py:meth:`vcstools.metadb_utils.get_common_obs_metadata`.
full_metadata : `dict`, optional
The dictionary of metadata generated from :py:meth:`vcstools.metadb_utils.getmeta`.
query : psrqpy object, optional
A previous psrqpy.QueryATNF query. Can be supplied to prevent performing a new query.
min_z_power : `float`, optional
Zenith normalised power cut off. |br| Default: 0.3.
trcvr : `str`
The location of the MWA receiver temp csv file. |br| Default: <vcstools_data_dir>MWA_Trcvr_tile_56.csv
Returns
-------
t_sys : `float`
The system temperature in K.
t_sys_err : `float`
The system temperature's uncertainty.
gain : `float`
The system gain in K/Jy.
gain_err : `float`
The gain's uncertainty.
"""
# get ra and dec if not supplied
if query is None:
logger.debug("Obtaining pulsar RA and Dec from ATNF")
query = psrqpy.QueryATNF(psrs=[pulsar], loadfromdb=data_load.ATNF_LOC).pandas
query_id = list(query['PSRJ']).index(pulsar)
if not p_ra or not p_dec:
p_ra = query["RAJ"][query_id]
p_dec= query["DECJ"][query_id]
# get metadata if not supplied
if not common_metadata:
logger.debug("Obtaining obs metadata")
common_metadata = get_common_obs_metadata(obsid)
obsid, obs_ra, obs_dec, _, delays, centrefreq, channels = common_metadata
if not dect_beg or not dect_end:
# Estimate integration time from when the source enters and exits the beam
dect_beg, dect_end = source_beam_coverage_and_times(obsid, pulsar,
p_ra=p_ra, p_dec=p_dec,
obs_beg=obs_beg, obs_end=obs_end,
min_z_power=min_z_power,
common_metadata=common_metadata,
query=query)[:2]
start_time = dect_end - int(obsid)
t_int = dect_end - dect_beg + 1
#Get important info
ntiles = 128 #TODO actually we excluded some tiles during beamforming, so we'll need to account for that here
beam_power = get_beam_power_over_time(np.array([[pulsar, p_ra, p_dec]]),
common_metadata=[obsid, obs_ra, obs_dec, t_int, delays,
centrefreq, channels],
dt=100, start_time=start_time)
mean_beam_power = np.mean(beam_power)
# Usa a primary beam function to convolve the sky temperature with the primary beam
# prints suppressed
sys.stdout = open(os.devnull, 'w')
_, _, Tsky_XX, _, _, _, Tsky_YY, _ = pbtant.make_primarybeammap(int(obsid), delays, centrefreq*1e6, 'analytic', plottype='None')
sys.stdout = sys.__stdout__
#TODO can be inaccurate for coherent but is too difficult to simulate
t_sky = (Tsky_XX + Tsky_YY) / 2.
# Get T_sys by adding Trec and Tsky (other temperatures are assumed to be negligible
t_sys_table = t_sky + get_Trec(centrefreq, trcvr_file=trcvr)
t_sys = np.mean(t_sys_table)
t_sys_err = t_sys*0.02 #TODO: figure out what t_sys error is
logger.debug("pul_ra: {} pul_dec: {}".format(p_ra, p_dec))
_, _, zas = mwa_alt_az_za(obsid, ra=p_ra, dec=p_dec)
theta = np.radians(zas)
gain = from_power_to_gain(mean_beam_power, centrefreq*1e6, ntiles, coh=True)
logger.debug("mean_beam_power: {} theta: {} pi: {}".format(mean_beam_power, theta, np.pi))
gain_err = gain * ((1. - mean_beam_power)*0.12 + 2.*(theta/(0.5*np.pi))**2. + 0.1)
return t_sys, t_sys_err, gain, gain_err
logger = logging.getLogger(__name__)
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description="""Returns information on an input Obs ID""")
parser.add_argument("obsid", type=int, help="Input Observation ID")
parser.add_argument(
"-c",
"--cal_best",
action="store_true",
help="If this option is used it will list "
"calibration observations within 2 days that have the same observing channels and "
"list them from closest in time to furthest.")
args = parser.parse_args()
beam_common_data, data_dict = get_common_obs_metadata(args.obsid,
return_all=True)
channels = data_dict["rfstreams"]["0"]["frequencies"]
centre_freq = (min(channels) + max(channels)) / 2. * 1.28
array_phase = get_obs_array_phase(args.obsid)
fov = field_of_view(args.obsid, common_metadata=beam_common_data)
print("------------------------- Obs Info -------------------------")
print("Obs Name: {}".format(data_dict["obsname"]))
print("Creator: {}".format(
data_dict["rfstreams"]["0"]["creator"]))
print("Array phase: {}".format(array_phase))
print("RA Pointing (deg): {:6.2f}".format(
data_dict["metadata"]["ra_pointing"]))
print("DEC Pointing (deg): {:6.2f}".format(
data_dict["metadata"]["dec_pointing"]))
print("Centrefreq (MHz): {}".format(centre_freq))
def multi_psr_snfe(pulsar_list, obsid, obs_beg, obs_end,
common_metadata=None, full_metadata=None,
query=None, plot_flux=False,
min_z_power=0.3, trcvr=data_load.TRCVR_FILE):
"""Runs :py:meth:`vcstools.sn_flux_utils.est_pulsar_sn` for multiple pulsars in the same MWA observation.
Parameters
----------
pulsar : `list`
A list of the pulsar Jnames.
obsid : `int`
The MWA Observation ID.
obs_beg, obs_end : `int`
Beginning and end GPS time of the observation.
If not supplied will use :py:meth:`vcstools.metadb_utils.obs_max_min` to find it.
common_metadata : `list`, optional
The list of common metadata generated from :py:meth:`vcstools.metadb_utils.get_common_obs_metadata`
full_metadata : `dict`, optional
The dictionary of metadata generated from :py:meth:`vcstools.metadb_utils.getmeta`
query : psrqpy object, optional
A previous psrqpy.QueryATNF query. Can be supplied to prevent performing a new query.
plot_flux : `boolean`, optional
If `True` will produce a plot of the flux estimation. |br| Default: False
min_z_power : `float`, optional
Zenith normalised power cut off. |br| Default: 0.3.
trcvr : `str`
The location of the MWA receiver temp csv file. |br| Default: <vcstools_data_dir>MWA_Trcvr_tile_56.csv
Returns
-------
sn_dict : `dict`
A dictionary where eacy key is the pulsar jname and contains a list of the following
sn_dict[pulsar]=[sn, sn_e, s, s_e]
sn : `float`
The expected signal to noise ratio for the given inputs.
sn_err : `float`
The uncertainty in the signal to noise ratio.
s : `float`
The expected flux density of the pulsar.
s_e : `float`
The uncertainty expected flux density of the pulsar.
"""
logger.info("""This script may use estimations where data is missing.
For full verbosity, use the DEBUG logger (ie. -L DEBUG)""")
if common_metadata is None or full_metadata is None:
logger.debug("Obtaining obs metadata")
common_metadata, full_metadata = get_common_obs_metadata(obsid, return_all=True, full_metadata=full_metadata)
if obs_beg is None or obs_end is None:
obs_beg, obs_end = obs_max_min(obsid)
mega_query = psrqpy.QueryATNF(psrs=pulsar_list, loadfromdb=data_load.ATNF_LOC).pandas
sn_dict = {}
for i, pulsar in enumerate(progress_bar(mega_query["PSRJ"], "Calculating pulsar SN: ")):
psr_query = {}
for key in mega_query.keys():
psr_query[key] = [mega_query[key][i]]
sn, sn_e, s, s_e = est_pulsar_sn(pulsar, obsid,
obs_beg=obs_beg, obs_end=obs_end,
common_metadata=common_metadata, full_metadata=full_metadata,
plot_flux=plot_flux, query=psr_query,
min_z_power=min_z_power, trcvr=trcvr)
sn_dict[pulsar]=[sn, sn_e, s, s_e]
return sn_dict
def find_fold_times(pulsars,
obsid,
beg,
end,
min_z_power=(0.3, 0.1),
metadata=None,
full_meta=None,
query=None):
"""
Finds the fractional time the pulsar is in the beam at some zenith normalized power
Parameters:
-----------
pulsar: list
Pulsar J names of pulsars to evaluate
obsid: int
The observation ID
beg: int
The beginning of the observation time in gps time
end: int
The end of the observation time in gps time
min_z_power: tuple/list
OPTIONAL - evaluated the pulsar as 'in the beam' at this normalized zenith power. If None will use [0.3, 0.1] Default: None
Returns:
fold_times_dict: dict
keys:
psr: dict
keys:
power: float
The power of the recorded enter and leave times
enter: float
The normalised time the pulsar enters the beam
exit: float
The normalisd time the pulsar leaves the beam
"""
if not metadata or not full_meta:
metadata, full_meta = get_common_obs_metadata(obsid, return_all=True)
min_z_power = sorted(min_z_power, reverse=True)
names_ra_dec = grab_source_alog(pulsar_list=pulsars)
pow_dict, _ = find_pulsars_power(obsid,
powers=min_z_power,
names_ra_dec=names_ra_dec,
metadata_list=[[metadata, full_meta]])
fold_time_dict = {}
for psr in pulsars:
fold_time_dict[psr] = {}
for power in pow_dict.keys():
fold_time_dict[psr]["power"] = power
fold_time_dict[psr]["enter"] = None
fold_time_dict[psr]["leave"] = None
psr_list = pow_dict[power][obsid]
if psr_list: # if pulsar is in beam for this power coverage
enter, leave = snfu.pulsar_beam_coverage(obsid,
psr,
beg=beg,
end=end,
min_z_power=power,
metadata=metadata,
full_meta=full_meta,
query=query)
if enter is not None and leave is not None:
fold_time_dict[psr]["enter"] = enter
fold_time_dict[psr]["leave"] = leave
break
return fold_time_dict
def plot_beam(obs, target, cal, freq):
metadata = get_common_obs_metadata(obs)
phi = np.linspace(0, 360, 3600)
theta = np.linspace(0, 90, 900)
# make coordinate grid
az, za = np.meshgrid(np.radians(phi), np.radians(theta))
# compute beam and plot
delays = metadata[4] #x and y delays
logger.debug("delays: {0}".format(delays))
logger.debug("freq*1e6: {0}".format(freq * 1e6))
logger.debug("za: {0}".format(za))
logger.debug("az: {0}".format(az))
gx, gy = pb.MWA_Tile_analytic(za,
az,
freq=int(freq * 1e6),
delays=delays,
power=True,
zenithnorm=True)
beam = (gx + gy) / 2.0
fig = plt.figure(figsize=(10, 8))
ax = fig.add_subplot(111, polar=True, aspect='auto')
# filled contour setup
lower_contour = 7e-3
upper_contour = beam.max()
fill_min = 1e-2 # 1% of zenith power
fill_max = 0.95 * beam.max() # 95% max beam power ( != zenith power)
Z = np.copy(beam)
Z[Z <= fill_min] = 0
Z[Z >= fill_max] = fill_max
cc_levels = np.logspace(np.log10(lower_contour),
np.log10(upper_contour),
num=20)
cc_cmap = plt.get_cmap('gray_r')
cc_norm = LogNorm(vmin=cc_levels.min(), vmax=beam.max())
cc = ax.contourf(az,
za,
Z,
levels=cc_levels,
cmap=cc_cmap,
norm=cc_norm,
zorder=1000)
cc.cmap.set_over('black') # anything over max is black
cc.cmap.set_under('white') # anything under min is white
# contour lines steup
cs_levels = np.logspace(np.log10(fill_min), np.log10(fill_max), num=5)
cs_cmap = plt.get_cmap('gist_heat')
cs_norm = LogNorm(vmin=cs_levels.min(), vmax=cs_levels.max())
cs = ax.contour(az,
za,
beam,
levels=cs_levels,
cmap=cs_cmap,
norm=cs_norm,
zorder=1001)
# color bar setup
cbarCC = plt.colorbar(cc, pad=0.08, shrink=0.9)
cbarCC.set_label(label="zenith normalised power", size=20)
cbarCC.set_ticks(cs_levels)
cbarCC.set_ticklabels([r"${0:.2f}$".format(x) for x in cs_levels])
cbarCC.ax.tick_params(labelsize=18)
#add lines from the contours to the filled color map
cbarCC.add_lines(cs)
# plot the pointing centre of the tile beam
_, pointing_AZ, pointing_ZA = mwa_alt_az_za(obs)
ax.plot(np.radians(pointing_AZ),
np.radians(pointing_ZA),
ls="",
marker="+",
ms=8,
color='C3',
zorder=1002,
label="pointing centre")
if target is not None:
# color map for tracking target positions
target_colors = cm.viridis(np.linspace(0, 1, len(target.obstime.gps)))
logger.info("obs time length: {0} seconds".format(
len(target.obstime.gps)))
for t, color in zip(target, target_colors):
target_az = t.altaz.az.rad
target_za = np.pi / 2 - t.altaz.alt.rad
# get beam power for target
bpt_x, bpt_y = pb.MWA_Tile_analytic(
target_za,
target_az,
freq=freq * 1e6,
delays=[metadata[4][0], metadata[4][1]],
power=True,
zenithnorm=True)
bpt = (bpt_x + bpt_y) / 2.0
lognormbpt = log_normalise(bpt, cc_levels.min(), beam.max())
logger.debug("bpt, cc_levels, beam: {0}, {1}, {2}".format(
bpt, cc_levels.min(), beam.max()))
logger.info("Beam power @ source @ gps: {0}: {1}".format(
t.obstime.gps, bpt))
logger.info("log-normalised BP: {0}".format(lognormbpt))
# plot the target position on sky
ax.plot(target_az,
target_za,
ls="",
marker="o",
color=color,
zorder=1002,
label="target @ {0} ({1})".format(t.obstime.gps, bpt))
# plot the target on the color bar
cbarCC.ax.plot(0.5, lognormbpt, color=color, marker="o")
if cal is not None:
# color map for tracking calibrator position
calibrator_colors = cm.viridis(np.linspace(0, 1, len(cal.obstime.gps)))
for c, color in zip(cal, calibrator_colors):
cal_az = c.altaz.az.rad
cal_za = np.pi / 2 - c.altaz.alt.rad
# get the beam power for calibrator
bpc_x, bpc_y = pb.MWA_Tile_analytic([[cal_za]], [[cal_az]],
freq=freq * 1e6,
delays=metadata[4],
power=True,
zenithnorm=True)
bpc = (bpc_x + bpc_y) / 2.0
lognormbpc = log_normalise(bpc, cc_levels.min(), beam.max())
logger.info("Beam power @ calibrator @ gps:{0}: {1:.3f}".format(
c.obstime.gps, bpc[0][0]))
logger.info("log-normalised BP: {0:.3f}".format(lognormbpc[0][0]))
# plot the calibrator position on sky
ax.plot(cal_az,
cal_za,
ls="",
marker="^",
color=color,
zorder=1002,
label="cal @ {0} ({1:.2f})".format(c.obstime.gps,
bpc[0][0]))
# plot the calibrator on the color bar
cbarCC.ax.plot(0.5, lognormbpc, color=color, marker="^")
# draw grid
ax.grid(color='k', ls="--", lw=0.5)
# azimuth labels
ax.set_xticks(np.radians([0, 45, 90, 135, 180, 225, 270, 315]))
ax.set_xticklabels([
r"${0:d}^\circ$".format(int(np.ceil(x)))
for x in np.degrees(ax.get_xticks())
],
color='k')
#Zenith angle labels
ax.set_rlabel_position(250)
ax.set_yticks(np.radians([20, 40, 60, 80]))
ax.set_yticklabels([
r"${0:d}^\circ$".format(int(np.ceil(x)))
for x in np.degrees(ax.get_yticks())
],
color='k')
# Title
ax.set_title(
"MWA Tile beam (FEE)\naz = {0:.2f}, za = {1:.2f}, freq = {2:.2f}MHz\nobsid = {3}"
.format(pointing_AZ, pointing_ZA, freq, obs))
plt.legend(bbox_to_anchor=(0.95, -0.05), ncol=2)
#plt.savefig("{0}_{1:.2f}MHz_tile.eps".format(obs, freq), bbox_inches="tight", format="eps")
logger.info("Saving plot as output file: {0}_{1:.2f}MHz_tile.png".format(
obs, freq))
plt.savefig("{0}_{1:.2f}MHz_tile.png".format(obs, freq),
bbox_inches="tight")
