def test_find_t_sys_gain(): """ Tests the find_t_sys_gain function """ print("find_t_sys_gain") obsid_1=1223042480 pulsar_1= "J2234+2114" obsid_2=1222697776 pulsar_2= "J2330-2005" md_1 = mwa_metadb_utils.get_common_obs_metadata(obsid_1) #this is to speed up the tests md_2 = mwa_metadb_utils.get_common_obs_metadata(obsid_2) q_1 = psrqpy.QueryATNF(psrs=[pulsar_1], loadfromdb=ATNF_LOC) q_2 = psrqpy.QueryATNF(psrs=[pulsar_2], loadfromdb=ATNF_LOC) test_cases = [] test_cases.append((pulsar_1, obsid_1, 1223042887, 600, q_1, md_1,\ 325.17339020027805, 6.5034678040055613, 0.12547449408456718, 0.095633857616224477)) test_cases.append((pulsar_2, obsid_2, None, None, q_2, md_2,\ 295.73944858721245, 5.9147889717442492, 0.29127750035795497, 0.049367868667752078)) for pulsar, obsid, beg, t_int, query, metadata,\ exp_t_sys, exp_t_sys_err, exp_gain, exp_gain_err in test_cases: t_sys, t_sys_err, gain, gain_err = snfe.find_t_sys_gain(pulsar, obsid, beg=beg, t_int=t_int, query=query, obs_metadata=metadata, trcvr='database/MWA_Trcvr_tile_56.csv') assert_almost_equal(exp_t_sys, t_sys, decimal=6) assert_almost_equal(exp_t_sys_err, t_sys_err, decimal=6) assert_almost_equal(exp_gain, gain, decimal=6) assert_almost_equal(exp_gain_err, gain_err, decimal=6)
def multi_psr_snfe(pulsar_list, obsid,\ beg=None, end=None, obs_metadata=None, full_meta=None, plot_flux=False,\ query=None, min_z_power=0.3, trcvr=data_load.TRCVR_FILE): if obs_metadata is None or full_meta is None: logger.debug("Obtaining obs metadata") obs_metadata, full_meta = mwa_metadb_utils.get_common_obs_metadata( obsid, return_all=True) obs_beg, obs_end = mwa_metadb_utils.obs_max_min(obsid, meta=full_meta) if beg is None: beg = obs_beg if end is None: end = obs_end mega_query = psrqpy.QueryATNF(psrs=pulsar_list, loadfromdb=data_load.ATNF_LOC).pandas sn_dict = {} for i, pulsar in enumerate(mega_query["PSRJ"]): psr_query = {} for key in mega_query.keys(): psr_query[key] = [mega_query[key][i]] sn, sn_e = est_pulsar_sn(pulsar, obsid,\ beg=beg, end=end, obs_metadata=obs_metadata, full_meta=full_meta, plot_flux=plot_flux,\ query=psr_query, min_z_power=min_z_power, trcvr=trcvr) sn_dict[pulsar] = [sn, sn_e] return sn_dict
def get_obs_info(prof_path=None, obsid=None): #prof path is for bestrprofs only if obsid is None: if prof_path is not None: f = open(prof_path, "r") line = f.read() line = line.split() line = line[4].split("_") obsid = str(line[0]) f.close() #the return is in the format: #[obs, ra, dec, dura, [xdelays, ydelays], centrefreq, channels] else: logger.error("No obsid or profile path supplied. Cannot get metadata") system.exit(1) for _ in range(10): try: return get_common_obs_metadata(obsid) except RuntimeError as err: logger.warn("Error ocurred when trying to access metadata for obsid: {0}".format(obsid)) logger.warn("Here is the exception: {0}".format(err)) logger.warn("The metadata server may be down at the moment... Trying again in 30 seconds...") time.sleep(30) logger.error("Could not obtain metadata for obsid {0}".format(obsid)) logger.error("Exiting...") sys.exit(1)
def upload_file_to_db(obsid, pulsar, filepath, filetype, metadata=None, coh=True): """ Uploads a file to the pulsar database Parameters: ----------- obsid: str The observation ID pulsar: str The name of the pulsar filepath: str The file path of the file to upload filetype: int The type of file to upload: 1: Archive, 2: Timeseries, 3: Diagnostics, 4: Calibration Solution, 5: Bestprof metadata: list OPTINOAL - The output of mwa_metadb_utils.get_common_obs_metadata(obsid). If None, will generate it. coh: boolean OPTINOAL - Whether this is a coherent detection or not. Default: True """ if not metadata: metadata = get_common_obs_metadata(obsid) subbands = get_subbands(metadata) web_address, auth = get_db_auth_addr() client.detection_file_upload(web_address, auth, observationid = str(obsid), pulsar = pulsar, filetype = int(filetype), coherent = coh, subband = subbands, filepath = filepath)
def upload_formatted_file(filename, obsid, pulsar, bins, cal_id, filetype, name_info="", extension=""): """ Creates a new filename and uploads an archive file to the pulsar database if the same named file does not already exist on the database Parameters: ----------- archive: string The name of the archive file to upload obsid: int The observation ID pulsar: string Then name of the pulsar bins: int The Number of bins cal_id: int The calibrator ID filetype: int The type of file to upload: 1: Archive, 2: Timeseries, 3: Diagnostics, 4: Calibration Solution, 5: Bestprof name_info: str OPTIONAL - additional info to add to the name of the uploaded file. Default: '' extention: str OPTIONAL - The file extension of the uploaded file. Default: '' """ all_ftypes = std.get_filetypes_from_db(obsid, pulsar, filetype) fname_pref = std.filename_prefix(obsid, pulsar, bins=bins, cal=cal_id) upname = "{}".format(fname_pref) upname += name_info upname += extension metadata = get_common_obs_metadata(obsid) subbands = std.get_subbands(metadata) if os.path.basename(upname) not in all_ftypes: logger.info("Archive file not on databse. Uploading...") shutil.copy(filename, upname) std.upload_file_to_db(obsid, pulsar, upname, filetype, metadata=metadata, coh=True) os.remove(upname) else: logger.info("file on database. Not uploading")
'--fwhm_dec', type=float, help= 'Manualy give the declination FWHM in degrees instead of it estimating it from the array phase and frequency' ) args = parser.parse_args() # Set up plots fig = plt.figure() plt.rc("font", size=8) fig.add_subplot(111) ax = plt.axes() ax.axis('equal') # Get the fwhm of the observation meta_data = get_common_obs_metadata(args.obsid) channels = meta_data[-1] oap = get_obs_array_phase(args.obsid) if oap == "OTH": #Assume it's phase 2 extended array oap = "P2E" centrefreq = 1.28 * float(min(channels) + max(channels)) / 2. fwhm = calc_ta_fwhm(centrefreq, array_phase=oap) print("Observation ID: {}".format(args.obsid)) print("FWHM: {} deg".format(fwhm)) detections = [] if args.bestprof_dir: for bestprof_file in glob.glob("{}/*bestprof".format( args.bestprof_dir)): with open(bestprof_file, "r") as bestprof:
def est_pulsar_sn(pulsar, obsid,\ beg=None, end=None, p_ra=None, p_dec=None, obs_metadata=None, plot_flux=False,\ query=None, o_enter=None, o_exit=None, trcvr="/group/mwaops/PULSAR/MWA_Trcvr_tile_56.csv"): """ Estimates the signal to noise ratio for a pulsar in a given observation using the radiometer equation S/N = (s_mean * gain * sqrt(n_p * t_int * df * (period - W_50)/W_50)) / t_sys Note that W_50 should be W_equiv but we can't figure that out so we're estimating Parameters: ---------- pulsar: string Name of the pulsar e.g. J2241-5236 obsid: int Observation ID e.g. 1226406800 beg: int OPTIONAL - beginning of the observing time end: int OPTIONAL - end of the observing time p_ra: str OPTIONAL - the target's right ascension p_dec: str OPTIONAL - the target's declination obs_metadata: list OPTIONAL - the array generated from mwa_metadb_utils.get_common_obs_metadata(obsid) plot_flux: boolean OPTIONAL - whether or not to produce a plot of the flux estimation. Default = False o_enter: float OPTIONAL - The normalised o_enter time of the pulsar's coverage in the beam (between 0 and 1) o_exit: float OPTIONAL - The normalised o_exit time of the pulsar's covreage in the beam (between 0 and 1) Returns: -------- sn: float The expected signal to noise ratio for the given inputs sn_err: float The uncertainty in the signal to noise ratio """ #We will attain uncertainties for s_mean, gain, t_sys and W_50. # other uncertainties are considered negligible if query is None: query = psrqpy.QueryATNF(psrs=pulsar, loadfromdb=ATNF_LOC).pandas if p_ra is None or p_dec is None: #Get some basic pulsar and obs info info p_ra = query["RAJ"][0] p_dec = query["DECJ"][0] #get metadata if not supplied if obs_metadata is None: logger.debug("Obtaining obs metadata") obs_metadata = mwa_metadb_utils.get_common_obs_metadata(obsid) n_p = 2 #constant df = 30.72e6 #(24*1.28e6) #estimate flux s_mean, s_mean_err = est_pulsar_flux(pulsar, obsid, plot_flux=plot_flux,\ metadata=obs_metadata, query=query) #fluxes may be Nones. If so, return None if s_mean is None and s_mean_err is None: return None, None #find integration time if o_enter is not None and o_exit is not None: t_int = o_exit - o_enter if beg is not None and end is not None: t_int = t_int * (end - beg) else: t_int = t_int * obs_metadata[3] #duration else: beg, end, t_int = find_times(obsid, pulsar, beg=beg, end=end) if t_int <= 0.: logger.warning( "Pulsar not in beam for obs files or specificed beginning and end times" ) return 0., 0. #find system temp and gain t_sys, t_sys_err, gain, gain_err = find_t_sys_gain(pulsar, obsid,\ beg=beg, p_ra=p_ra, p_dec=p_dec, query=query,\ obs_metadata=obs_metadata, trcvr=trcvr) #Find W_50 W_50, W_50_err = find_pulsar_w50(pulsar, query=query) #calculate SN period = float(query["P0"][0]) SN = ((s_mean * gain) / t_sys) * np.sqrt(n_p * t_int * df * (period - W_50) / W_50) #Calculate SN uncertainty using variance formula. Assuming error from period, df and t_int is zero dc_expr = np.sqrt((period - W_50) / W_50) var_s_mean = (gain * np.sqrt(n_p * t_int * df)) * dc_expr / t_sys var_gain = s_mean * np.sqrt(n_p * t_int * df) * dc_expr / t_sys var_W_50 = s_mean * gain * np.sqrt( n_p * t_int * df) / t_sys * (period / (-2. * W_50**2.)) * dc_expr**-1 var_t_sys = -s_mean * gain * np.sqrt( n_p * t_int * df) * dc_expr / t_sys**2. var_s_mean = var_s_mean**2. * s_mean_err**2. var_gain = var_gain**2. * gain_err**2. var_W_50 = var_W_50**2. * W_50_err**2. var_t_sys = var_t_sys**2. * t_sys_err**2. logger.debug("variance estimates for s_mean: {0}, gain: {1}, W_50: {2}, t_sys: {3}"\ .format(var_s_mean, var_gain, var_W_50, var_t_sys)) SN_err = np.sqrt(var_s_mean + var_gain + var_W_50 + var_t_sys) logger.debug("S_mean: {0} +/- {1}".format(s_mean, s_mean_err)) logger.debug("Gain: {0} +/- {1}".format(gain, gain_err)) logger.debug("t_int: {0}".format(t_int)) logger.debug("df: {0}".format(df)) logger.debug("period: {0}".format(period)) logger.debug("W_50: {0} +/- {1}".format(W_50, W_50_err)) logger.debug("t_sys: {0} +/- {1}".format(t_sys, t_sys_err)) return SN, SN_err
def find_t_sys_gain(pulsar, obsid, beg=None, t_int=None, p_ra=None, p_dec=None,\ obs_metadata=None, query=None, trcvr="/group/mwaops/PULSAR/MWA_Trcvr_tile_56.csv"): """ Finds the system temperature and gain for an observation. A function snippet originally written by Nick Swainston - adapted for general VCS use. Parameters: ----------- pulsar: str the J name of the pulsar. e.g. J2241-5236 obsid: int The observation ID. e.g. 1226406800 beg: int The beginning of the observing time t_int: float The total time that the target is in the beam p_ra: str OPTIONAL - the target's right ascension p_dec: str OPTIONAL - the target's declination obs_metadata: list OPTIONAL - the array generated from mwa_metadb_utils.get_common_obs_metadata(obsid) query: object OPTIONAL - The return of the psrqpy function for this pulsar trcvr: str The location of the MWA receiver temp csv file. Default = '/group/mwaops/PULSAR/MWA_Trcvr_tile_56.csv' Returns: -------- t_sys: float The system temperature t_sys_err: float The system temperature's uncertainty gain: float The system gain gain_err: float The gain's uncertainty """ #get ra and dec if not supplied if p_ra is None or p_dec is None and query is None: logger.debug("Obtaining pulsar RA and Dec from ATNF") query = psrqpy.QueryATNF(psrs=[pulsar], loadfromdb=ATNF_LOC).pandas p_ra = query["RAJ"][0] p_dec = query["DECJ"][0] elif query is not None: query = psrqpy.QueryATNF(psrs=[pulsar], loadfromdb=ATNF_LOC).pandas p_ra = query["RAJ"][0] p_dec = query["DECJ"][0] #get metadata if not supplied if obs_metadata is None: logger.debug("Obtaining obs metadata") obs_metadata = mwa_metadb_utils.get_common_obs_metadata(obsid) obsid, obs_ra, obs_dec, _, delays, centrefreq, channels = obs_metadata #get beg if not supplied if beg is None or t_int is None: logger.debug("Calculating beginning time for pulsar coverage") beg, _, t_int = find_times(obsid, pulsar, beg=beg) #Find 'start_time' for fpio - it's usually about 7 seconds #obs_start, _ = mwa_metadb_utils.obs_max_min(obsid) start_time = beg - int(obsid) #Get important info trec_table = Table.read(trcvr, format="csv") ntiles = 128 #TODO actually we excluded some tiles during beamforming, so we'll need to account for that here beam_power = fpio.get_beam_power_over_time([obsid, obs_ra, obs_dec, t_int, delays,\ centrefreq, channels],\ np.array([[pulsar, p_ra, p_dec]]),\ dt=100, start_time=start_time) beam_power = np.mean(beam_power) # Usa a primary beam function to convolve the sky temperature with the primary beam # (prints suppressed) sys.stdout = open(os.devnull, 'w') _, _, Tsky_XX, _, _, _, Tsky_YY, _ = pbtant.make_primarybeammap( int(obsid), delays, centrefreq * 1e6, 'analytic', plottype='None') sys.stdout = sys.__stdout__ #TODO can be inaccurate for coherent but is too difficult to simulate t_sky = (Tsky_XX + Tsky_YY) / 2. # Get T_sys by adding Trec and Tsky (other temperatures are assumed to be negligible t_sys_table = t_sky + submit_to_database.get_Trec(trec_table, centrefreq) t_sys = np.mean(t_sys_table) t_sys_err = t_sys * 0.02 #TODO: figure out what t_sys error is logger.debug("pul_ra: {} pul_dec: {}".format(p_ra, p_dec)) _, _, zas = mwa_metadb_utils.mwa_alt_az_za(obsid, ra=p_ra, dec=p_dec) theta = np.radians(zas) gain = submit_to_database.from_power_to_gain(beam_power, centrefreq * 1e6, ntiles, coh=True) logger.debug("beam_power: {} theta: {} pi: {}".format( beam_power, theta, np.pi)) gain_err = gain * ((1. - beam_power) * 0.12 + 2. * (theta / (0.5 * np.pi))**2. + 0.1) # Removed the below error catch because couldn't find an obs that breaks it #sometimes gain_err is a numpy array and sometimes it isnt so i have to to this... #try: # gain_err.shape # gain_err = gain_err[0] #except: # pass return t_sys, t_sys_err, gain, gain_err
def est_pulsar_flux(pulsar, obsid, plot_flux=False, metadata=None, query=None): """ Estimates a pulsar's flux from archival data by assuming a power law relation between flux and frequency. Frist tries to attain a apectral index from the ATNF database. If this fails, try to work out a spectrla index. If this fails, uses an index of -1.4 with uncertainty of 1. Parameters: ----------- pulsar: string The puslar's name. e.g. 'J2241-5236' obsid: int The observation ID plot_flux: boolean OPTIONAL - Whether or not to make a plot of the flux estimation. Default = False metadata: list OPTIONAL - The metadata call for this obsid query: object OPTIONAL - The return from psrqpy.QueryATNF for this pulsar Returns: ------- flux: float The estimated flux in Jy flux_err: float The estimated flux's uncertainty in Jy """ if metadata is None: logger.debug("obtaining mean freq from obs metadata") metadata = mwa_metadb_utils.get_common_obs_metadata(obsid) f_mean = metadata[5] * 1e6 if query is None: query = psrqpy.QueryATNF(psrs=[pulsar], loadfromdb=ATNF_LOC).pandas flux_queries = ["S40", "S50", "S60", "S80", "S100", "S150", "S200",\ "S300", "S400", "S600", "S700", "S800", "S900",\ "S1400", "S1600", "S2000", "S3000", "S4000", "S5000",\ "S6000", "S8000"] freq_all = [] flux_all = [] flux_err_all = [] #Get all available data from dataframe and check for missing values for flux_query in flux_queries: flux = query[flux_query][0] if not np.isnan(flux): #sometimes error values don't exist, causing a key error in pandas try: flux_err = query[flux_query + "_ERR"][0] if flux_err == 0.0: logger.warning( "Flux error for query: {0}, pulsar {1}, is zero. Assuming 20% uncertainty" .format(query[flux_query][0], pulsar)) flux_err = flux * 0.2 except KeyError: logger.warning( "flux error value for {0}, pulsar {1}, not available. assuming 20% uncertainty" .format(query[flux_query][0], pulsar)) flux_err = flux * 0.2 if np.isnan(flux_err): logger.warning( "flux error value for {0}, pulsar {1}, not available. assuming 20% uncertainty" .format(query[flux_query][0], pulsar)) flux_err = flux * 0.2 freq_all.append(int(flux_query.split()[0][1:]) * 1e6) #convert to Hz flux_all.append(flux * 1e-3) #convert to Jy flux_err_all.append(flux_err * 1e-3) #convert to Jy #Also get spectral index if it exists spind = query["SPINDX"][0] spind_err = query["SPINDX_ERR"][0] logger.debug("Freqs: {0}".format(freq_all)) logger.debug("Fluxes: {0}".format(flux_all)) logger.debug("Flux Errors: {0}".format(flux_err_all)) logger.info( "There are {0} flux values available on the ATNF database for {1}". format(len(flux_all), pulsar)) #Attempt to estimate flux if len(flux_all) > 1: logger.info("Fitting power law to archive data") for i, _ in enumerate(flux_all): flux_all[i] = flux_all[i] flux_err_all[i] = flux_err_all[i] #apply spectral index bounds if an error already exists initial_spind = -1.5 spind_bounds = None if not np.isnan(spind): initial_spind = spind if not np.isnan(spind_err): #if spind_error exists on cat, use it to create 5 sigma bounds for fitting spind_bounds = [spind - spind_err * 5, spind + spind_err * 5] spind, c, covar_matrix = fit_plaw_psr(freq_all, flux_all, alpha_initial=initial_spind, alpha_bound=spind_bounds) logger.info("Derived spectral index: {0} +/- {1}".format( spind, covar_matrix.item(0))) #plot if in debug mode if plot_flux == True: plot_flux_estimation(freq_all, flux_all, flux_err_all, pulsar, obsid, alpha=spind, c=c) #flux calc. flux_est = np.exp(spind * np.log(f_mean) + c) #calculate error from covariance matrix a_mat = np.matrix([np.log(f_mean), 1]) log_flux_err = np.sqrt(a_mat * covar_matrix * a_mat.T) #to find the error, we take the average log error in linear space b = np.exp(log_flux_err) flux_est_err = flux_est / 2. * (b - (1 / b)) flux_est_err = flux_est_err.item(0) #Do something different if there is only one flux value in archive elif len(flux_all) == 1: logger.warning("Only a single flux value available on the archive") if not np.isnan(spind) and np.isnan(spind_err): logger.warning( "Spectral index error not available. Assuming 20% error") spind_err = spind * 0.2 if np.isnan(spind): logger.warning( "Insufficient archival data to estimate spectral index. Using alpha=-1.4 +/- 1.0 as per Bates2013" ) spind = -1.4 spind_err = 1. #formula for flux: #S_1 = nu_1^a * nu_2 ^-a * S_2 nu_1 = f_mean #in Hz nu_2 = freq_all[0] #in Hz s_2 = flux_all[0] #in Jy s_2_err = flux_err_all[0] a = spind a_err = spind_err logger.debug("nu1 {0}".format(nu_1)) logger.debug("nu2 {0}".format(nu_2)) logger.debug("s2 {0}".format(s_2)) logger.info( "calculating flux using spectral index: {0} and error: {1}".format( spind, spind_err)) flux_est = nu_1**a * nu_2**(-a) * s_2 #variance formula error est s_2_var = nu_1**a * nu_2**(-a) a_var = s_2 * nu_1**a * nu_2**(-a) * (np.log(nu_1) - np.log(nu_2)) a_var = a_var**2 * a_err**2 s_2_var = s_2_var**2 * s_2_err**2 flux_est_err = np.sqrt(s_2_var + a_var) elif len(flux_all) < 1: logger.warning( "No flux values on archive for {0}. Cannot estimate flux. Will return Nones" .format(pulsar)) return None, None logger.info("Source flux estimate at {0} MHz: {1} +/- {2} Jy".format( f_mean / 1e6, flux_est, flux_est_err)) return flux_est, flux_est_err
pulsar_name, pul_ra, pul_dec = pulsar_ra_dec[0] else: logger.info('Congratulations you have detected ' + pulsar + ' for the first time with the MWA') #gets Ra and DEC from PSRCAT pulsar_ra_dec = fpio.get_psrcat_ra_dec(pulsar_list=[pulsar]) pulsar_name, pul_ra, pul_dec = pulsar_ra_dec[0] #then adds it to the database client.pulsar_create(web_address, auth, name = pulsar, ra = pul_ra, dec = pul_dec) #get meta data from obsid metadata = get_common_obs_metadata(obsid) obsid,ra_obs,dec_obs,time_obs,delays,centrefreq,channels = metadata minfreq = float(min(channels)) maxfreq = float(max(channels)) bandwidth = 30720000. #In Hz #bandwidth = 30720000. / 12. #In Hz #calc obstype if (maxfreq - minfreq) == 23: obstype = 1 else: obstype = 2 if args.bestprof or args.ascii:
def plot_beam_pattern(obsid, obsfreq, obstime, ra, dec, cutoff=0.1): # extra imports from MWA_Tools to access database and beam models from mwa_pb import primary_beam as pb _az = np.linspace(0, 360, 3600) _za = np.linspace(0, 90, 900) az, za = np.meshgrid(_az, _za) ## TARGET TRACKING ## times = Time(obstime, format='gps') target = SkyCoord(ra, dec, unit=(u.hourangle, u.deg)) location = EarthLocation(lat=-26.7033 * u.deg, lon=116.671 * u.deg, height=377.827 * u.m) altaz = target.transform_to(AltAz(obstime=times, location=location)) targetAZ = altaz.az.deg targetZA = 90 - altaz.alt.deg colours = cm.viridis(np.linspace(1, 0, len(targetAZ))) ## MWA BEAM CALCULATIONS ## delays = get_common_obs_metadata(obsid)[ 4] # x-pol and y-pol delays for obsid _, ptAZ, ptZA = mwa_alt_az_za(obsid) #ptAZ, _, ptZA = dbq.get_beam_pointing(obsid) # Az and ZA in degrees for obsid logger.info("obs pointing: ({0}, {1})".format(ptAZ, ptZA)) xp, yp = pb.MWA_Tile_analytic(np.radians(za), np.radians(az), obsfreq * 1e6, delays=delays, zenithnorm=True) #interp=True) pattern = np.sqrt((xp + yp) / 2.0).real # sum to get "total intensity" logger.debug("Pattern: {0}".format(pattern)) pmax = pattern.max() hpp = 0.5 * pmax # half-power point logger.info("tile pattern maximum: {0:.3f}".format(pmax)) logger.info("tile pattern half-max: {0:.3f}".format(hpp)) pattern[np.where(pattern < cutoff)] = 0 # ignore everything below cutoff # figure out the fwhm fwhm_idx = np.where((pattern > 0.498 * pmax) & (pattern < 0.502 * pmax)) fwhm_az_idx = fwhm_idx[1] fwhm_za_idx = fwhm_idx[0] # collapse beam pattern along axes pattern_ZAcol = pattern.mean(axis=0) pattern_AZcol = pattern.mean(axis=1) # figure out beam pattern value at target tracking points track_lines = [] logger.info("beam power at target track points:") for ta, tz in zip(targetAZ, targetZA): xp, yp = pb.MWA_Tile_analytic(np.radians([[tz]]), np.radians([[ta]]), obsfreq * 1e6, delays=delays, zenithnorm=True) #interp=False bp = (xp + yp) / 2 track_lines.append(bp[0]) logger.info("({0:.2f},{1:.2f}) = {2:.3f}".format(ta, tz, bp[0][0])) ## PLOTTING ## fig = plt.figure(figsize=(10, 8)) gs = gridspec.GridSpec(4, 4) axP = plt.subplot(gs[1:, 0:3]) axAZ = plt.subplot(gs[0, :3]) axZA = plt.subplot(gs[1:, 3]) axtxt = plt.subplot(gs[0, 3]) # info text in right-hand corner axis axtxt.axis('off') infostr = """Obs ID: {0} Frequency: {1:.2f}MHz Beam Pmax: {2:.3f} Beam half-Pmax: {3:.3f} """.format(obsid, obsfreq, pmax, hpp) axtxt.text(0.01, 0.5, infostr, verticalalignment='center') logger.debug("az: {0}, za: {1}, pattern: {2}, pmax: {3}".format( az, za, pattern, pmax)) # plot the actual beam patter over sky p = axP.contourf(az, za, pattern, 100, cmap=plt.get_cmap('gist_yarg'), vmax=pmax) # plot beam contours axP.scatter(_az[fwhm_az_idx], _za[fwhm_za_idx], marker='.', s=1, color='r') # plot the fwhm border axP.plot(ptAZ, ptZA, marker="+", ms=8, ls="", color='C0') # plot the tile beam pointing for ta, tz, c in zip(targetAZ, targetZA, colours): axP.plot(ta, tz, marker="x", ms=8, ls="", color=c) # plot the target track through the beam axP.set_xlim(0, 360) axP.set_ylim(0, 90) axP.set_xticks(np.arange(0, 361, 60)) axP.set_xlabel("Azimuth (deg)") axP.set_ylabel("Zenith angle (deg)") # setup and configure colourbar cbar_ax = fig.add_axes([0.122, -0.01, 0.58, 0.03]) cbar = plt.colorbar(p, cax=cbar_ax, orientation='horizontal', label="Zenith normalised power") cbar.ax.plot(hpp / pmax, [0.5], 'r.') for l, c in zip(track_lines, colours): cbar.ax.plot([l / pmax, l / pmax], [0, 1], color=c, lw=1.5) # plot collapsed beam patterns: axAZ.plot(_az, pattern_ZAcol, color='k') # collapsed along ZA for ta, c in zip(targetAZ, colours): # draw tracking points axAZ.axvline(ta, color=c) axAZ.set_xlim(0, 360) axAZ.set_xticks(np.arange(0, 361, 60)) axAZ.set_xticklabels([]) axAZ.set_yscale('log') axZA.plot(pattern_AZcol, _za, color='k') # collapsed along AZ for tz, c in zip(targetZA, colours): # draw tracking points axZA.axhline(tz, color=c) axZA.set_ylim(0, 90) axZA.set_yticklabels([]) axZA.set_xscale('log') plt.savefig("{0}_{1:.2f}MHz_flattile.png".format(obsid, obsfreq), bbox_inches='tight')
def find_sources_in_obs(obsid_list, names_ra_dec, obs_for_source=False, dt_input=100, beam='analytic', min_power=0.3, cal_check=False, all_volt=False, degrees_check=False): """ Either creates text files for each MWA obs ID of each source within it or a text file for each source with each MWA obs is that the source is in. Args: obsid_list: list of MWA obs IDs names_ra_dec: [[source_name, ra, dec]] dt: the time step in seconds to do power calculations beam: beam simulation type ['analytic', 'advanced', 'full_EE'] min_power: if above the minium power assumes it's in the beam cal_check: checks the MWA pulsar database if there is a calibration for the obsid all_volt: Use all voltages observations including some inital test data with incorrect formats degrees_check: if false ra and dec is in hms, if true in degrees Output [output_data, obsid_meta]: output_data: The format of output_data is dependant on obs_for_source. If obs_for_source is True: output_data = {jname:[[obsid, duration, enter, exit, max_power], [obsid, duration, enter, exit, max_power]]} If obs_for_source is False: ouput_data = {obsid:[[jname, enter, exit, max_power], [jname, enter, exit, max_power]]} obsid_meta: a list of the output of get_common_obs_metadata for each obsid """ #prepares metadata calls and calculates power powers = [] #powers[obsid][source][time][freq] obsid_meta = [] obsid_to_remove = [] for obsid in obsid_list: beam_meta_data, full_meta = get_common_obs_metadata(obsid, return_all=True) #beam_meta_data = obsid,ra_obs,dec_obs,time_obs,delays,centrefreq,channels if dt_input * 4 > beam_meta_data[3]: # If the observation time is very short then a smaller dt time is required # to get enough ower imformation dt = int(beam_meta_data[3] / 4.) else: dt = dt_input logger.debug("obsid: {0}, time_obs {1} s, dt {2} s".format( obsid, beam_meta_data[3], dt)) #check for raw volatge files filedata = full_meta[u'files'] keys = filedata.keys() check = False for k in keys: if '.dat' in k: #TODO check if is still robust check = True if check or all_volt: powers.append( get_beam_power_over_time(beam_meta_data, names_ra_dec, dt=dt, centeronly=True, verbose=False, option=beam, degrees=degrees_check)) obsid_meta.append(beam_meta_data) else: logger.warning('No raw voltage files for %s' % obsid) obsid_to_remove.append(obsid) for otr in obsid_to_remove: obsid_list.remove(otr) #chooses whether to list the source in each obs or the obs for each source output_data = {} if obs_for_source: for sn, source in enumerate(names_ra_dec): source_data = [] for on, obsid in enumerate(obsid_list): source_ob_power = powers[on][sn] if max(source_ob_power) > min_power: duration = obsid_meta[on][3] logger.debug( "Running beam_enter_exit on obsid: {}".format(obsid)) enter, exit = beam_enter_exit(source_ob_power, duration, dt=dt, min_power=min_power) if enter is not None: source_data.append([ obsid, duration, enter, exit, max(source_ob_power)[0] ]) # For each source make a dictionary key that contains a list of # lists of the data for each obsid output_data[source[0]] = source_data else: #output a list of sorces for each obs for on, obsid in enumerate(obsid_list): duration = obsid_meta[on][3] obsid_data = [] for sn, source in enumerate(names_ra_dec): source_ob_power = powers[on][sn] if max(source_ob_power) > min_power: enter, exit = beam_enter_exit(source_ob_power, duration, dt=dt, min_power=min_power) obsid_data.append( [source[0], enter, exit, max(source_ob_power)[0]]) # For each obsid make a dictionary key that contains a list of # lists of the data for each source/pulsar output_data[obsid] = obsid_data return output_data, obsid_meta
def pulsar_beam_coverage(obsid, pulsar, beg=None, end=None, metadata=None, full_meta=None, ondisk=False, min_z_power=0.3, query=None): """ Finds the normalised time that a pulsar is in the beam for a given obsid If pulsar is not in beam, returns None, None Parameters: ----------- obsid: int The observation ID pulsar: string The pulsar's J name beg: int OPTIONAL - The beginning of the processed observing time in gps time end: int OPTIONAL - The end of the processed observing time in gps time obs_beg: int OPTIONAL - The beginning of the observation in gps time obs_end: int OPTIONAL - The end of the observation in gps time ondisk: boolean Whether to use files that are on-disk for beginning and end times. Default=False Returns: -------- enter_files: float A float between 0 and 1 that describes the normalised time that the pulsar enters the beam exit_files: float a float between 0 and 1 that describes the normalised time that the pulsar exits the beam """ if not metadata or not full_meta: metadata, full_meta = mwa_metadb_utils.get_common_obs_metadata( obsid, return_all=True) #Find the beginning and end of obs obs_beg, obs_end = files_beg, files_end = mwa_metadb_utils.obs_max_min( obsid, meta=full_meta) obs_dur = obs_end - obs_beg + 1 #Logic loop: if ondisk == True: #find the beginning and end time of the observation FILES you have on disk files_beg, files_end = process_vcs.find_combined_beg_end(obsid) files_duration = files_end - files_beg + 1 elif beg is None and end is None: logger.warning( "ondisk==False so beg and end can not be None. Returning Nones") return None, None else: #uses manually input beginning and end times to find beam coverage files_beg = beg files_end = end files_duration = files_end - files_beg + 1 #find the enter and exit times of pulsar normalized with the observing time names_ra_dec = fpio.grab_source_alog(pulsar_list=[pulsar], query=query) beam_source_data, _ = fpio.find_sources_in_obs( [obsid], names_ra_dec, min_power=min_z_power, metadata_list=[[metadata, full_meta]]) if beam_source_data[obsid]: enter_obs_norm = beam_source_data[obsid][0][1] exit_obs_norm = beam_source_data[obsid][0][2] else: logger.warn("{} not in beam".format(pulsar)) return None, None #times the source enters and exits beam time_enter = obs_beg + obs_dur * enter_obs_norm - 1 time_exit = obs_beg + obs_dur * exit_obs_norm - 1 #normalised time the source enters/exits the beam in the files enter_files = (time_enter - files_beg) / files_duration exit_files = (time_exit - files_beg) / files_duration if enter_files < 0.: enter_files = 0. if exit_files > 1.: exit_files = 1. if enter_files > 1. or exit_files < 0.: logger.warning( "source {0} is not in the beam for the files on disk".format( pulsar)) enter_files = None exit_files = None return enter_files, exit_files
args = parser.parse_args() opts_string = "" for k in args.__dict__: if args.__dict__[k] is not None: if k == "pointing": opts_string = opts_string + ' --' + str(k) + ' "' + str(args.__dict__[k][0]) +\ ' ' + str(args.__dict__[k][1]) + '"' else: opts_string = opts_string + ' --' + str(k) + ' ' + str( args.__dict__[k]) if args.obsid: obs, ra, dec, duration, xdelays, centrefreq, channels = \ meta.get_common_obs_metadata(args.obsid) #get fwhm in radians centre_fwhm = np.radians(args.deg_fwhm) #all_pointing parsing if (args.loop != 1) and args.all_pointings: print( "Can't use --loop and --all_poinitings as all_pointings calculates the " "loops required. Exiting.") quit() if args.pointing and args.all_pointings: print( "Can't use --pointing and --all_poinntings as --all_pointings calculates " "the pointing. Exiting.") quit()
def plotSkyMap(obsfile, targetfile, oname, show_psrcat=False, show_mwa_sky=False, show_mwa_unique=False): fig = plt.figure() ax = fig.add_subplot(111, projection="mollweide") mwa_dec_lim = 30 mwa_only_dec_lim = -50 if show_mwa_sky: # Make a patch that is transformable through axis projection # and highlight the visible sky for the MWA Path = mpath.Path ra_start = -np.pi ra_end = np.pi dec_start = -np.pi / 2 dec_end = np.radians(mwa_dec_lim) path_data = [ (Path.MOVETO, (ra_start, dec_start)), (Path.LINETO, (ra_start, dec_end)), (Path.LINETO, (ra_end, dec_end)), (Path.LINETO, (ra_end, dec_start)), (Path.CLOSEPOLY, (ra_end, dec_start)), ] codes, verts = zip(*path_data) path = mpath.Path(verts, codes) patch = mpatches.PathPatch(path, lw=0, ec="lightskyblue", fc="lightskyblue", label="All MWA sky") ax.add_patch(patch) if show_mwa_unique: # Make a patch that is transformable through axis projection # and highlight the part of the sky ONLY visible to the MWA ra_start = -np.pi ra_end = np.pi dec_start = -np.pi / 2 dec_end = np.radians(mwa_only_dec_lim) path_data = [ (Path.MOVETO, (ra_start, dec_start)), (Path.LINETO, (ra_start, dec_end)), (Path.LINETO, (ra_end, dec_end)), (Path.LINETO, (ra_end, dec_start)), (Path.CLOSEPOLY, (ra_end, dec_start)), ] codes, verts = zip(*path_data) path = mpath.Path(verts, codes) patch = mpatches.PathPatch(path, lw=0, ec="lightgreen", fc="lightgreen", label="Unique MWA sky") ax.add_patch(patch) if show_psrcat: # Retrieve the local installed PSRCAT catalogue and plot those pulsar positions cmd = 'psrcat -o short_csv -nocand -nonumber -c "Jname RAJ DECJ" | sed "2d" > psrcat.csv' subprocess.call(cmd, shell=True) psrcat_coords = read_data("psrcat.csv", delim=";") os.remove("psrcat.csv") # Create a mask for pulsar in and out of the declination limit of the MWA maskGood = psrcat_coords.dec.wrap_at(180*u.deg).deg < mwa_dec_lim maskBad = psrcat_coords.dec.wrap_at(180*u.deg).deg >= mwa_dec_lim psrcat_ra_good = -psrcat_coords.ra.wrap_at(180*u.deg).rad[maskGood] # negative because RA increases to the West psrcat_dec_good = psrcat_coords.dec.wrap_at(180*u.deg).rad[maskGood] psrcat_ra_bad = -psrcat_coords.ra.wrap_at(180*u.deg).rad[maskBad] # negative because RA increases to the West psrcat_dec_bad = psrcat_coords.dec.wrap_at(180*u.deg).rad[maskBad] # Now plot the pulsar locations ax.scatter(psrcat_ra_good, psrcat_dec_good, 0.01, marker="x", color="0.4", zorder=1.4) ax.scatter(psrcat_ra_bad, psrcat_dec_bad, 0.01, marker="x", color="0.8", zorder=1.4) # Calculate beam patterns and plot contours levels = np.arange(0.25, 1., 0.05) cmap = plt.get_cmap("cubehelix_r") obsids = read_data(obsfile, coords=False)["OBSID"] for obsid in obsids: #print "Accessing database for observation: {0}".format(obsid) logger.info("Accessing database for observation: {0}".format(obsid)) # TODO: I think this process is now implemented in a function in mwa_metadb_utils, need to double check beam_meta_data = getmeta(service='obs', params={'obs_id': obsid}) ra = beam_meta_data[u'metadata'][u'ra_pointing'] dec = beam_meta_data[u'metadata'][u'dec_pointing'] duration = beam_meta_data[u'stoptime'] - beam_meta_data[u'starttime'] #gps time delays = beam_meta_data[u'rfstreams'][u'0'][u'xdelays'] minfreq = float(min(beam_meta_data[u'rfstreams'][u"0"][u'frequencies'])) maxfreq = float(max(beam_meta_data[u'rfstreams'][u"0"][u'frequencies'])) centrefreq = 1.28e6 * (minfreq + (maxfreq-minfreq)/2) #in MHz channels = beam_meta_data[u'rfstreams'][u"0"][u'frequencies'] metadata = [obsid, ra, dec, duration, delays, centrefreq, channels] beam_meta_data, full_meta = get_common_obs_metadata(obsid, return_all = True) # Create a meshgrid over which to iterate Dec = [] ; RA = [] map_dec_range = np.arange(-89, 90, 3) map_ra_range = np.arange(0, 360, 3) for i in map_dec_range: for j in map_ra_range: Dec.append(i) RA.append(j) RADecIndex = np.arange(len(RA)) names_ra_dec = np.column_stack((RADecIndex, RA, Dec)) #print "Creating beam patterns..." time_intervals = 600 # seconds powout = get_beam_power_over_time(beam_meta_data, names_ra_dec, dt=time_intervals, degrees=True) z=[] ; x=[] ; y=[] for c in range(len(RA)): temppower = 0. for t in range(powout.shape[1]): power_ra = powout[c,t,0] if power_ra > temppower: temppower = power_ra z.append(temppower) if RA[c] > 180: x.append(-RA[c]/180.*np.pi+2*np.pi) else: x.append(-RA[c]/180.*np.pi) y.append(Dec[c]/180.*np.pi) nx=np.array(x) ; ny=np.array(y); nz=np.array(z) #print "Plotting beam pattern contours..." logger.info("Plotting beam pattern contours...") # Set vmin and vmax to ensure the beam patterns are on the same color scale # and plot the beam pattern contours on the map #c = ax.tricontour(wrapped_ra, wrapped_dec, final_pattern_power, levels=levels, cmap=cmap, vmin=levels.min(), vmax=levels.max(), zorder=1.3) c = ax.tricontour(nx, ny, nz, levels=levels, cmap=cmap, vmin=levels.min(), vmax=levels.max(), zorder=1.3) # Make a figure color scale based on the contour sets (which all have the same max/min values fig.colorbar(c, fraction=0.02, pad=0.03, label="Zenith normalised power") # Plot the target positions, using Astropy to wrap at correct RA target_coords = read_data(targetfile) wrapped_target_ra = -target_coords.ra.wrap_at(180*u.deg).rad # negative because RA increases to the West wrapped_target_dec = target_coords.dec.wrap_at(180*u.deg).rad #print "Plotting target source positions..." logger.info("Plotting target source positions...") ax.scatter(wrapped_target_ra, wrapped_target_dec, 10, marker="x", color="red", zorder=1.6, label="Target sources") xtick_labels = ["10h", "8h", "6h", "4h", "2h", "0h", "22h", "20h", "18h", "16h", "14h"] ax.set_xticklabels(xtick_labels, color="0.2") ax.set_xlabel("Right Ascension") ax.set_ylabel("Declination") ax.grid(True, color="k", lw=0.5, ls=":") # Place upper-right corner of legend at specified Axis coordinates ax.legend(loc="upper right", bbox_to_anchor=(1.02, 0.08), numpoints=1, borderaxespad=0., fontsize=6) plt.savefig(oname, format="eps", bbox_inches="tight")
def set_freq_from_metadata(self, obsid): self.freq = get_common_obs_metadata(obsid)[5]
def est_pulsar_flux(pulsar, obsid, plot_flux=False, metadata=None, query=None): """ Estimates a pulsar's flux from archival data by assuming a power law relation between flux and frequency Parameters: ----------- pulsar: string The puslar's name. e.g. 'J2241-5236' obsid: int The observation ID plot_flux: boolean OPTIONAL - Whether or not to make a plot of the flux estimation. Default = False metadata: list OPTIONAL - The metadata call for this obsid query: object OPTIONAL - The return from psrqpy.QueryATNF for this pulsar Returns: ------- flux: float The estimated flux in Jy flux_err: float The estimated flux's uncertainty in Jy """ if metadata is None: logger.debug("obtaining mean freq from obs metadata") metadata = mwa_metadb_utils.get_common_obs_metadata(obsid) f_mean = metadata[5] * 1e6 freq_all, flux_all, flux_err_all, spind, spind_err = flux_from_atnf( pulsar, query=query) logger.debug("Freqs: {0}".format(freq_all)) logger.debug("Fluxes: {0}".format(flux_all)) logger.debug("Flux Errors: {0}".format(flux_err_all)) logger.info("{0} there are {1} flux values available on the ATNF database"\ .format(pulsar, len(flux_all))) if not spind or spind_err: spind, spind_err, K, covar_mat = find_spind(pulsar, freq_all, flux_all, flux_err_all) if K and covar_mat is not None and spind: flux_est, flux_est_err = flux_from_plaw(f_mean, K, spind, covar_mat) elif spind and spind_err: flux_est, flux_est_err = flux_from_spind(f_mean, freq_all[0], flux_all[0], flux_err_all[0],\ spind, spind_err) else: logger.warning( "{} no flux values on archive. Cannot estimate flux. Will return Nones" .format(pulsar)) return None, None if plot_flux == True: plot_flux_estimation(pulsar, freq_all, flux_all, flux_err_all, spind, my_nu=f_mean, my_S=flux_est, my_S_e=flux_est_err, obsid=obsid, a_err=spind_err, K=K, covar_mat=covar_mat) logger.info("{0} flux estimate at {1} MHz: {2} +/- {3} Jy"\ .format(pulsar, f_mean/1e6, flux_est, flux_est_err)) return flux_est, flux_est_err
lst = str( first_gps_time.sidereal_time('apparent').to_string(unit=u.hour, sep=":")) lst_sec = float( first_gps_time.sidereal_time('apparent').to_string( unit=u.hour, decimal=True)) * 3600.0 logger.debug("LST = {0}".format(lst)) logger.debug("LST seconds = {0}".format(lst_sec)) mjd = first_gps_time.mjd mjd_trunc = int(mjd) logger.debug("MJD = {0}".format(mjd)) logger.info("Getting observation metadata") metadata = get_common_obs_metadata(args.obsID) logger.info("Organising channels") channels = metadata[-1] nchans = len(channels) ch_offset = channels[-1] - channels[0] if ch_offset != nchans - 1: logger.error( "Picket fence observation - cannot combine picket fence incoherent sum data (with this script)" ) sys.exit(1) bw = nchans * 1.28 cfreq = (1.28 * min(channels) - 0.64) + bw / 2.0 logger.info("Centre frequency: {0} MHz".format(cfreq))
def find_times(obsid, pulsar, beg=None, end=None, metadata=None, full_meta=None, min_z_power=0.3, query=None): """ Find the total integration time of a pulsar in the primary beam of an obsid Parameters: ---------- obsid: int The observation ID pulsar: string The J name of the pulsar beg: int OPTIONAL - The beginning of the processed observing time end: int OPTIONAL - The end of the processed observing time Returns: ------- enter_time: int The time when the pulsar enters the beam in gps exit_time: int The time when the pulsar exits the beam in gps t_int: int The total time that the pulsar is in the beam in seconds """ #type assurances obsid = int(obsid) if not metadata or not full_meta: metadata, full_meta = mwa_metadb_utils.get_common_obs_metadata( obsid, return_all=True) obs_beg, obs_end = mwa_metadb_utils.obs_max_min(obsid, meta=full_meta) t_int = None if beg is None or end is None: logger.info("Using duration for entire observation") beg = obs_beg end = obs_end dur = end - beg + 1 enter_norm, exit_norm = pulsar_beam_coverage(obsid, pulsar, beg=beg, end=end, metadata=metadata, full_meta=full_meta, min_z_power=min_z_power, query=query) enter_time = beg + enter_norm * dur exit_time = beg + exit_norm * dur t_int = (exit_norm - enter_norm) * dur if t_int is None: enter_norm, exit_norm = pulsar_beam_coverage(obsid, pulsar, beg=beg, end=end, metadata=metadata, full_meta=full_meta, min_z_power=min_z_power, query=query) if beg is not None and end is not None: dur = end - beg else: #use entire obs duration beg = obs_beg end = obs_end dur = end - beg + 1 if enter_norm is not None and exit_norm is not None: enter_time = beg + enter_norm * dur exit_time = beg + exit_norm * dur t_int = dur * (exit_norm - enter_norm) else: logger.warn("Integration time calculation failed") enter_time = 0 exit_time = 0 t_int = 0 return enter_time, exit_time, t_int
for i, ob in enumerate(observations): obs_foun_check = False for omf in obsid_meta_file: if int(ob) == int(omf[0]): print("getting obs metadata from obs_meta.csv") ob, ra, dec, time, delays,centrefreq, channels = omf ob = int(ob) time = int(time) delays = map(int,delays[1:-1].split(",")) centrefreq = float(centrefreq) channels = map(int,channels[1:-1].split(",")) obs_foun_check = True if not obs_foun_check: print("Getting metadata for {}".format(ob)) ob, ra, dec, time, delays,centrefreq, channels =\ meta.get_common_obs_metadata(ob) with open('obs_meta.csv', 'a') as csvfile: spamwriter = csv.writer(csvfile) spamwriter.writerow([ob, ra, dec, time, delays,centrefreq, channels]) cord = [ob, ra, dec, time, delays,centrefreq, channels] ra_list.append(ra) dec_list.append(dec) delays_list.append(delays) #Loop over observations and calc beam power for i, ob in enumerate(observations): ra = ra_list[i] dec = dec_list[i] delays = delays_list[i]
from numpy.testing import assert_almost_equal import mwa_metadb_utils import psrqpy from vcstools import data_load import sn_flux_est as snfe import logging logger = logging.getLogger(__name__) #Global obsid information to speed things up md_dict = {} obsid_list = [1223042480, 1222697776, 1226062160, 1225713560, 1117643248] for obs in obsid_list: md_dict[str(obs)] = mwa_metadb_utils.get_common_obs_metadata( obs, return_all=True) def test_pulsar_beam_coverage(): """ Tests the pulsar_beam_coverage funtion """ print("test_pulsar_beam_coverage") obsid = 1223042480 pulsar = "J2234+2114" test_cases = [] test_cases.append((1223042487, 1223042587, 0.0, 1.0)) test_none_cases = [] test_none_cases.append((None, None, 0.0, 0.072731816627943369)) test_none_cases.append((-1e10, -1e4, 0.0, 0.072731816627943369))