def prepare_file(self, file):
"""
Prepares the PSRFITS file in the correct format for the program.
"""
try:
hdul = fits.open(file)
except OSError:
return -1
name = hdul[0].header['SRC_NAME']
fe = hdul[0].header['FRONTEND']
if hdul[0].header[
'OBS_MODE'] != "PSR" or name != self.psr_name or fe != self.frontend:
hdul.close()
return -1
hdul.close()
ar = Archive(file, verbose=self.verbose)
ar.tscrunch(nsubint=1)
ar.fscrunch(nchan=1)
n = 1
nbin = ar.getNbin()
data = ar.getData()
return np.copy(data), n, nbin
def load_archive(file, tscrunch=False):
ar = Archive(file, verbose=False)
if tscrunch:
ar.tscrunch(nsubint=4)
#ar.imshow()
name = ar.getName()
mjd = int(ar.getMJD())
fe = ar.getFrontend()
nbin = ar.getNbin()
data = ar.getData().reshape((ar.getNchan() * ar.getNsubint(), nbin))
return name, mjd, fe, nbin, data
def get_Jy_per_count(dir, psr_cal_file, fitAA, fitBB):
file = dir + psr_cal_file
ar = Archive(file, verbose=False)
rfi = RFIMitigator(ar)
ar.tscrunch()
s_duty = ar.getValue("CAL_PHS")
duty = ar.getValue("CAL_DCYC")
nchan = ar.getNchan()
npol = ar.getNpol()
nbin = ar.getNbin()
BW = ar.getBandwidth()
data = ar.getData()
CTR_FREQ = ar.getCenterFrequency(weighted=True)
converted_data = IQUV_to_AABB(data, basis="cartesian")
frequencies = chan_to_freq(CTR_FREQ, BW, nchan)
psr_cal, high_psr, low_psr = np.zeros((2, nchan, nbin)), np.zeros(
(2, nchan)), np.zeros((2, nchan))
for i in np.arange(2):
for j in np.arange(nchan):
psr_cal[i][j], high_psr[i][j], low_psr[i][j] = prepare_cal_profile(
converted_data[0][i][j], s_duty, duty)
# Calculate jy_per_count{p, f}
jy_per_count_factor = np.zeros_like(high_psr)
# for i in np.arange( 2 ):
for j in np.arange(nchan):
jy_per_count_factor[0][j] = fitAA(frequencies[j]) / (
high_psr[0][j] - low_psr[0][j]) # A has units Jy / count
for j in np.arange(nchan):
jy_per_count_factor[1][j] = fitBB(frequencies[j]) / (
high_psr[1][j] - low_psr[1][j]) # A has units Jy / count
return jy_per_count_factor
def get_AABB_Fcal(dir, continuum_on, continuum_off, args, G=10.0, T0=1.0):
ON, OFF = dir + continuum_on, dir + continuum_off
if args.freq_zap is not None:
for i, arg in enumerate(args.freq_zap):
args.freq_zap[i] = int(args.freq_zap[i])
ar_on, ar_off = Archive(ON, verbose=False), Archive(OFF, verbose=False)
rfi_on, rfi_off = RFIMitigator(ar_on), RFIMitigator(ar_off)
s_duty_on, s_duty_off = ar_on.getValue("CAL_PHS"), ar_off.getValue(
"CAL_PHS")
duty_on, duty_off = ar_on.getValue("CAL_DCYC"), ar_off.getValue("CAL_DCYC")
nchan_on, nchan_off = ar_on.getNchan(), ar_off.getNchan()
npol_on, npol_off = ar_on.getNpol(), ar_off.getNpol()
nbin_on, nbin_off = ar_on.getNbin(), ar_off.getNbin()
BW_on, BW_off = ar_on.getBandwidth(), ar_off.getBandwidth()
CTR_FREQ_on, CTR_FREQ_off = ar_on.getCenterFrequency(
weighted=True), ar_off.getCenterFrequency(weighted=True)
ar_on.tscrunch()
ar_off.tscrunch()
if args.freq_zap is not None:
if len(args.freq_zap) == 1:
if args.channel_space:
rfi_on.zap_channels(args.freq_zap)
rfi_off.zap_channels(args.freq_zap)
else:
print(
"No zapping occurred (tried to zap channels in frequency space). Carrying on with calibration..."
)
elif len(args.freq_zap) == 2 and not args.channel_space:
rfi_on.zap_frequency_range(args.freq_zap[0], args.freq_zap[1])
rfi_off.zap_frequency_range(args.freq_zap[0], args.freq_zap[1])
else:
rfi_on.zap_channels(args.freq_zap)
rfi_off.zap_channels(args.freq_zap)
data_on, data_off = ar_on.getData(squeeze=True), ar_off.getData(
squeeze=True)
converted_data_on = IQUV_to_AABB(data_on, basis="cartesian")
converted_data_off = IQUV_to_AABB(data_off, basis="cartesian")
# Initialize the continuum data. SUBINT, POL,
continuum_on_source, high_on_mean, low_on_mean = np.zeros(
(2, nchan_on, nbin_on)), np.zeros((2, nchan_on)), np.zeros(
(2, nchan_on))
continuum_off_source, high_off_mean, low_off_mean = np.zeros(
(2, nchan_off, nbin_off)), np.zeros((2, nchan_off)), np.zeros(
(2, nchan_off))
f_on, f_off, C0 = np.zeros_like(high_on_mean), np.zeros_like(
high_off_mean), np.zeros_like(high_off_mean)
T_sys = np.zeros_like(C0)
F_cal = np.zeros_like(T_sys)
# Load the continuum data
for i in np.arange(2):
for j in np.arange(nchan_on):
continuum_on_source[i][j], high_on_mean[i][j], low_on_mean[i][
j] = prepare_cal_profile(converted_data_on[0][i][j], s_duty_on,
duty_on)
continuum_off_source[i][j], high_off_mean[i][j], low_off_mean[i][
j] = prepare_cal_profile(converted_data_off[0][i][j],
s_duty_off, duty_off)
f_on[i][j] = (high_on_mean[i][j] / low_on_mean[i][j]) - 1
f_off[i][j] = (high_off_mean[i][j] / low_off_mean[i][j]) - 1
if np.isnan(f_on[i][j]):
f_on[i][j] = 1
if np.isnan(f_on[i][j]):
f_off[i][j] = 1
C0[i][j] = T0 / ((1 / f_on[i][j]) - (1 / f_off[i][j]))
T_sys[i][j] = C0[i][j] / f_off[i][j]
F_cal[i][j] = (T_sys[i][j] *
f_off[i][j]) / G # F_cal has units Jy / cal
if np.isnan(F_cal[i][j]):
F_cal[i][j] = 0
frequencies_on_off = chan_to_freq(CTR_FREQ_on, BW_on, nchan_on)
f1, f2 = interp1d(frequencies_on_off,
F_cal[0],
kind='cubic',
fill_value='extrapolate'), interp1d(
frequencies_on_off,
F_cal[1],
kind='cubic',
fill_value='extrapolate')
return f1, f2
class Zap():
"""
Master class for zapping data.
Requires:
file - .FITS (must be PSRFITS v5+ format)
Optional:
template - ASCII format: BIN# Flux (Required if not doing NN exicison)
method - Either 'chauvenet', 'DMAD' or 'NN'
verbose - Prints more information to the console
**kwargs - Get parsed to plot.histogram_and_curves() or
"""
def __init__(self,
file,
template,
method='chauvenet',
nn_params=None,
verbose=False,
**kwargs):
self.file = file
if "cal" in self.file:
raise ValueError(f"File {self.file} is not in PSR format.")
elif "59071" in self.file:
raise ValueError(f"Not doing 59071...")
self.method = method
self.verbose = verbose
self.ar = Archive(file, verbose=False)
if method != 'NN':
_, self.template = u.get_data_from_asc(template)
self.opw = u.get_1D_OPW_mask(self.template, windowsize=128)
self.omit, self.rms_mu, self.rms_sigma = self.get_omission_matrix(
**kwargs)
unique, counts = np.unique(self.omit, return_counts=True)
print(f"Good channels: {100*(counts[0]/sum(counts)):.3f}%")
print(f"Bad channels: {100*(counts[1]/sum(counts)):.3f}%")
elif nn_params != None:
df = pd.DataFrame(
np.reshape(self.ar.getData(),
(self.ar.getNsubint() * self.ar.getNchan(),
self.ar.getNbin())))
scaler = MinMaxScaler()
scaled_df = scaler.fit_transform(df.iloc[:, :])
scaled_df = pd.DataFrame(scaled_df)
self.x = scaled_df.iloc[:, :].values.transpose()
self.nn = NeuralNet(self.x, np.array([[0], [0]]))
self.nn.dims = [self.ar.getNbin(), 512, 10, 13, 8, 6, 6, 4, 4, 1]
self.nn.threshold = 0.5
self.nn.load_params(root=nn_params)
self.omit = self.nn_get_omission()
np.set_printoptions(threshold=sys.maxsize)
unique, counts = np.unique(self.omit, return_counts=True)
print(f"Good channels: {100*(counts[0]/sum(counts)):.3f}%")
print(f"Bad channels: {100*(counts[1]/sum(counts)):.3f}%")
else:
sys.exit()
def nn_get_omission(self):
pred = np.around(np.squeeze(self.nn.pred_data(self.x, False)),
decimals=0).astype(np.int)
pred = np.reshape(pred, (self.ar.getNsubint(), self.ar.getNchan()))
return pred
def get_omission_matrix(self, **kwargs):
rms, lin_rms, mu, sigma = u.rms_arr_properties(
self.ar.getData(), self.opw, 1.0) # Needs to input 2D array
# Creates the histogram
plot.histogram_and_curves(
lin_rms,
mean=mu,
std_dev=sigma,
bins=(self.ar.getNchan() * self.ar.getNsubint()) // 4,
x_axis='Root Mean Squared',
y_axis='Frequency Density',
title=r'$M={},\ \sigma={}$'.format(mu, sigma),
**kwargs)
if self.method == 'chauvenet':
rej_arr = physics.chauvenet(rms,
median=mu,
std_dev=sigma,
threshold=2.0)
elif self.method == 'DMAD':
rej_arr = physics.DMAD(lin_rms, threshold=3.5)
rej_arr = np.reshape(rej_arr,
(self.ar.getNsubint(), self.ar.getNchan()))
if self.verbose:
print("Rejection criterion created.")
return rej_arr, mu, sigma
def plot_mask(self, **kwargs):
fig = plt.figure(figsize=(7, 7))
ax = fig.add_subplot(111)
ax.imshow(self.omit.T,
cmap=plt.cm.gray,
interpolation='nearest',
aspect='auto')
plt.show()
def save_training_set(self, val_size=0.2):
# From Chauvenet or DMAD. 1 is bad channel
with open(
f'{self.ar.getName()}_{int(self.ar.getMJD())}_{self.ar.getFrontend()}_{self.ar.getNbin()}.training',
'w') as t:
t.write(
f'# Training set for {self.ar.getName()} taken on {int(self.ar.getMJD())} at {self.ar.getFrontend()}\n'
)
with open(
f'{self.ar.getName()}_{int(self.ar.getMJD())}_{self.ar.getFrontend()}_{self.ar.getNbin()}.validation',
'w') as t:
t.write(
f'# Validation set for {self.ar.getName()} taken on {int(self.ar.getMJD())} at {self.ar.getFrontend()}\n'
)
ps_0 = np.zeros(2049)[np.newaxis, :]
ps_1 = np.zeros(2049)[np.newaxis, :]
d = self.ar.getData().reshape(
(self.ar.getNchan() * self.ar.getNsubint(), self.ar.getNbin()))
omission = self.omit.reshape(
(self.ar.getNchan() * self.ar.getNsubint()))
i = 1
for omit, profile in zip(omission, d):
try:
choice = int(omit)
if choice == 1:
choice = 0
elif choice == 0:
choice = 1
except ValueError:
choice = -1
print(i, end='\r')
if choice != -1:
# Creates the profile / choice pairs and doubles up with the reciprocal profiles.
p = np.append(profile, choice)
#inv_p = np.append( -1*profile, choice )
if choice == 0:
ps_0 = np.append(ps_0, p[np.newaxis, :], axis=0)
else:
ps_1 = np.append(ps_1, p[np.newaxis, :], axis=0)
i += 1
ps_0, ps_1 = np.delete(ps_0, 0, 0), np.delete(ps_1, 0, 0)
# Sort into training / validation sets
train, validation = train_test_split(ps_0, test_size=val_size)
ones_t, ones_v = train_test_split(ps_1, test_size=val_size)
train, validation = np.append(train, ones_t,
axis=0), np.append(validation,
ones_v,
axis=0)
np.random.shuffle(train), np.random.shuffle(validation)
for k in train:
with open(
f'{self.ar.getName()}_{int(self.ar.getMJD())}_{self.ar.getFrontend()}_{self.ar.getNbin()}.training',
'a') as t:
np.savetxt(t, k, fmt='%1.5f ', newline='')
t.write("\n")
#np.savetxt( t, inv_p, fmt = '%1.5f ', newline = '' )
#t.write( "\n" )
for k in validation:
with open(
f'{self.ar.getName()}_{int(self.ar.getMJD())}_{self.ar.getFrontend()}_{self.ar.getNbin()}.validation',
'a') as t:
np.savetxt(t, k, fmt='%1.5f ', newline='')
t.write("\n")
# Save as ASCII text file
def save(self, outroot="zap_out", ext='.ascii'):
outfile = outroot + ext
with open(outfile, 'w+') as f:
for i, t in enumerate(self.omit):
for j, rej in enumerate(t):
if rej == True:
f.write(str(i) + " " + str(self.ar.freq[i][j]) + "\n")
#f.write( f'{k} {self.ar.freq[k][i]}\n' )
return outfile
t.write(f'{n} {ar.freq[n][math.floor(event.ydata)]}\n')
def onclick(event):
print('%s click: button=%d, x=%d, y=%d, xdata=%f, ydata=%f' %
('double' if event.dblclick else 'single', event.button, event.x,
event.y, math.floor(event.xdata), math.floor(event.ydata)))
with open(f'Zap/zap_{file}.ascii', 'a+') as t:
t.write(
f'{math.floor(event.xdata)} {ar.freq[math.floor(event.xdata)][math.floor(event.ydata)]}\n'
)
print("Subints / 2 = ", (ar.getNsubint() // 2) + 1)
data = ar.getData()
mask = np.zeros(ar.getNbin())
np.set_printoptions(threshold=sys.maxsize)
mask[ar.opw] = 1
rms = np.array(calculate_rms_matrix(data, mask=mask))
rms_mean, rms_std = np.mean(rms), np.std(rms)
data_lin = np.array([])
sub_pol = ar.getNsubint() * ar.getNpol()
num_profs = sub_pol * ar.getNchan()
#for i in np.arange( ar.getNsubint() ):
# data_lin = np.append( data_lin, data[ i, 324, : ] )
#data = np.reshape( data, ( num_profs * ar.getNbin() ) )
# D_FAC = 32
# for i in range(D_FAC):
# st, ed = i*(chan // D_FAC), (i + 1)*(chan // D_FAC)