def test_plotting_doesnt_cause_exceptions():
""" Try running the plotting routines. They should not raise expections even without X windows """
a = bl.Waterfall(voyager_h5)
b = bl.Waterfall(voyager_fil)
a.plot_all()
plt.clf()
a.plot_kurtosis()
plt.clf()
a.plot_spectrum()
plt.clf()
a.plot_spectrum_min_max()
plt.clf()
a.plot_waterfall()
plt.clf()
a.plot_time_series()
plt.clf()
b.plot_all()
plt.clf()
b.plot_kurtosis()
plt.clf()
b.plot_spectrum()
plt.clf()
b.plot_spectrum_min_max()
plt.clf()
b.plot_waterfall()
plt.clf()
b.plot_time_series()
plt.clf()
def test_waterfall_data_load_range_freq():
fw = bl.Waterfall('Voyager_data/Voyager1.single_coarse.fine_res.fil',
f_start=8419.24,
f_stop=8419.35)
hw = bl.Waterfall('Voyager_data/Voyager1.single_coarse.fine_res.h5',
f_start=8419.24,
f_stop=8419.35)
print(fw.data.shape)
print(hw.data.shape)
print(hw.data[0].max(), hw.data[0].argmax())
print(fw.data[0].max(), fw.data[0].argmax())
print(hw.data[-1].max(), hw.data[-1].argmax())
print(fw.data[-1].max(), fw.data[-1].argmax())
# Assert data is loaded to the same shape and has same values
assert hw.data.shape == fw.data.shape == (16, 1, 39370)
assert np.allclose(hw.data, fw.data)
# Check the Voyager carrier has the known amplitudes at first and last integration
assert np.allclose(hw.data[0].max(), fw.data[0].max(), 3.09333e+11)
assert np.allclose(hw.data[-1].max(), fw.data[-1].max(), 2.74257e+11)
# Check the tone is in the same bin for both
assert hw.data[0].argmax() == fw.data[0].argmax() == 18959
assert hw.data[-1].argmax() == fw.data[-1].argmax() == 18996
# And plot(
plt.subplot(2, 1, 1)
fw.plot_spectrum()
plt.subplot(2, 1, 2)
hw.plot_spectrum()
plt.tight_layout()
def compare_waterfall_fil_to_h5():
""" Load Voyager dataset and test that both fil and hdf5 readers return same headers and data """
print("Loading FIL and HDF5 data with Waterfall()..."),
a = bl.Waterfall(voyager_h5)
b = bl.Waterfall(voyager_fil)
print("OK")
print("Reading headers..")
print("\nHDF5 file header:")
pprint(a.header)
print("\nFIL file header:")
pprint(b.header)
print("Headers are loading OK")
print("\nChecking header values match..."),
for key in b.header.keys():
assert b.header[key] == a.header[key]
print("OK")
print("Checking datatype matches..."),
assert a.data.dtype == b.data.dtype
print("OK")
print("Checking data matches..."),
assert np.allclose(a.data, b.data)
assert a.data.dtype == b.data.dtype
print("OK")
def test_fil_write():
try:
a = bl.Waterfall(voyager_h5)
b = bl.Waterfall(voyager_fil)
a.write_to_fil('test.fil')
b.write_to_fil('test2.fil')
c = bl.Waterfall('test.fil')
d = bl.Waterfall('test2.fil')
for key in a.header.keys():
if key != 'DIMENSION_LABELS':
assert a.header[key] == c.header[key]
assert key in c.header.keys()
assert a.header[key] == d.header[key]
assert key in d.header.keys()
assert np.allclose(a.data, c.data)
assert np.allclose(a.data, d.data)
except AssertionError:
print(key, a.header[key], b.header[key], c.header[key], d.header[key])
raise
finally:
os.remove('test.fil')
os.remove('test2.fil')
def plot_candidate_events(candidate_event_dataframe,
fil_file_list,
filter_level,
source_name_list,
offset=0,
**kwargs):
#load in the data for each individual hit
for i in range(0, len(candidate_event_dataframe)):
candidate = candidate_event_dataframe.iloc[i]
on_source_name = candidate['Source']
f_mid = candidate['Freq']
drift_rate = candidate['DriftRate']
#calculate the length of the total cadence from the fil files' headers
first_fil = bl.Waterfall(fil_file_list[0], load_data=False)
tfirst = first_fil.header['tstart']
last_fil = bl.Waterfall(fil_file_list[-1], load_data=False)
tlast = last_fil.header['tstart']
t_elapsed = Time(tlast, format='mjd').unix - Time(
tfirst, format='mjd').unix + (last_fil.n_ints_in_file -
1) * last_fil.header['tsamp']
#calculate the width of the plot based on making sure the full drift is visible
bandwidth = 2.4 * abs(drift_rate) / 1e6 * t_elapsed
bandwidth = np.max((bandwidth, 500. / 1e6))
#Get start and stop frequencies based on midpoint and bandwidth
f_start, f_stop = np.sort(
(f_mid - (bandwidth / 2), f_mid + (bandwidth / 2)))
#Print useful values
print('')
print('*************************************************')
print('*** The Parameters for This Plot Are: ****')
print('Target = ', on_source_name)
print('Bandwidth = ', round(bandwidth, 5), ' MHz')
print('Time Elapsed (inc. Slew) = ', round(t_elapsed), ' s')
print('Middle Frequency = ', round(f_mid, 4), " MHz")
print('Expected Drift = ', round(drift_rate, 4), " Hz/s")
print('*************************************************')
print('*************************************************')
print('')
#Pass info to make_waterfall_plots() function
make_waterfall_plots(fil_file_list,
on_source_name,
f_start,
f_stop,
drift_rate,
f_mid,
filter_level,
source_name_list,
bandwidth,
offset=offset,
**kwargs)
return
def compare_waterfall_fil_to_h5_methods_and_attributes():
""" Compare attributes and check methods """
a = bl.Waterfall(voyager_h5)
b = bl.Waterfall(voyager_fil)
print("Comparing attributes of classes match where expected")
assert a.beam_axis == b.beam_axis
assert a.freq_axis == b.freq_axis
assert a.time_axis == b.time_axis
#assert a.stokes_axis == b.stokes_axis # Fil shouldn't have stokes axis ...
assert a.calc_n_coarse_chan() == b.calc_n_coarse_chan()
assert a.file_shape == b.file_shape
assert a.n_channels_in_file == b.n_channels_in_file
assert a.n_ints_in_file == b.n_ints_in_file
assert a.selection_shape == b.selection_shape
print("Checking if basic methods run without raising Exceptions")
# Check they can be run
a.container.populate_freqs()
a.container.populate_timestamps()
a.info()
a.blank_dc(1)
a.calibrate_band_pass_N1()
b.container.populate_freqs()
b.container.populate_timestamps()
b.info()
b.blank_dc(1)
b.calibrate_band_pass_N1()
# TODO: 'compute_lsrk' -- need PYSLALIB
# TODO: 'compute_lst' -- need PYSLALIB
# Expected to differ between: 'ext', 'file_size_bytes', 'filename', 'container'
# Unused? Inherited from Filterbank.py 'gen_from_header', 'generate_freqs', 'read_filterbank', 'read_hdf5'
# TODO: 'grab_data',
# TODO: 'read_data',
# TODO: 'write_to_fil',
# TODO: 'write_to_filterbank',
# TODO: 'write_to_hdf5']
dir_a = dir(a)
dir_b = dir(b)
print("Attr/methods in HDF5 but not in FIL:")
for item in dir_a:
if item not in dir_b:
print(item)
print("Attr/methods in FIL but not in HDF5:")
for item in dir_b:
if item not in dir_a:
print(item)
def test_read_fns():
""" These read functions are currently not implemented. """
a = bl.Waterfall(voyager_fil)
b = bl.Waterfall(voyager_h5)
with pytest.raises(NotImplementedError):
a.container.read_all()
a.container.read_row(0)
a.container.read_rows(0, 2)
b.container.read_all()
b.container.read_row(0)
b.container.read_rows(0, 2)
def test_read_fns():
""" These read functions are currently not implemented. """
a = bl.Waterfall('Voyager_data/Voyager1.single_coarse.fine_res.fil')
b = bl.Waterfall('Voyager_data/Voyager1.single_coarse.fine_res.h5')
with pytest.raises(NotImplementedError):
a.container.read_all()
a.container.read_row(0)
a.container.read_rows(0, 2)
b.container.read_all()
b.container.read_row(0)
b.container.read_rows(0, 2)
def test_get_freqs():
wf = bl.Waterfall(voyager_h5)
freqs = wf.container.populate_freqs()
sum1 = np.sum(freqs)
freqs = wf.get_freqs()
sum2 = np.sum(freqs)
assert sum1 == sum2
wf = bl.Waterfall(voyager_fil)
freqs = wf.container.populate_freqs()
sum1 = np.sum(freqs)
freqs = wf.get_freqs()
sum2 = np.sum(freqs)
assert sum1 == sum2
def test_read_fns():
""" These read functions are currently not implemented. """
""" L: if that is true, why do the first two lines
NOT raise NotImplementedError's?"""
a = bl.Waterfall(voyager_fil)
b = bl.Waterfall(voyager_h5)
with pytest.raises(NotImplementedError):
a.container.read_all()
a.container.read_row(0)
a.container.read_rows(0, 2)
b.container.read_all()
b.container.read_row(0)
b.container.read_rows(0, 2)
def plot_hit(fil_filename, dat_filename, hit_id, bw=None, offset=0):
""" Plot a candidate from a .dat file
Args:
fil_filename (str): Path to filterbank file to plot
dat_filename (str): Path to turbosSETI generated .dat output file of events
hit_id (int): ID of hit in the dat file to plot (TopHitNum)
offset (float): Offset drift line on plot. Default 0.
"""
# Load hit details
dat = make_table(dat_filename)
hit = dat.iloc[hit_id]
f0 = hit['Freq']
if bw is None:
bw_mhz = np.abs(hit['FreqStart'] - hit['FreqEnd'])
else:
bw_mhz = bw * 1e-6
fil = bl.Waterfall(fil_filename,
f_start=f0 - bw_mhz / 2,
f_stop=f0 + bw_mhz / 2)
t_duration = (fil.n_ints_in_file - 1) * fil.header['tsamp']
fil.plot_waterfall()
overlay_drift(f0, hit['DriftRate'], t_duration, offset)
def test_plot_waterfall_classmethod():
""" Load Voyager dataset and test plotting """
a = bl.Waterfall(voyager_h5)
plt.figure("TEST PLOTTING CLASS", figsize=(10, 8))
plt.subplot(3, 2, 1)
a.plot_waterfall()
plt.subplot(3, 2, 2)
a.plot_spectrum()
plt.subplot(3, 2, 3)
a.plot_spectrum_min_max()
plt.subplot(3, 2, 4)
a.plot_kurtosis()
plt.subplot(3, 2, 5)
a.plot_time_series()
plt.tight_layout()
plt.savefig("test_plotting_classmethod.png")
plt.figure("TEST PLOT_ALL CLASS", figsize=(10, 8))
a.plot_all()
plt.savefig("test_plotting_plot_all_classmethod.png")
def test_plot_waterfall():
""" Load Voyager dataset and test plotting """
a = bl.Waterfall(voyager_h5)
plt.figure("TEST PLOTTING", figsize=(10, 8))
plt.subplot(3, 2, 1)
plot_waterfall(a)
plt.subplot(3, 2, 2)
plot_spectrum(a)
plt.subplot(3, 2, 3)
plot_spectrum_min_max(a)
plt.subplot(3, 2, 4)
plot_kurtosis(a)
plt.subplot(3, 2, 5)
plot_time_series(a)
plt.tight_layout()
plt.savefig("test_plotting.png")
plt.figure("TEST PLOT_ALL", figsize=(10, 8))
plot_all(a)
plt.savefig("test_plotting_plot_all.png")
def test_grab_data_works_across_all_fil_h5():
fw = bl.Waterfall(voyager_fil)
hw = bl.Waterfall(voyager_h5)
all_readers = [fw, hw]
for ii, rr in enumerate(all_readers):
f, d = rr.grab_data(f_start=8419.29, f_stop=8419.30)
print(f.shape, d.shape)
assert f.shape == (3580,)
assert d.shape == (16, 3580)
for ii, rr in enumerate(all_readers):
f, d = rr.grab_data(f_start=8419.29685, f_stop=8419.2971)
print(f.shape, d.shape)
assert f.shape == (91,)
assert d.shape == (16, 91)
def compare_waterfall_fil_h5_conatiners():
""" Compare the two containers for fil.container and h5.container """
a = bl.Waterfall(voyager_h5)
b = bl.Waterfall(voyager_fil)
dir_a = dir(a.container)
dir_b = dir(b.container)
print("Attr/methods in HDF5 container but not in FIL:")
for item in dir_a:
if item not in dir_b:
print(item)
print("Attr/methods in FIL container but not in HDF5:")
for item in dir_b:
if item not in dir_a:
print(item)
def test_grab_data_works_across_all_fil_h5():
ff = bl.Filterbank('Voyager_data/Voyager1.single_coarse.fine_res.fil')
hf = bl.Filterbank('Voyager_data/Voyager1.single_coarse.fine_res.h5')
fw = bl.Waterfall('Voyager_data/Voyager1.single_coarse.fine_res.fil')
hw = bl.Waterfall('Voyager_data/Voyager1.single_coarse.fine_res.h5')
all_readers = [ff, hf, fw, hw]
for ii, rr in enumerate(all_readers):
f, d = rr.grab_data(f_start=8419.29, f_stop=8419.30)
print(f.shape, d.shape)
assert f.shape == (3580, )
assert d.shape == (16, 3580)
for ii, rr in enumerate(all_readers):
f, d = rr.grab_data(f_start=8419.29685, f_stop=8419.2971)
print(f.shape, d.shape)
assert f.shape == (91, )
assert d.shape == (16, 91)
def grab_parameters(dat_file, GBT_band, use_defaults, h5_fil_path):
"""
takes dat file of GBT data and returns frequency parameters
used to calculate where the DC spikes will be
Arguments
----------
dat_file : str
filepath to the .dat file
GBT_band : str
the band at which the data was collected
choose from {"L", "S", "C", "X"}
use_defaults : bool
if True, the program uses the preset,
hardcoded fch1 and foff values; if False,
fch1 and foff are retrieved from the h5/fil
files.
h5_fil_path : str
the file path to the h5 or fil file
corresponding to the given dat file.
Program will crash if use_defaults=False
and no h5_fil_path is specified.
Returns : fch1, foff, nfpc
which will be used internally within remove_DC_spike
"""
tbl = find.read_dat(dat_file)
if use_defaults:
if GBT_band == "L":
fch1 = 1926.2695326677515 # LBAND -- this is hardcoded, it would be nice to fix that
if GBT_band == "C":
fch1 = 8201.66015625 # CBAND ""
if GBT_band == "S":
fch1 = 2720.80078125 # SBAND ""
if GBT_band == "X":
fch1 = 11102.05078125 # XBAND ""
foff = float(tbl["DELTAF"][0]) * 1e-6
else:
fil = bp.Waterfall(h5_fil_path)
fch1 = fil.header['fch1']
foff = fil.header['foff']
#for Karen's files
fch1 = float(fch1 - (foff / 2.0))
nfpc = (1500.0 / 512.0) / abs(foff)
num_course_channels = np.max(tbl["CoarseChanNum"])
#print('fch1: {0}, foff: {1}'.format(fch1, foff))
return fch1, foff, nfpc, num_course_channels
def test_info():
a = bl.Waterfall(voyager_h5)
print(a)
a.info()
a.blank_dc(n_coarse_chan=1)
a.calibrate_band_pass_N1()
'Below: orphaned functions. That is a little strange.'
a.grab_data()
a.read_data()
'It is VERY strange for internal functions to be orphaned...'
a._get_chunk_dimensions()
# plenty of missed if suites
a._get_blob_dimensions((300, 300, 300, 300))
a._update_header()
def test_h5_io(frame_setup_no_data):
frame = copy.deepcopy(frame_setup_no_data)
fil_fn = 'temp.h5'
frame.save_hdf5(fil_fn)
temp_frame = stg.Frame(waterfall=fil_fn)
assert_allclose(temp_frame.get_data(), frame.get_data())
wf = bl.Waterfall(fil_fn)
temp_frame = stg.Frame(waterfall=wf)
assert_allclose(temp_frame.get_data(), frame.get_data())
os.remove(fil_fn)
def test_info():
a = bl.Waterfall(voyager_h5)
print(a)
a.info()
a.blank_dc(n_coarse_chan=1)
a.calibrate_band_pass_N1()
# Below: orphaned functions (not used anywhere else).
# That is a little strange...
a.grab_data()
a.read_data()
# It is VERY strange for INTERNAL functions to be orphaned!
a._get_chunk_dimensions()
# plenty of missed if suites
a._get_blob_dimensions((300, 300, 300, 300))
a._update_header()
def calc_max_load(arg_path, verbose=False):
r''' Calculate the max_load parameter value for a subsequent Waterfall instantiation.
Algorithm:
* A = minimum Waterfall object size.
* B = data array size within one polarisation.
* Return ceil(A + B in GB)
'''
wf = bl.Waterfall(arg_path, load_data=False)
min_size = float(sys.getsizeof(wf.header)) + float(sys.getsizeof(wf))
data_size = float(
wf.header['nchans'] * wf.n_ints_in_file * wf.header['nbits']) / 8.0
ngbytes = (min_size + data_size) / 1e9
max_load = np.ceil(ngbytes)
if verbose:
print(
'calc_max_load: Waterfall object size excluding data = {}, data array size = {}, total GBs = {:.1f}'
.format(min_size, data_size, ngbytes))
return max_load
def test_plotting():
""" Some basic plotting tests
TODO: Improve these tests (and the functions for that matter!
"""
filename_fil = os.path.join(HERE, 'Voyager1.single_coarse.fine_res.h5')
fil = bl.Waterfall(filename_fil)
# Test make_waterfall_plots -- needs 6x files
filenames_list = [filename_fil] * 6
target = 'Voyager'
drates = [-0.392226]
fvals = [8419.274785]
f_start = 8419.274374 - 600e-6
f_stop = 8419.274374 + 600e-6
node_string = 'test'
filter_level = 1
plot_event.make_waterfall_plots(filenames_list, target, drates, fvals, f_start, f_stop, node_string, filter_level)
plt.show()
def make_waterfall_plots(file_list,
plot_dir,
f_start=None,
f_stop=None,
**kwargs):
r"""
Make waterfall plots of a file set, view from top to bottom.
Parameters
----------
file_list : str
List of filterbank files to plot in a stacked mode.
f_start : float
Start frequency, in MHz.
f_stop : float
Stop frequency, in MHz.
source_name_list : list
List of source names in the set, in order.
kwargs : dict
Keyword args to be passed to matplotlib imshow().
"""
# prepare for plotting
matplotlib.rc("font", **font)
# set up the sub-plots
n_plots = len(file_list)
fig = plt.subplots(n_plots,
sharex=True,
sharey=True,
figsize=(10, 2 * n_plots))
# get directory path for storing PNG files
if plot_dir is None:
dirpath = dirname(abspath(file_list[0])) + "/"
else:
if not isdir(plot_dir):
os.mkdir(plot_dir)
dirpath = plot_dir
# read in data for the first panel
max_load = bl.calcload.calc_max_load(file_list[0])
#print("plot_event make_waterfall_plots: max_load={} is required for {}".format(max_load, file_list[0]))
wf1 = bl.Waterfall(file_list[0], max_load=max_load)
t0 = wf1.header["tstart"]
source_name = wf1.header["source_name"]
# Fix frequency boundaries if required.
freqs = wf1.container.populate_freqs()
if f_start is None:
f_start = freqs[0]
if f_stop is None:
f_stop = freqs[-1]
the_lowest, the_highest = sort2(f_start, f_stop)
# rebin data to plot correctly with fewer points
plot_f1, plot_data1 = wf1.grab_data(f_start=the_lowest, f_stop=the_highest)
dec_fac_x, dec_fac_y = 1, 1
if plot_data1.shape[0] > MAX_IMSHOW_POINTS[0]:
dec_fac_x = plot_data1.shape[0] / MAX_IMSHOW_POINTS[0]
if plot_data1.shape[1] > MAX_IMSHOW_POINTS[1]:
dec_fac_y = int(np.ceil(plot_data1.shape[1] / MAX_IMSHOW_POINTS[1]))
plot_data1 = rebin(plot_data1, dec_fac_x, dec_fac_y)
# Compute the midpoint.
the_midpoint = np.abs(the_lowest + the_highest) / 2.
# Protect RAM utilisation.
del wf1, freqs, plot_f1, plot_data1
gc.collect()
# Fill in each subplot for the full plot
subplots = []
for ii, filename in enumerate(file_list):
logger.debug("make_waterfall_plots: file {} in list: {}".format(
ii, filename))
# identify panel
subplot = plt.subplot(n_plots, 1, ii + 1)
subplots.append(subplot)
# read in data
max_load = bl.calcload.calc_max_load(filename)
wf = bl.Waterfall(filename, max_load=max_load)
# Validate frequency range.
freqs = wf.container.populate_freqs()
ii_lowest, ii_highest = sort2(freqs[0], freqs[-1])
logger.info("Processing: {}, freq lowest={}, highest={}".format(
filename, ii_lowest, ii_highest))
if the_lowest < ii_lowest or the_highest > ii_highest:
logger.warning(
"Frequency range not compatible! Ignoring this file.")
# Protect RAM utilisation.
del wf, freqs
gc.collect()
continue # Skip this file.
# make plot with plot_waterfall
source_name = wf.header["source_name"]
this_plot = plot_waterfall(wf,
f_start=the_lowest,
f_stop=the_highest,
**kwargs)
# Title the full plot if processing the first file.
if ii == 0:
plot_title = "%s \n MJD:%5.5f (first file)" % (source_name, t0)
plt.title(plot_title)
# Format full plot.
if ii < len(file_list) - 1:
plt.xticks(np.linspace(the_lowest, the_highest, num=4),
["", "", "", ""])
# Protect RAM utilisation.
del wf, freqs
gc.collect()
# More overall plot formatting, axis labelling.
factor = 1e6
units = "Hz"
xloc = np.linspace(the_lowest, the_highest, 5)
xticks = [round(loc_freq) for loc_freq in (xloc - the_midpoint) * factor]
if np.max(xticks) > 1000:
xticks = [xt / 1000 for xt in xticks]
units = "kHz"
plt.xticks(xloc, xticks)
plt.xlabel("Relative Frequency [%s] from %f MHz" % (units, the_midpoint),
fontdict=font)
# Add colorbar.
cax = fig[0].add_axes([0.94, 0.11, 0.03, 0.77])
fig[0].colorbar(this_plot,
cax=cax,
label="Normalized Power (Arbitrary Units)")
# Adjust plots
plt.subplots_adjust(hspace=0, wspace=0)
# Save the figures.
path_png = dirpath + source_name + "_fstart_{:0.6f}".format(f_start) \
+ "_fstop_{:0.6f}".format(f_stop) + ".png"
plt.savefig(path_png, bbox_inches="tight")
logger.info("Saved plot: {}".format(path_png))
# show figure before closing if this is an interactive context
mplbe = matplotlib.get_backend()
logger.debug("make_waterfall_plots: backend = {}".format(mplbe))
if mplbe != "agg":
plt.show()
# close all figure windows
plt.close("all")
def plot_candidate_events(candidate_event_dataframe,
fil_file_list,
filter_level,
source_name_list,
offset=0,
plot_snr_list=False,
**kwargs):
r'''
Calls :func:`~make_waterfall_plots` on each event in the input .csv file.
Arguments
---------
candidate_event_dataframe : dict
A pandas dataframe containing information
about a candidate event. The necessary data
includes the start and stop frequencies, the
drift rate, and the source name. To determine
the required variable names and formatting
conventions, see the output of
find_event_pipeline.
fil_file_list : list
A Python list that contains a series of
strings corresponding to the filenames of .fil
files, each on a new line, that corresponds to
the cadence used to create the .csv file used
for event_csv_string.
filter_level : int
A string indicating the filter level of the
cadence used to generate the
candidate_event_dataframe. Used only for
output file naming, convention is "f1", "f2",
or "f3". Descriptions for the three levels of
filtering can be found in the documentation
for find_event.py
source_name_list : list
A Python list that contains a series of strings
corresponding to the source names of the
cadence in chronological (descending through
the plot panels) cadence.
offset : int, optional
The amount that the overdrawn "best guess"
line from the event parameters in the csv
should be shifted from its original position
to enhance readability. Can be set to 0
(default; draws line on top of estimated
event) or 'auto' (shifts line to the left by
an auto-calculated amount, with addition lines
showing original position).
plot_snr_list : bool (*** NOT YET IN USE***)
kwargs : dict
Examples
--------
It is highly recommended that users interact with this program via the
front-facing plot_event_pipeline.py script. See the usage of that file in
its own documentation.
If you would like to run plot_candidate_events without calling
plot_event_pipeline.py, the usage is as follows:
>>> plot_event.plot_candidate_events(candidate_event_dataframe, fil_file_list,
... filter_level, source_name_list, offset=0)
'''
global logger_plot_event
# load in the data for each individual hit
if candidate_event_dataframe is None:
print(
'*** plot_candidate_events: candidate_event_dataframe is None, nothing to do.'
)
return
len_df = len(candidate_event_dataframe)
if len_df < 1:
print(
'*** plot_candidate_events: len(candidate_event_dataframe) = 0, nothing to do.'
)
return
for i in range(0, len_df):
candidate = candidate_event_dataframe.iloc[i]
on_source_name = candidate['Source']
f_mid = candidate['Freq']
drift_rate = candidate['DriftRate']
# calculate the length of the total cadence from the fil files' headers
first_fil = bl.Waterfall(fil_file_list[0], load_data=False)
tfirst = first_fil.header['tstart']
last_fil = bl.Waterfall(fil_file_list[-1], load_data=False)
tlast = last_fil.header['tstart']
t_elapsed = Time(tlast, format='mjd').unix - Time(
tfirst, format='mjd').unix + (last_fil.n_ints_in_file -
1) * last_fil.header['tsamp']
# calculate the width of the plot based on making sure the full drift is visible
bandwidth = 2.4 * abs(drift_rate) / 1e6 * t_elapsed
bandwidth = np.max((bandwidth, 500. / 1e6))
# Get start and stop frequencies based on midpoint and bandwidth
f_start, f_stop = np.sort(
(f_mid - (bandwidth / 2), f_mid + (bandwidth / 2)))
# logger_plot_event.debug useful values
logger_plot_event.debug(
'*************************************************')
logger_plot_event.debug(
'*** The Parameters for This Plot Are: ****')
logger_plot_event.debug('Target = {}'.format(on_source_name))
logger_plot_event.debug('Bandwidth = {} MHz'.format(round(
bandwidth, 5)))
logger_plot_event.debug('Time Elapsed (inc. Slew) = {} s'.format(
round(t_elapsed)))
logger_plot_event.debug('Middle Frequency = {} MHz'.format(
round(f_mid, 4)))
logger_plot_event.debug('Expected Drift = {} Hz/s'.format(
round(drift_rate, 4)))
logger_plot_event.debug(
'*************************************************')
# Pass info to make_waterfall_plots() function
make_waterfall_plots(fil_file_list,
on_source_name,
f_start,
f_stop,
drift_rate,
f_mid,
filter_level,
source_name_list,
offset=offset,
**kwargs)
def make_waterfall_plots(fil_file_list,
on_source_name,
f_start,
f_stop,
drift_rate,
f_mid,
filter_level,
source_name_list,
offset=0,
**kwargs):
r'''
Makes waterfall plots of an event for an entire on-off cadence.
Parameters
----------
fil_file_list : str
List of filterbank files in the cadence.
on_source_name : str
Name of the on_source target.
f_start : float
Start frequency, in MHz.
f_stop : float
Stop frequency, in MHz.
drift_rate : float
Drift rate in Hz/s.
f_mid : float
<iddle frequency of the event, in MHz.
filter_level : int
Filter level (1, 2, or 3) that produced the event.
source_name_list : list
List of source names in the cadence, in order.
bandwidth : int
Width of the plot, incorporating drift info.
kwargs : dict
Keyword args to be passed to matplotlib imshow().
Notes
-----
Makes a series of waterfall plots, to be read from top to bottom, displaying a full cadence
at the frequency of a recorded event from find_event. Calls :func:`~plot_waterfall`
'''
global logger_plot_event
# prepare for plotting
matplotlib.rc('font', **font)
# set up the sub-plots
n_plots = len(fil_file_list)
fig = plt.subplots(n_plots,
sharex=True,
sharey=True,
figsize=(10, 2 * n_plots))
# get directory path for storing PNG files
dirpath = dirname(fil_file_list[0]) + '/'
# read in data for the first panel
fil1 = bl.Waterfall(fil_file_list[0], f_start=f_start, f_stop=f_stop)
t0 = fil1.header['tstart']
dummy, plot_data1 = fil1.grab_data()
# rebin data to plot correctly with fewer points
dec_fac_x, dec_fac_y = 1, 1
if plot_data1.shape[0] > MAX_IMSHOW_POINTS[0]:
dec_fac_x = plot_data1.shape[0] / MAX_IMSHOW_POINTS[0]
if plot_data1.shape[1] > MAX_IMSHOW_POINTS[1]:
dec_fac_y = int(np.ceil(plot_data1.shape[1] / MAX_IMSHOW_POINTS[1]))
plot_data1 = rebin(plot_data1, dec_fac_x, dec_fac_y)
# define more plot parameters
# never used: delta_f = 0.000250
mid_f = np.abs(f_start + f_stop) / 2.
subplots = []
# Fill in each subplot for the full plot
for i, filename in enumerate(fil_file_list):
logger_plot_event.debug(
'make_waterfall_plots: file {} in list: {}'.format(i, filename))
# identify panel
subplot = plt.subplot(n_plots, 1, i + 1)
subplots.append(subplot)
# read in data
fil = bl.Waterfall(filename, f_start=f_start, f_stop=f_stop)
# make plot with plot_waterfall
source_name = source_name_list[i]
this_plot = plot_waterfall(fil,
source_name,
f_start=f_start,
f_stop=f_stop,
**kwargs)
# calculate parameters for estimated drift line
t_elapsed = Time(fil.header['tstart'], format='mjd').unix - Time(
t0, format='mjd').unix
t_duration = (fil.n_ints_in_file - 1) * fil.header['tsamp']
f_event = f_mid + drift_rate / 1e6 * t_elapsed
# plot estimated drift line
overlay_drift(f_event, f_start, f_stop, drift_rate, t_duration, offset)
# Title the full plot
if i == 0:
plot_title = "%s \n $\dot{\\nu}$ = %2.3f Hz/s , MJD:%5.5f" % (
on_source_name, drift_rate, t0)
plt.title(plot_title)
# Format full plot
if i < len(fil_file_list) - 1:
plt.xticks(np.linspace(f_start, f_stop, num=4), ['', '', '', ''])
# More overall plot formatting, axis labelling
factor = 1e6
units = 'Hz'
ax = plt.gca()
ax.get_xaxis().get_major_formatter().set_useOffset(False)
xloc = np.linspace(f_start, f_stop, 5)
xticks = [round(loc_freq) for loc_freq in (xloc - mid_f) * factor]
if np.max(xticks) > 1000:
xticks = [xt / 1000 for xt in xticks]
units = 'kHz'
plt.xticks(xloc, xticks)
plt.xlabel("Relative Frequency [%s] from %f MHz" % (units, mid_f),
fontdict=font)
# Add colorbar
cax = fig[0].add_axes([0.94, 0.11, 0.03, 0.77])
fig[0].colorbar(this_plot,
cax=cax,
label='Normalized Power (Arbitrary Units)')
# Adjust plots
plt.subplots_adjust(hspace=0, wspace=0)
# save the figures
path_png = dirpath + filter_level + '_' + on_source_name + '_dr_' + "{:0.2f}".format(
drift_rate) + '_freq_' "{:0.6f}".format(f_start) + ".png"
plt.savefig(path_png, bbox_inches='tight')
logger_plot_event.debug(
'make_waterfall_plots: Saved file {}'.format(path_png))
# show figure before closing if this is an interactive context
mplbe = matplotlib.get_backend()
logger_plot_event.debug('make_waterfall_plots: backend = {}'.format(mplbe))
if mplbe in matplotlib.rcsetup.interactive_bk:
plt.show()
# close all figure windows
plt.close('all')
return subplots
def test_write_to_fil():
""" Load Voyager dataset and test plotting """
a = bl.Waterfall(voyager_h5)
a.write_to_fil('test_out.fil')
def test_info():
a = bl.Waterfall(voyager_h5)
print(a)
a.info()
a.blank_dc(n_coarse_chan=1)
a.calibrate_band_pass_N1()
def plot_candidate_events_individually(full_candidate_event_dataframe,
correct_fils,
source_name,
node_string,
filter_level,
show=False,
overwrite=False,
offset=0,
**kwargs):
trimmed_candidate_event_dataframe = pd.read_csv(
full_candidate_event_dataframe)
print('full_candidate_event_dataframe', full_candidate_event_dataframe)
#get only the events in the dataframe that are from the right target
#full_candidate_event_dataframe=full_candidate_event_dataframe['Source']
print('trimmed_candidate_event_dataframe_Source',
trimmed_candidate_event_dataframe.loc[:, ['Source']])
#print ('full_candidate_event_dataframe_Source', full_candidate_event_dataframe['Source'])
candidate_event_dataframe = trimmed_candidate_event_dataframe.loc[:, [
'Source', 'Freq', 'DriftRate', 'FreqStart'
]]
#candidate_event_dataframe = full_candidate_event_dataframe.loc[full_candidate_event_dataframe['Source'] == source_name]
print('full_candidate_event_dataframe', candidate_event_dataframe)
#load in the data for each individual hit
for i in range(0, len(candidate_event_dataframe)):
candidate = candidate_event_dataframe.iloc[i]
source_id = candidate['Source']
f_mid = candidate['Freq']
drate = -1 * candidate['DriftRate']
#load in the list of .fil files
filelist = open(correct_fils).readlines()
filelist = [files.replace('\n', '') for files in filelist]
#print ('dat_file_list', dat_file_list)
filelist = [files.replace(',', '') for files in filelist]
#filelist = correct_fils
print('filelist', filelist)
#calculate the length of the total ABABAB from the fil files' headers
fil0 = bl.Waterfall(filelist[0], load_data=True)
print('fil0', fil0)
fil0header = fil0.header
print('fil0header', fil0header)
y = {}
for key, value in fil0header.items():
y[key.decode("utf-8")] = value
fil0header = y
print('fil0header', fil0header)
t0 = fil0header['tstart']
fil6 = bl.Waterfall(filelist[-1], load_data=True)
fil6header = fil6.header
z = {}
for key, value in fil6header.items():
z[key.decode("utf-8")] = value
fil6header = z
t6 = fil6header['tstart']
t_elapsed = Time(t6, format='mjd').unix - Time(
t0, format='mjd').unix + (fil6.n_ints_in_file -
1) * fil6header['tsamp']
#calculate the width of the plot based on making sure the full drift is visible
bw = 2.4 * abs(drate) / 1e6 * t_elapsed
bw = np.max((bw, 500. / 1e6))
#Get start and stop frequencies based on midpoint and bandwidth
f_start, f_stop = np.sort((f_mid - bw / 2, f_mid + bw / 2))
#Print useful values
print('*************************************************')
print('*** The Parameters for This Plot Are: ***')
print('*************************************************')
print('Target = ', source_id)
print('Bandwidth (MHz) = ', bw)
print('Total Time Elapsed (s) = ', t_elapsed)
print('Start Frequency (MHz) = ', f_start)
print('Middle Frequency (MHz) = ', f_mid)
print('Stop Frequency (MHz) = ', f_stop)
print('Expected Drift (Hz/s) = ', drate)
print('*************************************************')
print('*************************************************')
#Pass info to make_waterfall_plots() function
subplots = make_waterfall_plots(filelist, [source_id], [drate],
[f_mid],
f_start,
f_stop,
node_string,
filter_level,
ion=False,
epoch=None,
local_host='',
plot_name='',
save_pdf_plot=True,
saving_fig=True,
offset=offset,
**kwargs)
return
def make_waterfall_plots(filenames_list,
target,
drates,
fvals,
f_start,
f_stop,
node_string,
filter_level,
ion=False,
epoch=None,
bw=250.0,
local_host='',
plot_name='',
save_pdf_plot=False,
saving_fig=False,
offset=0,
dedoppler=False,
**kwargs):
""" Makes waterfall plots per group of ON-OFF pairs (up to 6 plots.)
"""
#prepares for plotting
print('Preparing to plot: ', target)
matplotlib.rc('font', **font)
if ion:
plt.ion()
#defines a minimum and maximum of... something
min_val = 0
max_val = 5.
factor = 1e6
units = 'Hz'
#sets up the sub-plots
n_plots = len(filenames_list)
fig = plt.subplots(n_plots,
sharex=True,
sharey=True,
figsize=(10, 2 * n_plots))
#finding plotting values range for the first panel (A1)
fil = bl.Waterfall(filenames_list[0], f_start=f_start, f_stop=f_stop)
filheader = fil.header
f = {}
for key, value in filheader.items():
f[key] = value
filheader = f
print('filheader', filheader)
t0 = filheader['tstart']
plot_f, plot_data = fil.grab_data(f_start=f_start, f_stop=f_stop)
dec_fac_x, dec_fac_y = 1, 1
#print ('plot_data', plot_data)
#rebinning data to plot correctly with fewer plots
if plot_data.shape[0] > MAX_IMSHOW_POINTS[0]:
dec_fac_x = plot_data.shape[0] / MAX_IMSHOW_POINTS[0]
print('dec_fac_x', dec_fac_x)
if plot_data.shape[1] > MAX_IMSHOW_POINTS[1]:
dec_fac_y = plot_data.shape[1] / MAX_IMSHOW_POINTS[1]
print('dec_fac_y', dec_fac_y)
#insert rebin definition code from utils.py in blimpy and edit it because this is the problem
d = plot_data
n_x = dec_fac_x
n_y = dec_fac_y
d = rebin(d, n_x, n_y)
plot_data = d
#print('d', d)
#plot_data = rebin(plot_data, dec_fac_x, dec_fac_y)
#investigate intensity values for A1 (first panel)
plot_data = 10 * np.log10(plot_data)
A1_avg = np.median(plot_data)
A1_max = plot_data.max()
A1_std = np.std(plot_data)
#defining more plot parameters
delta_f = 0.000250
epoch = filheader['tstart']
mid_f = np.abs(f_start + f_stop) / 2.
drate_max = np.max(np.abs(drates))
subplots = []
#working out intensity scale
if kwargs.get('clim', None) is None:
vmin = A1_avg - A1_std * min_val - 2
vmax = A1_avg + max_val * A1_std
else:
vmin, vmax = kwargs['clim']
#Filling in each subplot for the full plot
for i, filename in enumerate(filenames_list):
subplot = plt.subplot(n_plots, 1, i + 1)
subplots.append(subplot)
fil = bl.Waterfall(filename, f_start=f_start, f_stop=f_stop)
filheader = fil.header
try:
this_plot = plot_waterfall(fil,
f_start=f_start,
f_stop=f_stop,
drate=drate_max,
vmin=vmin,
vmax=vmax,
**kwargs)
for drate, fval in zip(drates, fvals):
t_elapsed = Time(filheader['tstart'],
format='mjd').unix - Time(t0,
format='mjd').unix
t_duration = (fil.n_ints_in_file - 1) * filheader['tsamp']
f_event = fval + drate / 1e6 * t_elapsed
overlay_drift(f_event, drate, t_duration, offset)
except:
raise
#Titling the plot
if i == 0:
srcname = "%s $\dot{\\nu}$=%2.3f Hzs$^{-1}$" % (target, drate_max)
plt.title(srcname)
#Plot formatting
if i < len(filenames_list) - 1:
plt.xticks(np.arange(f_start, f_stop, delta_f / 4.),
['', '', '', ''])
#More plot formatting.
ax = plt.gca()
ax.get_xaxis().get_major_formatter().set_useOffset(False)
xloc = np.linspace(f_start, f_stop, 5)
xticks = [round(loc_freq) for loc_freq in (xloc - mid_f) * factor]
if np.max(xticks) > 1000:
xticks = [xt / 1000 for xt in xticks]
units = 'kHz'
plt.xticks(xloc, xticks)
plt.xlabel("Relative Frequency [%s] from %f MHz" % (units, mid_f),
fontdict=font)
#Colorbar
cax = fig[0].add_axes([0.94, 0.11, 0.03, 0.77])
fig[0].colorbar(this_plot, cax=cax, label='Power [dB counts]')
#Adjust plots
plt.subplots_adjust(hspace=0, wspace=0)
#save the figures
plt.savefig(node_string + '_f' + str(filter_level) + '_' + target[0] +
'_dr_' + "{:0.2f}".format(drate_max) + '_freq_'
"{:0.2f}".format(f_start) + ".png",
bbox_inches='tight')
return subplots
def make_waterfall_plots(fil_file_list,
on_source_name,
f_start,
f_stop,
drift_rate,
f_mid,
filter_level,
source_name_list,
bandwidth,
offset=0,
**kwargs):
''' Makes waterfall plots of an event for an entire on-off cadence
Args:
fil_file_list (str): list of filterbank files in the cadence
on_source_name (str): name of the on_source target
f_start (float): start frequency, in MHz
f_stop (float): stop frequency, in MHz
drift_rate (float) = drift rate in Hz/s
f_mid = middle frequency of the event, in MHz
filter_level = filter level (1,2,or 3) that produced the event
source_name_list = list of source names in the cadence, in order
bandwidth = width of the plot, incorporating drift info
kwargs: keyword args to be passed to matplotlib imshow()
'''
#prepare for plotting
matplotlib.rc('font', **font)
#set up the sub-plots
n_plots = len(fil_file_list)
fig = plt.subplots(n_plots,
sharex=True,
sharey=True,
figsize=(10, 2 * n_plots))
#read in data for the first panel
fil1 = bl.Waterfall(fil_file_list[0], f_start=f_start, f_stop=f_stop)
t0 = fil1.header['tstart']
plot_f1, plot_data1 = fil1.grab_data()
#rebin data to plot correctly with fewer points
dec_fac_x, dec_fac_y = 1, 1
if plot_data1.shape[0] > MAX_IMSHOW_POINTS[0]:
dec_fac_x = plot_data1.shape[0] / MAX_IMSHOW_POINTS[0]
if plot_data1.shape[1] > MAX_IMSHOW_POINTS[1]:
dec_fac_y = int(np.ceil(plot_data1.shape[1] / MAX_IMSHOW_POINTS[1]))
plot_data1 = rebin(plot_data1, dec_fac_x, dec_fac_y)
#define more plot parameters
delta_f = 0.000250
mid_f = np.abs(f_start + f_stop) / 2.
subplots = []
#Fill in each subplot for the full plot
for i, filename in enumerate(fil_file_list):
#identify panel
subplot = plt.subplot(n_plots, 1, i + 1)
subplots.append(subplot)
#read in data
fil = bl.Waterfall(filename, f_start=f_start, f_stop=f_stop)
try:
#make plot with plot_waterfall
source_name = source_name_list[i]
this_plot = plot_waterfall(fil,
source_name,
f_start=f_start,
f_stop=f_stop,
drift_rate=drift_rate,
**kwargs)
#calculate parameters for estimated drift line
t_elapsed = Time(fil.header['tstart'], format='mjd').unix - Time(
t0, format='mjd').unix
t_duration = (fil.n_ints_in_file - 1) * fil.header['tsamp']
f_event = f_mid + drift_rate / 1e6 * t_elapsed
#plot estimated drift line
overlay_drift(f_event, f_start, f_stop, drift_rate, t_duration,
offset)
except:
raise
#Title the full plot
if i == 0:
plot_title = "%s \n $\dot{\\nu}$ = %2.3f Hz/s" % (on_source_name,
drift_rate)
plt.title(plot_title)
#Format full plot
if i < len(fil_file_list) - 1:
plt.xticks(np.arange(f_start, f_stop, delta_f / 4.),
['', '', '', ''])
#More overall plot formatting, axis labelling
factor = 1e6
units = 'Hz'
ax = plt.gca()
ax.get_xaxis().get_major_formatter().set_useOffset(False)
xloc = np.linspace(f_start, f_stop, 5)
xticks = [round(loc_freq) for loc_freq in (xloc - mid_f) * factor]
if np.max(xticks) > 1000:
xticks = [xt / 1000 for xt in xticks]
units = 'kHz'
plt.xticks(xloc, xticks)
plt.xlabel("Relative Frequency [%s] from %f MHz" % (units, mid_f),
fontdict=font)
#Add colorbar
cax = fig[0].add_axes([0.94, 0.11, 0.03, 0.77])
fig[0].colorbar(this_plot, cax=cax, label='Normalized Power')
#Adjust plots
plt.subplots_adjust(hspace=0, wspace=0)
#save the figures
plt.savefig(filter_level + '_' + on_source_name + '_dr_' +
"{:0.2f}".format(drift_rate) + '_freq_'
"{:0.6f}".format(f_start) + ".png",
bbox_inches='tight')
return subplots