#
# This version of the GNU Lesser General Public License incorporates the terms
# and conditions of version 3 of the GNU General Public License,
# supplemented by the additional permissions listed below.
import bz2
import datetime as dt
import matplotlib.pyplot as plt
import pytest
import warnings
import pydarn
with bz2.open('test/data/test.fitacf.bz2') as fp:
fitacf_stream = fp.read()
data = pydarn.SuperDARNRead(fitacf_stream, True).read_fitacf()
class TestFan_defaults:
def test_fan_defaults(self):
""" """
with warnings.catch_warnings(record=True):
pydarn.Fan.plot_fan(data)
def test_fov_series(self):
""" """
with warnings.catch_warnings(record=True):
pydarn.Fan.plot_fov(6, dt.datetime(2020, 4, 4, 6, 2))
@pytest.mark.parametrize('stid', [5, 97])
import pydarn
import bz2
import matplotlib.pyplot as plt
from pandas.plotting import register_matplotlib_converters
if __name__ == '__main__':
fitacf_file = "data/20190318.1001.00.rkn.fitacf.bz2"
with bz2.open(fitacf_file) as fp:
fitacf_stream = fp.read()
sdarn_read = pydarn.SuperDARNRead(fitacf_stream, True)
fitacf_data = sdarn_read.read_fitacf()
register_matplotlib_converters()
pydarn.RTP.plot_summary(fitacf_data, beam_num=2, slant=False)
plt.show()
def convert_fitacf_data(date, in_fname, radar_info):
day = in_fname.split('.')[0].split('/')[-1]
month = day[:-2]
# Keep track of fitACF files that have multiple beam definitions in a
# monthly log file
multiBeamLogDir = date.strftime(helper.FIT_NET_LOG_DIR) + month
multiBeamLogfile = '{dir}/multi_beam_defs_{m}.log'.format(dir = multiBeamLogDir, m = month)
# Store conversion info like returns outside FOV, missing slist, etc
# for each conversion
conversionLogDir = '{dir}/{d}'.format(dir = multiBeamLogDir, d = day)
fName = in_fname.split('/')[-1]
conversionLogfile = '{dir}/{fit}_to_nc.log'.format(dir = conversionLogDir, fit = fName)
# Define the name of the file holding the list of rawACFs used to
# create the fitACF
rawacfListFilename = '.'.join(in_fname.split('.')[:-1]) + '.rawacfList.txt'
SDarn_read = pydarn.SuperDARNRead(in_fname)
data = SDarn_read.read_fitacf()
bmdata = {
'rsep': [],
'frang': [],
}
for rec in data:
for k, v in bmdata.items():
bmdata[k].append(rec[k])
if 'slist' in rec.keys():
if radar_info['maxrg'] < rec['slist'].max():
radar_info['maxrg'] = rec['slist'].max() + 5
for k, v in bmdata.items():
val = np.unique(v)
if len(val) > 1:
os.makedirs(conversionLogDir, exist_ok=True)
os.makedirs(multiBeamLogDir, exist_ok=True)
# Log the multiple beams error in the monthly mutli beam def log
logText = '{fitacfFullFile} has {numBeamDefs} beam definitions - skipping file conversion.\n'.format(fitacfFullFile = in_fname, numBeamDefs = len(val))
with open(multiBeamLogfile, "a+") as fp:
fp.write(logText)
# Log the multiple beams error in this fitACF's conversion log
with open(conversionLogfile, "a+") as fp:
fp.write(logText)
return MULTIPLE_BEAM_DEFS_ERROR_CODE, MULTIPLE_BEAM_DEFS_ERROR_CODE
bmdata[k] = int(val)
# Define FOV
fov = radFov.fov(
frang=bmdata['frang'], rsep=bmdata['rsep'], site=None, nbeams=int(radar_info['maxbeams']),
ngates=int(radar_info['maxrg']), bmsep=radar_info['beamsep'], recrise=radar_info['risetime'], siteLat=radar_info['glat'],
siteLon=radar_info['glon'], siteBore=radar_info['boresight'], siteAlt=radar_info['alt'], siteYear=date.year,
elevation=None, altitude=300., hop=None, model='IS',
coords='geo', date_time=date, coord_alt=0., fov_dir='front',
)
# Define fields
short_flds = 'tfreq', 'noise.sky', 'cp',
fov_flds = 'mjd', 'beam', 'range', 'lat', 'lon',
data_flds = 'p_l', 'v', 'v_e', 'gflg',
elv_flds = 'elv', 'elv_low', 'elv_high',
# Figure out if we have elevation information
elv_exists = True
for rec in data:
if 'elv' not in rec.keys():
elv_exists = False
if elv_exists:
data_flds += elv_flds
# Set up data storage
out = {}
for fld in (fov_flds + data_flds + short_flds):
out[fld] = []
# Run through each beam record and store
for rec in data:
time = dt.datetime(rec['time.yr'], rec['time.mo'], rec['time.dy'], rec['time.hr'], rec['time.mt'], rec['time.sc'])
# slist is the list of range gates with backscatter
if 'slist' not in rec.keys():
os.makedirs(conversionLogDir, exist_ok=True)
logText = 'Could not find slist in record {recordTime} - skipping\n'.format(recordTime = time.strftime('%Y-%m-%d %H:%M:%S'))
with open(conversionLogfile, "a+") as fp:
fp.write(logText)
continue
# Can't deal with returns outside of FOV
if rec['slist'].max() >= fov.slantRCenter.shape[1]:
os.makedirs(conversionLogDir, exist_ok=True)
# Log returns outside of FOV
logText = 'Record {recordTime} found to have a max slist of {maxSList} - skipping record/n'.format(recordTime = time.strftime('%Y-%m-%d %H:%M:%S'), maxSList = rec['slist'].max())
with open(conversionLogfile, "a+") as fp:
fp.write(logText)
continue
time = dt.datetime(rec['time.yr'], rec['time.mo'], rec['time.dy'], rec['time.hr'], rec['time.mt'], rec['time.sc'])
one_obj = np.ones(len(rec['slist']))
mjd = jdutil.jd_to_mjd(jdutil.datetime_to_jd(time))
bmnum = one_obj * rec['bmnum']
fovi = fov.beams == rec['bmnum']
out['mjd'] += (one_obj * mjd).tolist()
out['beam'] += bmnum.tolist()
out['range'] += fov.slantRCenter[fovi, rec['slist']].tolist()
out['lat'] += fov.latCenter[fovi, rec['slist']].tolist()
out['lon'] += fov.lonCenter[fovi, rec['slist']].tolist()
for fld in data_flds:
out[fld] += rec[fld].tolist()
for fld in short_flds: # expand out to size
out[fld] += (one_obj * rec[fld]).tolist()
# Convert to numpy arrays
for k, v in out.items():
out[k] = np.array(v)
# Calculate beam azimuths assuming 20 degrees elevation
beam_off = radar_info['beamsep'] * (fov.beams - (radar_info['maxbeams'] - 1) / 2.0)
el = 15.
brng = np.zeros(beam_off.shape)
for ind, beam_off_elzero in enumerate(beam_off):
brng[ind] = radFov.calcAzOffBore(el, beam_off_elzero, fov_dir=fov.fov_dir) + radar_info['boresight']
# Pull the fit version out of the fitACF filename
fit_version = '.'.join(in_fname.split('.')[-3:-1])
# Load the list of rawacf files used to create the fitacf and netcdf
with open(rawacfListFilename, "rb") as fp:
rawacf_source_files = pickle.load(fp)
# Once the list of rawacf source files has been loaded, delete the file used to
# temporarily store that information
os.system('rm {rawacfListFile}'.format(rawacfListFile = rawacfListFilename))
hdr = {
'lat': radar_info['glat'],
'lon': radar_info['glon'],
'alt': radar_info['alt'],
'rsep': bmdata['rsep'],
'maxrg': radar_info['maxrg'],
'bmsep': radar_info['beamsep'],
'boresight': radar_info['boresight'],
'beams': fov.beams,
'brng_at_15deg_el': brng,
'fitacf_version': fit_version,
'rawacf_source': rawacf_source_files
}
return out, hdr
# document, but changing it is not allowed.
#
# This version of the GNU Lesser General Public License incorporates the terms
# and conditions of version 3 of the GNU General Public License,
# supplemented by the additional permissions listed below.
import bz2
import datetime as dt
import matplotlib.pyplot as plt
import pytest
import warnings
import pydarn
data = pydarn.SuperDARNRead('test/data/test.grd').read_grid()
class TestGrid_defaults:
def test_grid_defaults(self):
""" """
with warnings.catch_warnings(record=True):
pydarn.Grid.plot_grid(data)
@pytest.mark.parametrize('colorbar', [False])
@pytest.mark.parametrize('colorbar_label', 'green')
@pytest.mark.parametrize('title', [False])
@pytest.mark.parametrize('cmap', [plt.get_cmap('rainbow')])
@pytest.mark.parametrize('record', [2])
@pytest.mark.parametrize('ranges', [(5,70)])
def get_data(in_filename, AtoGHeight):
print('Processing %s' % in_filename)
map_data = pydarn.SuperDARNRead(in_filename).read_map()
latMagVector = []
lonMagVector = []
azimuthMagVector = []
latGeoVector = []
lonGeoVector = []
azimuthGeoVector = []
velocityVector = []
sdVector = []
timeVector = []
for entry in map_data:
numPoints = len(entry['vector.vel.median'])
t = dt.datetime(
int(entry['start.year']),
int(entry['start.month']),
int(entry['start.day']),
int(entry['start.hour']),
int(entry['start.minute']),
int(entry['start.second']),
)
# Convert time to seconds since 1970-01-01 00:00
times = [t.timestamp()] * numPoints
timeVector.append(times)
azM = entry['vector.kvect']
velocityVector.append(entry['vector.vel.median'])
sdVector.append(entry['vector.vel.sd'])
latMagVector.append(entry['vector.mlat'])
lonMagVector.append(entry['vector.mlon'])
azimuthMagVector.append(azM)
latG, lonG, heightG = aacgmv2.convert_latlon_arr(
entry['vector.mlat'],
entry['vector.mlon'],
AtoGHeight,
t,
method_code='A2G',
)
latGeoVector.append(latG)
lonGeoVector.append(lonG)
azG = convert_azm_aacgm2geo(azM, latG, lonG, t, refAlt=AtoGHeight)
azimuthGeoVector.append(azG)
data = {}
data['times'] = np.array(np.concatenate(timeVector))
data['mLat'] = np.array(np.concatenate(latMagVector))
data['mLon'] = np.array(np.concatenate(lonMagVector))
data['mAz'] = np.array(np.concatenate(azimuthMagVector))
data['gAz'] = np.array(np.concatenate(azimuthGeoVector))
data['gLat'] = np.array(np.concatenate(latGeoVector))
data['gLon'] = np.array(np.concatenate(lonGeoVector))
data['vel'] = np.array(np.concatenate(velocityVector))
data['sd'] = np.array(np.concatenate(sdVector))
return data
def PickleFITACF_occ(station, date, beam_range):
"""
Take a fitACF file and for each possible echo, record whether or not there was a good echo there
Note: occurrence rate is not actually computed here, but all the data required to compute it is put into the df
From Koustov and Syd's paper
"The echo occurrence rate was computed as a ratio of the number of registered echoes in selected beams and gates
to the total number of possible echo detections in the same gates over the same period. 15-min averaging
intervals were considered."
:param station: str:
The radar station to consider, a 3 character string (e.g. "rkn").
For a complete listing of available stations, please see https://superdarn.ca/radar-info
:param date: str:
The date as a string of the form 'yyyymmdd'. Single digit months and days need to be zero padded.
:param beam_range: (<int>, <int>) (optional):
Inclusive. The beam range to consider. If omitted (or None), then all beams will be considered.
Note that beams start at 0, so beams (0, 3) is 4 beams.
In the past the following beams have been used:
- INV 13-15 - DCE 10-12
- RKN 1-3 - MCM 6-8
- CLY 4-6 - SPS 3-5
"""
gate_range = (0, 74) # TODO: Update this to read in gate range from hardware?
epoch, date_time = [], []
slist, beam = [], []
frang, rsep, tfreq = [], [], []
good_iono_echo, good_grndscat_echo = [], []
# Loop through all the files for this station/date
in_dir = "data/" + station + "/" + station + date
for in_file in glob.iglob(in_dir + "/*.fitacf.bz2"):
# Unpack and open the file
print(" Reading " + in_file + "...")
try:
with bz2.open(in_file) as fp:
fitacf_stream = fp.read()
sdarn_read = pydarn.SuperDARNRead(fitacf_stream, True)
fitacf_data = sdarn_read.read_fitacf()
except BaseException as e:
# Sometimes files are corrupted, or there is something wrong with them
print(e)
pass
# print("List of available parameters: " + str(fitacf_data[0].keys()))
# Loop through every record in this file
for record in range(len(fitacf_data)):
beam_here = fitacf_data[record]['bmnum']
if beam_here < beam_range[0] or beam_here > beam_range[1]:
continue # We are not interested in records for this beam
date_time_here, epoch_here = build_datetime_epoch(year=fitacf_data[record]['time.yr'],
month=fitacf_data[record]['time.mo'],
day=fitacf_data[record]['time.dy'],
hour=fitacf_data[record]['time.hr'],
minute=fitacf_data[record]['time.mt'],
second=fitacf_data[record]['time.sc'])
try:
gates_reporting = np.ndarray.tolist(fitacf_data[record]['slist'])
except:
# Sometimes there won't be any gates reporting, this is legacy behaviour and results in partial records
gates_reporting = []
for gate in range(gate_range[0], gate_range[1], 1):
epoch.append(epoch_here)
date_time.append(date_time_here)
slist.append(gate)
beam.append(beam_here)
tfreq.append(fitacf_data[record]['tfreq'])
frang.append(fitacf_data[record]['frang'])
rsep.append(fitacf_data[record]['rsep'])
try:
gate_index = gates_reporting.index(gate)
if fitacf_data[record]['qflg'][gate_index] == 1 and fitacf_data[record]['p_l'][gate_index] >= 3:
# We have a good echo
if fitacf_data[record]['gflg'][gate_index] == 0:
# We have a good ionospheric echo
good_iono_echo.append(True)
good_grndscat_echo.append(False)
else:
# We have a good ground scatter echo
good_grndscat_echo.append(True)
good_iono_echo.append(False)
else:
# Bad echo
good_iono_echo.append(False)
good_grndscat_echo.append(False)
except ValueError:
# The gate is not in the list of reporting gates
good_iono_echo.append(False)
good_grndscat_echo.append(False)
except BaseException as e:
print(e)
if len(epoch) == 0:
# We have found no data, panic
raise Exception("PickleFITACF_occ() found no data matching the provided criteria.")
# Put the data into a dataframe
print(" Building the data frame...")
df = pd.DataFrame({'epoch': epoch, 'datetime': date_time,
'slist': slist, 'bmnum': beam,
'frang': frang, 'rsep': rsep, 'tfreq': tfreq,
'good_iono_echo': good_iono_echo, 'good_grndscat_echo': good_grndscat_echo
})
# Save to file
out_file = in_dir + "/" + station + date + "_occ.pkl"
print(" Pickling as " + out_file + "...")
df.to_pickle(out_file)
def get_data_occ(station, year_range, month_range, day_range, gate_range,
beam_range, freq_range):
"""
Take a fitACF file and for each possible echo, record whether or not there was a good echo there
Note: occurrence rate is not actually computed here, but all the data required to compute it is put into the df
From Koustov and Syd's paper:
"The echo occurrence rate was computed as a ratio of the number of registered echoes in selected beams and gates
to the total number of possible echo detections in the same gates over the same period."
:param station: str:
The radar station to consider, as a 3 character string (e.g. "rkn").
For a complete listing of available stations, please see https://superdarn.ca/radar-info
:param year_range: (<int>, <int>):
Inclusive. The year range to consider.
:param month_range: (<int>, <int>) (optional):
Inclusive. The months of the year to consider. If omitted (or None), then all days will be considered.
:param day_range: (<int>, <int>) (optional):
Inclusive. The days of the month to consider. If omitted (or None), then all days will be considered.
:param gate_range: (<int>, <int>) (optional):
Inclusive. The gate range to consider. If omitted (or None), then all the gates will be considered.
Note that gates start at 0, so gates (0, 3) is 4 gates.
:param beam_range: (<int>, <int>) (optional):
Inclusive. The beam range to consider. If omitted (or None), then all beams will be considered.
Note that beams start at 0, so beams (0, 3) is 4 beams.
:param freq_range: (<float>, <float>) (optional):
Inclusive. The frequency range to consider in MHz.
If omitted (or None), then all frequencies are considered.
:return: pandas.DataFrame: A dataframe with select fitACF parameters.
"""
# loc_root = "/data/fitacf_30" # fitACF 3.0
loc_root = "/data/fitacf_25" # fitACF 2.5
# Create empty arrays for the parameters we need
epoch, date_time = [], []
slist, beam = [], []
frang, rsep, tfreq = [], [], []
good_iono_echo, good_grndscat_echo = [], []
# Data on maxwell.usask.ca is stored in a file structure that looks like this:
# /data/fitacf_30/<year>/<month>/<year><month><day>.<start-time>.<radar-site>.fitacf.bz2
for in_dir_parent in glob.iglob(loc_root + "/*"):
year_here = int(os.path.basename(in_dir_parent))
if year_range[0] <= year_here <= year_range[1]:
for in_dir in glob.iglob(in_dir_parent + "/*"):
month_here = int(os.path.basename(in_dir))
if month_range[0] <= month_here <= month_range[1]:
for in_file in glob.iglob(in_dir + "/*" + station + "*"):
day_here = int(os.path.basename(in_file)[6:8])
if day_range[0] <= day_here <= day_range[1]:
# We will read in the whole day, and worry about hour restrictions later
print(" Reading: " + str(in_file))
try:
with bz2.open(in_file) as fp:
fitacf_stream = fp.read()
sdarn_read = pydarn.SuperDARNRead(
fitacf_stream, True)
fitacf_data = sdarn_read.read_fitacf(
) # this is
except BaseException:
# Sometimes files are corrupted, or there is something wrong with them
pass
# Loop through every record in this file
for record in range(len(fitacf_data)):
beam_here = fitacf_data[record]['bmnum']
if beam_here < beam_range[
0] or beam_here > beam_range[1]:
continue # We are not interested in records for this beam
date_time_here, epoch_here = build_datetime_epoch_local(
year=fitacf_data[record]['time.yr'],
month=fitacf_data[record]['time.mo'],
day=fitacf_data[record]['time.dy'],
hour=fitacf_data[record]['time.hr'],
minute=fitacf_data[record]['time.mt'],
second=fitacf_data[record]['time.sc'])
try:
gates_reporting = np.ndarray.tolist(
fitacf_data[record]['slist'])
except:
# Sometimes there won't be any gates reporting, this is legacy behaviour and
# results in partial records
gates_reporting = []
for gate in range(gate_range[0], gate_range[1],
1):
epoch.append(epoch_here)
date_time.append(date_time_here)
slist.append(gate)
beam.append(beam_here)
tfreq.append(fitacf_data[record]['tfreq'])
frang.append(fitacf_data[record]['frang'])
rsep.append(fitacf_data[record]['rsep'])
try:
gate_index = gates_reporting.index(
gate)
if fitacf_data[record]['qflg'][gate_index] == 1 and \
fitacf_data[record]['p_l'][gate_index] >= 3:
# We have a good echo
if fitacf_data[record]['gflg'][
gate_index] == 0:
# We have a good ionospheric echo
good_iono_echo.append(True)
good_grndscat_echo.append(
False)
else:
# We have a good ground scatter echo
good_grndscat_echo.append(True)
good_iono_echo.append(False)
else:
# Bad echo
good_iono_echo.append(False)
good_grndscat_echo.append(False)
except ValueError:
# The gate is not in the list of reporting gates
good_iono_echo.append(False)
good_grndscat_echo.append(False)
except BaseException as e:
print(e)
if len(epoch) == 0:
# We have found no data, panic
warnings.warn(
"get_data_occ() found no data matching the provided criteria. "
"year_range: " + str(year_range) + ", month_range:" +
str(month_range) + ", day_range" + str(day_range),
category=Warning)
# Put the data into a dataframe
print(" Building the data frame...")
df = pd.DataFrame({
'epoch': epoch,
'datetime': date_time,
'slist': slist,
'bmnum': beam,
'frang': frang,
'rsep': rsep,
'tfreq': tfreq,
'good_iono_echo': good_iono_echo,
'good_grndscat_echo': good_grndscat_echo
})
# Filter the data for the needed beam, gate, and freq ranges
df = df.loc[(df['bmnum'] >= beam_range[0]) & (df['bmnum'] <= beam_range[1])
& (df['slist'] >= gate_range[0]) &
(df['slist'] <= gate_range[1]) &
# Note: freq_range is in MHz while data in 'tfreq' is in kHz
(df['tfreq'] >= freq_range[0] * 1000) &
(df['tfreq'] <= freq_range[1] * 1000)]
return df
def get_data(station, year_range, month_range, day_range, hour_range, gate_range, beam_range, freq_range):
"""
Get all of the SuperDARN data within the desired range/time, put it into a dataframe,
and then return the dataframe for plotting/analysis
Dataframe keys follow the same convention as fitACF:
https://radar-software-toolkit-rst.readthedocs.io/en/latest/references/general/fitacf/
:param station: str:
The radar station to consider, as a 3 character string (e.g. "rkn").
For a complete listing of available stations, please see https://superdarn.ca/radar-info
:param year_range: (<int>, <int>):
Inclusive. The year range to consider.
:param month_range: (<int>, <int>) (optional):
Inclusive. The months of the year to consider. If omitted (or None), then all days will be considered.
:param day_range: (<int>, <int>) (optional):
Inclusive. The days of the month to consider. If omitted (or None), then all days will be considered.
:param hour_range: (<int>, <int>) (optional):
The hour range to consider. If omitted (or None), then all hours will be considered.
Not inclusive: if you pass in (0, 5) you will get from 0:00-4:59 UT
:param gate_range: (<int>, <int>) (optional):
Inclusive. The gate range to consider. If omitted (or None), then all the gates will be considered.
Note that gates start at 0, so gates (0, 3) is 4 gates.
:param beam_range: (<int>, <int>) (optional):
Inclusive. The beam range to consider. If omitted (or None), then all beams will be considered.
Note that beams start at 0, so beams (0, 3) is 4 beams.
:param freq_range: (<float>, <float>) (optional):
Inclusive. The frequency range to consider in MHz.
If omitted (or None), then all frequencies are considered.
:return: pandas.DataFrame: A dataframe with select fitACF parameters.
"""
loc_root = "/data/fitacf_30" # fitACF 3.0
# loc_root = "/data/fitacf_25" # fitACF 2.5
pattern = "%Y.%m.%d %H:%M:%S"
# Create empty arrays for scalar parameters
epoch, date_time = [], []
bmnum = []
tfreq = []
frang, rsep = [], []
# Create empty arrays for vector parameters
slist = []
qflg, gflg = [], []
v, p_l, w_l = [], [], []
phi0, elv = [], []
# Data on maxwell.usask.ca is stored in a file structure that looks like this:
# /data/fitacf_30/<year>/<month>/<year><month><day>.<start-time>.<radar-site>.fitacf.bz2
for in_dir_parent in glob.iglob(loc_root + "/*"):
year_here = int(os.path.basename(in_dir_parent))
if year_range[0] <= year_here <= year_range[1]:
for in_dir in glob.iglob(in_dir_parent + "/*"):
month_here = int(os.path.basename(in_dir))
if month_range[0] <= month_here <= month_range[1]:
for in_file in glob.iglob(in_dir + "/*" + station + "*"):
day_here = int(os.path.basename(in_file)[6:8])
if day_range[0] <= day_here <= day_range[1]:
# We will read in the whole day, and worry about hour restrictions later
print(" Reading: " + str(in_file))
try:
with bz2.open(in_file) as fp:
fitacf_stream = fp.read()
sdarn_read = pydarn.SuperDARNRead(fitacf_stream, True)
fitacf_data = sdarn_read.read_fitacf() # this is
except BaseException:
# Sometimes files are corrupted, or there is something wrong with them
pass
# Loop through all the scans and build up the arrays
# As far as I know, list extensions are the fastest way to do this
for scan in range(len(fitacf_data)):
date_time_here, epoch_here = build_datetime_epoch_local(
year=fitacf_data[scan]['time.yr'],
month=fitacf_data[scan]['time.mo'],
day=fitacf_data[scan]['time.dy'],
hour=fitacf_data[scan]['time.hr'],
minute=fitacf_data[scan]['time.mt'],
second=fitacf_data[scan]['time.sc'])
try:
num_gates_reporting = len(fitacf_data[scan]['slist'])
except:
# Sometimes there won't be any gates reporting, this is legacy behaviour and
# results in partial records
num_gates_reporting = 0
if num_gates_reporting > 0:
# Build up scalar parameters
epoch.extend([epoch_here] * num_gates_reporting)
date_time.extend([date_time_here] * num_gates_reporting)
bmnum.extend([fitacf_data[scan]['bmnum']] * num_gates_reporting)
tfreq.extend([fitacf_data[scan]['tfreq']] * num_gates_reporting)
frang.extend([fitacf_data[scan]['frang']] * num_gates_reporting)
rsep.extend([fitacf_data[scan]['rsep']] * num_gates_reporting)
# Build up vector parameters
slist.extend(fitacf_data[scan]['slist'])
qflg.extend(fitacf_data[scan]['qflg'])
gflg.extend(fitacf_data[scan]['gflg'])
v.extend(fitacf_data[scan]['v'])
p_l.extend(fitacf_data[scan]['p_l'])
w_l.extend(fitacf_data[scan]['w_l'])
try:
phi0.extend(fitacf_data[scan]['phi0'])
except KeyError:
# Some files might not have this parameter
phi0.extend([math.nan] * num_gates_reporting)
except BaseException:
raise
try:
elv.extend(fitacf_data[scan]['elv'])
except KeyError:
# Some files might not have this parameter
elv.extend([math.nan] * num_gates_reporting)
except BaseException:
raise
if len(epoch) == 0:
# We have found no data, panic
raise Exception("get_data() found no data matching the provided criteria.")
df = pd.DataFrame({'epoch': epoch, 'datetime': date_time,
'bmnum': bmnum,
'tfreq': tfreq,
'frang': frang, 'rsep': rsep,
'slist': slist,
'qflg': qflg, 'gflg': gflg,
'v': v, 'p_l': p_l, 'w_l': w_l,
'phi0': phi0, 'elv': elv
})
# Filter the data for the needed time, beam, gate, and freq ranges
df = df.loc[(df['datetime'].hour >= hour_range[0]) & (df['datetime'].hour < hour_range[1]) &
(df['bmnum'] >= beam_range[0]) & (df['bmnum'] <= beam_range[1]) &
(df['slist'] >= gate_range[0]) & (df['slist'] <= gate_range[1]) &
# Note: freq_range is in MHz while data in 'tfreq' is in kHz
(df['tfreq'] >= freq_range[0] * 1000) & (df['tfreq'] <= freq_range[1] * 1000)]
# Until I have an application that requires bad quality points, I will assume they always need to be filtered out
df = df.loc[(df['qflg'] == 1)]
df.drop(columns=['qflg'], inplace=True)
df.reset_index(drop=True, inplace=True)
return df