def tai_to_datetime(cls, t):
tepoch = epochs.CDFepoch()
return tepoch.to_datetime(t * int(1e9) + tai_1958)
def cdf_load_var(cdf, varname):
'''
Read a variable and its metadata into a xarray.DataArray
Parameters
----------
cdf : `cdfread.CDF`
The CDF file object
varname : str
Name of the variable to read
Returns
-------
da : `xarray.DataArray`
The data with metadata
'''
time_types = ('CDF_EPOCH', 'CDF_EPOCH16', 'CDF_TIME_TT2000')
tepoch = epochs.CDFepoch()
varinq = cdf.varinq(varname)
# Convert epochs to datetimes
# - None is returned if tstart and tend are outside data interval
data = cdf.varget(variable=varname)
if varinq['Data_Type_Description'] in time_types:
try:
data = np.asarray([np.datetime64(t)
for t in tepoch.to_datetime(data)])
except TypeError:
pass
# Get dependent variables first because multi-dimensional
# dependent variables (e.g. those with record variance)
# will need to be parsed in order to set the dimensions of
# the current variable
coords = {}
dims = []
varatts = cdf.varattsget(varname)
for dim in range(len(data.shape)):
dim_name = None
try:
dim_name = varatts['DEPEND_{0}'.format(dim)]
except KeyError:
# Dimensions may have labels instead.
try:
dim_name = varatts['LABL_PTR_{0}'.format(dim)]
# DEPEND_0 variables for the record varying dimension
# typically do not depend on anything or have labels.
#
# DEPEND_N variables (where 1 <= N <= 3) are typically
# one dimensional if not record varying, and two
# dimensional if record varying. In either case, the
# non-record-varying dimension does not have a
# dependent variable or label because *it is* the
# dependent variable. In this case, name the dimension
# with the axis label.
except KeyError:
try:
dim_name = varatts['LABLAXIS']
except KeyError:
dim_name = varname
# if dim == 0:
# pass
# elif dim == 1:
# print(varname)
# dim_name = varatts['LABLAXIS']
# else:
# raise ValueError('Unknown coordinates for '
# 'dimension {}'.format(dim))
if dim_name == varname:
continue
# Sometimes the same DEPEND_N or LABL_PTR_N variable is
# used for multiple dimensions. A DataArray requires
# unique coordinate names, so append "_dimN" to duplicate
# names. This happens for, e.g., a temperature tensor
# variable with dimensions [3,3] and labels ['x', 'y', 'z']
# for each dimension.
coord_name = dim_name
if dim_name in coords:
coord_name += '_dim{}'.format(dim)
# DEPEND_# and LABL_PTR_# are pointers to other variables in the
# CDF file. Read the data from those variables as the coordinate
# values for this dimension.
if dim_name in cdf.cdf_info()['zVariables']:
# coords[coord_name] = cdf_load_var(cdf, dim_name)
coord_data = cdf_load_var(cdf, dim_name)
if len(coord_data.shape) == 1:
dim_name = coord_name
elif len(coord_data.shape) == 2:
dim_name = coord_data.dims[-1]
else:
ValueError('Coordinate has unexpected number of '
'dimensions (>2).')
coords[coord_name] = coord_data
dims.append(dim_name)
elif dim > 0:
dims.append(dim_name)
# If the variable is not record varying, the "records" dimension
# will be shallow and need to be removed so that data has the
# same number of dimensions as dims has elements.
if not varinq['Rec_Vary']:
data = data.squeeze()
# Unlike coords, dims cannot be empty.
if len(coords) == 0:
dims.append(varname)
da = xr.DataArray(data,
dims=dims,
coords=coords)
# Create the variable
da.name = varname
da.attrs['rec_vary'] = varinq['Rec_Vary']
da.attrs['cdf_name'] = varinq['Variable']
da.attrs['cdf_type'] = varinq['Data_Type_Description']
# Read the metadata
cdf_attget(cdf, da)
return da
def cdf_load_var(cdf, varname):
'''
Read a variable and its metadata into a xarray.DataArray
Parameters
----------
cdf : `cdfread.CDF`
The CDF file object
varname : str
Name of the variable to read
Returns
-------
data : dict
Variables read from file
'''
global cdf_vars_read
time_types = ('CDF_EPOCH', 'CDF_EPOCH16', 'CDF_TIME_TT2000')
tepoch = epochs.CDFepoch()
varinq = cdf.varinq(varname)
# Some variables have circular references. For example, the Epoch
# variable has a variable attribute DELTA_PLUS_VAR that points to
# to the Delta_Plus variable. The Delta_Plus variable has a DEPEND_0
# attribute that points back to the Epoch variable. To prevent an
# infinite loop, short circuit if a variable has already been read.
if varname in cdf_vars_read:
return
# Read the variable data
data = cdf.varget(variable=varname)
# Convert epochs to datetimes
if varinq['Data_Type_Description'] in time_types:
try:
data = np.asarray(
[np.datetime64(t) for t in tepoch.to_datetime(data)])
# None is returned if tstart and tend are outside data interval
except TypeError:
pass
# If the variable is not record varying, the "records" dimension
# will be shallow and need to be removed so that data has the
# same number of dimensions as dims has elements.
if not varinq['Rec_Vary']:
data = data.squeeze()
dims, coords = cdf_var_dims(cdf, varname, len(data.shape))
da = xr.DataArray(data, dims=dims, coords=coords)
# Indicate that the variable has been read.
cdf_vars_read[varname] = da
# Create the variable
da.name = varname
da.attrs['rec_vary'] = varinq['Rec_Vary']
da.attrs['cdf_name'] = varinq['Variable']
da.attrs['cdf_type'] = varinq['Data_Type_Description']
# Read the metadata
cdf_attget(cdf, da)
def cdf_to_df(cdf_files, cdf_vars, epoch='Epoch'):
'''
Read variables from CDF files into a dataframe
Parameters
----------
cdf_files : str or list
CDF files to be read
cdf_vars : str or list
Names of the variables to be read
epoch : str
Name of the time variable that serves as the data frame index
Returns
-------
out : `pandas.DataFrame`
The data. If a variable is 2D, "_#" is appended, where "#"
increases from 0 to var.shape[1]-1.
'''
tepoch = epochs.CDFepoch()
if isinstance(cdf_files, str):
cdf_files = [cdf_files]
if isinstance(cdf_vars, str):
cdf_vars = [cdf_vars]
if epoch not in cdf_vars:
cdf_vars.append(epoch)
out = []
for file in cdf_files:
file_df = pd.DataFrame()
cdf = cdfread.CDF(file)
for var_name in cdf_vars:
# Read the variable data
data = cdf.varget(var_name)
if var_name == epoch:
data = tepoch.to_datetime(data, to_np=True)
# Store as column in data frame
if data.ndim == 1:
file_df[var_name] = data
# 2D variables get "_#" appended to name for each column
elif data.ndim == 2:
for idx in range(data.shape[1]):
file_df['{0}_{1}'.format(var_name, idx)] = data[:, idx]
# 3D variables gets reshaped to 2D and treated as 2D
# This includes variables like the pressure and temperature tensors
elif data.ndim == 3:
dims = data.shape
data = data.reshape(dims[0], dims[1] * dims[2])
for idx in range(data.shape[1]):
file_df['{0}_{1}'.format(var_name, idx)] = data[:, idx]
else:
print('cdf_var.ndims > 3. Skipping. {0}'.format(var_name))
continue
# Close the file
cdf.close()
# Set the epoch variable as the index
file_df.set_index(epoch, inplace=True)
out.append(file_df)
# Concatenate all of the file data
out = pd.concat(out)
# Check that the index is unique
# Some contiguous low-level data files have data overlap at the edges of the files (e.g., AFG)
if not out.index.is_unique:
out['index'] = out.index
out.drop_duplicates(subset='index', inplace=True, keep='first')
out.drop(columns='index', inplace=True)
# File names are not always given in order, so sort the data
out.sort_index(inplace=True)
return out
def plot_context():
sc = 'mms1'
mode = 'srvy'
level = 'l2'
starttime = dt.datetime(2019, 10, 17)
# Find SROI
start_date, end_date = gls_get_sroi(starttime)
# Grab selections
abs_files = sdc.sitl_selections('abs_selections',
start_date=start_date,
end_date=end_date)
sitl_files = sdc.sitl_selections('sitl_selections',
start_date=start_date,
end_date=end_date)
gls_files = sdc.sitl_selections('gls_selections',
gls_type='mp-dl-unh',
start_date=start_date,
end_date=end_date)
# Read the files
abs_data = sdc.read_eva_fom_structure(abs_files[0])
sitl_data = sdc.read_eva_fom_structure(sitl_files[0])
gls_data = sdc.read_gls_csv(gls_files)
# SITL data time series
t_abs = []
x_abs = []
for tstart, tstop, fom in zip(abs_data['tstart'], abs_data['tstop'],
abs_data['fom']):
t_abs.extend([tstart, tstart, tstop, tstop])
x_abs.extend([0, fom, fom, 0])
t_sitl = []
x_sitl = []
for tstart, tstop, fom in zip(sitl_data['tstart'], sitl_data['tstop'],
sitl_data['fom']):
t_sitl.extend([tstart, tstart, tstop, tstop])
x_sitl.extend([0, fom, fom, 0])
t_gls = []
x_gls = []
for tstart, tstop, fom in zip(gls_data['tstart'], gls_data['tstop'],
gls_data['fom']):
t_gls.extend([tstart, tstart, tstop, tstop])
x_gls.extend([0, fom, fom, 0])
# FGM
tepoch = epochs.CDFepoch()
t_vname = 'Epoch'
b_vname = '_'.join((sc, 'fgm', 'b', 'gse', mode, level))
api = sdc.MrMMS_SDC_API(sc,
'fgm',
mode,
level,
start_date=start_date,
end_date=end_date)
files = api.download_files()
t_fgm = np.empty(0, dtype='datetime64')
b_fgm = np.empty((0, 4), dtype='float')
for file in files:
cdf = cdfread.CDF(file)
time = cdf.varget(t_vname)
t_fgm = np.append(t_fgm, tepoch.to_datetime(time, to_np=True), 0)
b_fgm = np.append(b_fgm, cdf.varget(b_vname), 0)
# FPI DIS
fpi_mode = 'fast'
api = sdc.MrMMS_SDC_API(sc,
'fpi',
fpi_mode,
level,
optdesc='dis-moms',
start_date=start_date,
end_date=end_date)
files = api.download_files()
ti = np.empty(0, dtype='datetime64')
ni = np.empty(0)
espec_i = np.empty((0, 32))
Ei = np.empty((0, 32))
ti = np.empty(0)
t_vname = 'Epoch'
ni_vname = '_'.join((sc, 'dis', 'numberdensity', fpi_mode))
espec_i_vname = '_'.join((sc, 'dis', 'energyspectr', 'omni', fpi_mode))
Ei_vname = '_'.join((sc, 'dis', 'energy', fpi_mode))
for file in files:
cdf = cdfread.CDF(file)
# tepoch = epochs.CDFepoch()
time = cdf.varget(t_vname)
ti = np.append(ti, tepoch.to_datetime(time, to_np=True), 0)
ni = np.append(ni, cdf.varget(ni_vname), 0)
espec_i = np.append(espec_i, cdf.varget(espec_i_vname), 0)
Ei = np.append(Ei, cdf.varget(Ei_vname), 0)
# FPI DES
fpi_mode = 'fast'
api.optdesc = 'des-moms'
files = api.download_files()
te = np.empty(0, dtype='datetime64')
ne = np.empty(0)
espec_e = np.empty((0, 32))
Ee = np.empty((0, 32))
te = np.empty(0)
t_vname = 'Epoch'
ne_vname = '_'.join((sc, 'des', 'numberdensity', fpi_mode))
espec_e_vname = '_'.join((sc, 'des', 'energyspectr', 'omni', fpi_mode))
Ee_vname = '_'.join((sc, 'des', 'energy', fpi_mode))
for file in files:
cdf = cdfread.CDF(file)
# tepoch = epochs.CDFepoch()
time = cdf.varget(t_vname)
te = np.append(te, tepoch.to_datetime(time, to_np=True), 0)
ne = np.append(ne, cdf.varget(ne_vname), 0)
espec_e = np.append(espec_e, cdf.varget(espec_e_vname), 0)
Ee = np.append(Ee, cdf.varget(Ee_vname), 0)
cdf.close()
# Create the figure
fig, axes = plt.subplots(ncols=1, nrows=7, figsize=(8, 9), sharex=True)
# Inset axes for colorbar
axins1 = inset_axes(axes[0],
width="5%",
height="80%",
loc='right',
bbox_to_anchor=(1.05, 0, 1, 1))
axins2 = inset_axes(axes[1],
width="5%",
height="80%",
loc='right',
bbox_to_anchor=(1.05, 0, 1, 1))
# FFT parameters -- resolve the oxygen gyrofrequency
im1 = axes[0].pcolormesh(np.tile(ti, (32, 1)).T,
Ei,
np.log10(espec_i),
cmap='nipy_spectral')
axes[0].set_xticklabels([])
axes[0].set_ylabel('ion E\n(eV)')
axes[0].set_yscale('log')
cbar1 = fig.colorbar(im1, cax=axins1)
cbar1.set_label('Flux')
axes[0].set_title('{} SITL Selections'.format(sc.upper()))
im2 = axes[1].pcolormesh(np.tile(te, (32, 1)).T,
Ee,
np.log10(espec_e),
cmap='nipy_spectral')
axes[1].set_xticklabels([])
axes[1].set_ylabel('elec E\n(ev)')
axes[1].set_yscale('log')
cbar2 = fig.colorbar(im2, ax=axins2)
cbar2.set_label('Flux')
axes[2].plot(ti, ni, color='blue', label='Ni')
axes[2].plot(te, ne, color='red', label='Ne')
axes[2].set_xticklabels([])
axes[2].set_ylabel('N\n(cm^3)')
axes[3].plot(t_fgm, b_fgm, label=['Bx', 'By', 'Bz', '|B|'])
axes[3].set_xticklabels([])
axes[3].set_ylabel('B\n(nT)')
# L_items = axes[3].get_legend().get_texts()
# L_items[0].set_text('Bx')
# L_items[1].set_text('By')
# L_items[2].set_text('Bz')
# L_items[3].set_text('|B|')
axes[4].plot(t_abs, x_abs)
axes[4].set_xticklabels([])
axes[4].set_ylabel('ABS')
axes[5].plot(t_sitl, x_sitl)
axes[5].set_xticklabels([])
axes[5].set_ylabel('SITL')
axes[6].plot(t_gls, x_gls)
axes[6].set_ylabel('GLS')
plt.setp(axes[6].xaxis.get_majorticklabels(), rotation=45)
plt.show()
pdb.set_trace()
return