def cdf_to_tplot(filenames, varformat=None, get_support_data=False,
prefix='', suffix='', plot=False, merge=False):
"""
This function will automatically create tplot variables from CDF files.
.. note::
Variables must have an attribute named "VAR_TYPE". If the attribute entry
is "data" (or "support_data"), then they will be added as tplot variables.
Additionally, data variables should have attributes named "DEPEND_TIME" or
"DEPEND_0" that describes which variable is x axis. If the data is 2D,
then an attribute "DEPEND_1" must describe which variable contains the
secondary axis.
Parameters:
filenames : str/list of str
The file names and full paths of CDF files.
varformat : str
The file variable formats to load into tplot. Wildcard character
"*" is accepted. By default, all variables are loaded in.
get_support_data: bool
Data with an attribute "VAR_TYPE" with a value of "support_data"
will be loaded into tplot. By default, only loads in data with a
"VAR_TYPE" attribute of "data".
prefix: str
The tplot variable names will be given this prefix. By default,
no prefix is added.
suffix: str
The tplot variable names will be given this suffix. By default,
no suffix is added.
plot: bool
The data is plotted immediately after being generated. All tplot
variables generated from this function will be on the same plot.
merge: bool
If True, then data from different cdf files will be merged into
a single pytplot variable.
Returns:
List of tplot variables created.
"""
stored_variables = []
global data_quants
if isinstance(filenames, str):
filenames = [filenames]
elif isinstance(filenames, list):
filenames = filenames
else:
print("Invalid filenames input.")
return stored_variables
var_type = ['data']
if varformat == None:
varformat = ".*"
if get_support_data:
var_type.append('support_data')
try:
varformat = varformat.replace("*", ".*")
var_regex = re.compile(varformat)
except:
print("Error reading the varformat.")
return
for filename in filenames:
cdf_file = cdflib.CDF(filename)
cdf_info = cdf_file.cdf_info()
all_cdf_variables = cdf_info['rVariables'] + cdf_info['zVariables']
# Find the data variables
for var in all_cdf_variables:
if not re.match(var_regex, var):
continue
var_atts = cdf_file.varattsget(var)
if 'VAR_TYPE' not in var_atts:
continue
if var_atts['VAR_TYPE'] in var_type:
var_properties = cdf_file.varinq(var)
if "DEPEND_TIME" in var_atts:
x_axis_var = var_atts["DEPEND_TIME"]
elif "DEPEND_0" in var_atts:
x_axis_var = var_atts["DEPEND_0"]
else:
print("Cannot find x axis.")
print("No attribute named DEPEND_TIME or DEPEND_0 in variable " + var)
continue
data_type_description = cdf_file.varinq(x_axis_var)['Data_Type_Description']
# Find data name and if it is already in stored variables
var_name = prefix + var + suffix
to_merge = False
if (var_name in data_quants.keys()) and (merge == True):
prev_data_quant = data_quants[var_name].data
to_merge = True
xdata = cdf_file.varget(x_axis_var)
if ('CDF_TIME' in data_type_description) or ('CDF_EPOCH' in data_type_description):
xdata = cdflib.cdfepoch.unixtime(xdata)
ydata = cdf_file.varget(var)
if ydata is None:
continue
if "FILLVAL" in var_atts:
if (var_properties['Data_Type_Description'] == 'CDF_FLOAT' or
var_properties['Data_Type_Description'] == 'CDF_REAL4' or
var_properties['Data_Type_Description'] == 'CDF_DOUBLE' or
var_properties['Data_Type_Description'] == 'CDF_REAL8'):
if ydata[ydata == var_atts["FILLVAL"]].size != 0:
ydata[ydata == var_atts["FILLVAL"]] = np.nan
tplot_data = {'x': xdata, 'y': ydata}
depend_1 = None
depend_2 = None
if "DEPEND_1" in var_atts:
if var_atts["DEPEND_1"] in all_cdf_variables:
depend_1 = cdf_file.varget(var_atts["DEPEND_1"])
if "DEPEND_2" in var_atts:
if var_atts["DEPEND_2"] in all_cdf_variables:
depend_2 = cdf_file.varget(var_atts["DEPEND_2"])
if (depend_1 is not None) and (depend_2 is not None):
tplot_data['v1'] = depend_1
tplot_data['v2'] = depend_2
elif depend_1 is not None:
tplot_data['v'] = depend_1
elif depend_2 is not None:
tplot_data['v'] = depend_2
store_data(var_name, data=tplot_data)
if var_name not in stored_variables:
stored_variables.append(var_name)
display_type = var_atts.get("DISPLAY_TYPE", "time_series")
scale_type = var_atts.get("SCALE_TYP", "linear")
if display_type == "spectrogram":
options(var, 'spec', 1)
if scale_type == 'log':
options(var, 'ylog', 1)
if to_merge:
cur_data_quant = data_quants[var_name].data
merged_data = [prev_data_quant, cur_data_quant]
data_quants[var_name].data = pd.concat(merged_data)
cdf_file.close() if hasattr(cdf_file, "close") else None
if plot:
tplot(stored_variables)
return stored_variables
def tplot_restore(filename):
"""
This function will restore tplot variables that have been saved with the "tplot_save" command.
.. note::
This function is compatible with the IDL tplot_save routine.
If you have a ".tplot" file generated from IDL, this procedure will restore the data contained in the file.
Not all plot options will transfer over at this time.
Parameters:
filename : str
The file name and full path generated by the "tplot_save" command.
Returns:
None
Examples:
>>> # Restore the saved data from the tplot_save example
>>> import pytplot
>>> pytplot.restore('C:/temp/variable1.pytplot')
"""
#Error check
if not (os.path.isfile(filename)):
print("Not a valid file name")
return
#Check if the restored file was an IDL file
if filename.endswith('.tplot'):
temp_tplot = readsav(filename)
for i in range(len(temp_tplot['dq'])):
data_name = temp_tplot['dq'][i][0].decode("utf-8")
temp_x_data = temp_tplot['dq'][i][1][0][0].squeeze()
#Pandas reads in data the other way I guess
if len(temp_tplot['dq'][i][1][0][2].shape) == 2:
temp_y_data = np.transpose(temp_tplot['dq'][i][1][0][2])
else:
temp_y_data = temp_tplot['dq'][i][1][0][2]
#If there are more than 4 fields, that means it is a spectrogram
if len(temp_tplot['dq'][i][1][0]) > 4:
temp_v_data = temp_tplot['dq'][i][1][0][4]
#Change from little endian to big endian, since pandas apparently hates little endian
#We might want to move this into the store_data procedure eventually
if (temp_x_data.dtype.byteorder == '>'):
temp_x_data = temp_x_data.byteswap().newbyteorder()
if (temp_y_data.dtype.byteorder == '>'):
temp_y_data = temp_y_data.byteswap().newbyteorder()
if (temp_v_data.dtype.byteorder == '>'):
temp_v_data = temp_v_data.byteswap().newbyteorder()
store_data(data_name,
data={
'x': temp_x_data,
'y': temp_y_data,
'v': temp_v_data
})
else:
#Change from little endian to big endian, since pandas apparently hates little endian
#We might want to move this into the store_data procedure eventually
if (temp_x_data.dtype.byteorder == '>'):
temp_x_data = temp_x_data.byteswap().newbyteorder()
if (temp_y_data.dtype.byteorder == '>'):
temp_y_data = temp_y_data.byteswap().newbyteorder()
store_data(data_name,
data={
'x': temp_x_data,
'y': temp_y_data
})
if temp_tplot['dq'][i][3].dtype.names is not None:
for option_name in temp_tplot['dq'][i][3].dtype.names:
options(data_name, option_name,
temp_tplot['dq'][i][3][option_name][0])
data_quants[data_name].attrs['plot_options'][
'trange'] = temp_tplot['dq'][i][4].tolist()
data_quants[data_name].attrs['plot_options'][
'create_time'] = temp_tplot['dq'][i][6]
for option_name in temp_tplot['tv'][0][0].dtype.names:
if option_name == 'TRANGE':
tplot_options('x_range',
temp_tplot['tv'][0][0][option_name][0])
if option_name == 'WSIZE':
tplot_options('wsize',
temp_tplot['tv'][0][0][option_name][0])
if option_name == 'VAR_LABEL':
tplot_options('var_label',
temp_tplot['tv'][0][0][option_name][0])
if 'P' in temp_tplot['tv'][0][1].tolist():
for option_name in temp_tplot['tv'][0][1]['P'][0].dtype.names:
if option_name == 'TITLE':
tplot_options(
'title',
temp_tplot['tv'][0][1]['P'][0][option_name][0])
#temp_tplot['tv'][0][1] is all of the "settings" variables
#temp_tplot['tv'][0][1]['D'][0] is "device" options
#temp_tplot['tv'][0][1]['P'][0] is "plot" options
#temp_tplot['tv'][0][1]['X'][0] is x axis options
#temp_tplot['tv'][0][1]['Y'][0] is y axis options
####################################################################
else:
temp = pickle.load(open(filename, "rb"))
num_data_quants = temp[0]
for i in range(0, num_data_quants):
data_quants[temp[i + 1].name] = temp[i + 1]
tplot_opt_glob = temp[num_data_quants + 1]
return
def tplot_restore(filename):
"""
This function will restore tplot variables that have been saved with the "tplot_save" command.
.. note::
This function is compatible with the IDL tplot_save routine.
If you have a ".tplot" file generated from IDL, this procedure will restore the data contained in the file.
Not all plot options will transfer over at this time.
Parameters:
filename : str
The file name and full path generated by the "tplot_save" command.
Returns:
None
Examples:
>>> # Restore the saved data from the tplot_save example
>>> import pytplot
>>> pytplot.tplot_restore('C:/temp/variable1.pytplot')
"""
#Error check
if not (os.path.isfile(filename)):
print("Not a valid file name")
return
#Check if the restored file was an IDL file
if filename.endswith('.tplot'):
temp_tplot = readsav(filename)
for i in range(len(temp_tplot['dq'])):
if isinstance(temp_tplot['dq'][i][0], str):
print(
"Error reading variable; this error occurs when the variable wasn't loaded in IDL when the SAV file was created."
)
continue
data_name = temp_tplot['dq'][i][0].decode("utf-8")
temp_x_data = temp_tplot['dq'][i][1][0][0].squeeze()
#Pandas reads in data the other way I guess
if len(temp_tplot['dq'][i][1][0][2].shape) == 4:
temp_y_data = np.transpose(temp_tplot['dq'][i][1][0][2],
axes=(3, 2, 1, 0))
elif len(temp_tplot['dq'][i][1][0][2].shape) == 3:
temp_y_data = np.transpose(temp_tplot['dq'][i][1][0][2],
axes=(2, 1, 0))
elif len(temp_tplot['dq'][i][1][0][2].shape) == 2:
temp_y_data = np.transpose(temp_tplot['dq'][i][1][0][2])
else:
temp_y_data = temp_tplot['dq'][i][1][0][2]
# variable contains V1, V2 and V3 (e.g., DF as a function of energy, theta, phi)
if len(temp_tplot['dq'][i][1][0]) == 10:
temp_v1_data = temp_tplot['dq'][i][1][0][4]
temp_v2_data = temp_tplot['dq'][i][1][0][6]
temp_v3_data = temp_tplot['dq'][i][1][0][8]
#Change from little endian to big endian, since pandas apparently hates little endian
#We might want to move this into the store_data procedure eventually
if (temp_x_data.dtype.byteorder == '>'):
temp_x_data = temp_x_data.byteswap().newbyteorder()
if (temp_y_data.dtype.byteorder == '>'):
temp_y_data = temp_y_data.byteswap().newbyteorder()
if (temp_v1_data.dtype.byteorder == '>'):
temp_v1_data = temp_v1_data.byteswap().newbyteorder()
if (temp_v2_data.dtype.byteorder == '>'):
temp_v2_data = temp_v2_data.byteswap().newbyteorder()
if (temp_v3_data.dtype.byteorder == '>'):
temp_v3_data = temp_v3_data.byteswap().newbyteorder()
# support time-varying depends
if len(temp_v1_data.shape) == 2:
temp_v1_data = np.transpose(temp_v1_data)
if len(temp_v2_data.shape) == 2:
temp_v2_data = np.transpose(temp_v2_data)
if len(temp_v3_data.shape) == 2:
temp_v3_data = np.transpose(temp_v3_data)
store_data(data_name,
data={
'x': temp_x_data,
'y': temp_y_data,
'v1': temp_v1_data,
'v2': temp_v2_data,
'v3': temp_v3_data
})
# variable contains V1, V2 (e.g., DF as a function of energy, angle)
elif len(temp_tplot['dq'][i][1][0]) == 8:
temp_v1_data = temp_tplot['dq'][i][1][0][4]
temp_v2_data = temp_tplot['dq'][i][1][0][6]
#Change from little endian to big endian, since pandas apparently hates little endian
#We might want to move this into the store_data procedure eventually
if (temp_x_data.dtype.byteorder == '>'):
temp_x_data = temp_x_data.byteswap().newbyteorder()
if (temp_y_data.dtype.byteorder == '>'):
temp_y_data = temp_y_data.byteswap().newbyteorder()
if (temp_v1_data.dtype.byteorder == '>'):
temp_v1_data = temp_v1_data.byteswap().newbyteorder()
if (temp_v2_data.dtype.byteorder == '>'):
temp_v2_data = temp_v2_data.byteswap().newbyteorder()
# support time-varying depends
if len(temp_v1_data.shape) == 2:
temp_v1_data = np.transpose(temp_v1_data)
if len(temp_v2_data.shape) == 2:
temp_v2_data = np.transpose(temp_v2_data)
store_data(data_name,
data={
'x': temp_x_data,
'y': temp_y_data,
'v1': temp_v1_data,
'v2': temp_v2_data
})
#If there are 4 fields, that means it is a spectrogram
# 6 fields is a spectrogram with a time varying Y axis
elif len(temp_tplot['dq'][i][1][0]) == 5 or len(
temp_tplot['dq'][i][1][0]) == 6:
temp_v_data = temp_tplot['dq'][i][1][0][4]
#Change from little endian to big endian, since pandas apparently hates little endian
#We might want to move this into the store_data procedure eventually
if (temp_x_data.dtype.byteorder == '>'):
temp_x_data = temp_x_data.byteswap().newbyteorder()
if (temp_y_data.dtype.byteorder == '>'):
temp_y_data = temp_y_data.byteswap().newbyteorder()
if (temp_v_data.dtype.byteorder == '>'):
temp_v_data = temp_v_data.byteswap().newbyteorder()
# support time-varying depends
if len(temp_v_data.shape) == 2:
temp_v_data = np.transpose(temp_v_data)
store_data(data_name,
data={
'x': temp_x_data,
'y': temp_y_data,
'v': temp_v_data
})
else:
#Change from little endian to big endian, since pandas apparently hates little endian
#We might want to move this into the store_data procedure eventually
if (temp_x_data.dtype.byteorder == '>'):
temp_x_data = temp_x_data.byteswap().newbyteorder()
if (temp_y_data.dtype.byteorder == '>'):
temp_y_data = temp_y_data.byteswap().newbyteorder()
store_data(data_name,
data={
'x': temp_x_data,
'y': temp_y_data
})
if temp_tplot['dq'][i][3].dtype.names is not None:
for option_name in temp_tplot['dq'][i][3].dtype.names:
options(data_name, option_name,
temp_tplot['dq'][i][3][option_name][0])
pytplot.data_quants[data_name].attrs['plot_options'][
'trange'] = temp_tplot['dq'][i][4].tolist()
pytplot.data_quants[data_name].attrs['plot_options'][
'create_time'] = temp_tplot['dq'][i][6]
for option_name in temp_tplot['tv'][0][0].dtype.names:
if option_name == 'TRANGE':
# x_range of [0, 0] causes tplot to create an empty figure
if temp_tplot['tv'][0][0][option_name][0][
0] != 0 or temp_tplot['tv'][0][0][option_name][0][
1] != 0:
tplot_options('x_range',
temp_tplot['tv'][0][0][option_name][0])
if option_name == 'WSIZE':
tplot_options('wsize',
temp_tplot['tv'][0][0][option_name][0])
if option_name == 'VAR_LABEL':
tplot_options('var_label',
temp_tplot['tv'][0][0][option_name][0])
if 'P' in temp_tplot['tv'][0][1].tolist():
for option_name in temp_tplot['tv'][0][1]['P'][0].dtype.names:
if option_name == 'TITLE':
tplot_options(
'title',
temp_tplot['tv'][0][1]['P'][0][option_name][0])
#temp_tplot['tv'][0][1] is all of the "settings" variables
#temp_tplot['tv'][0][1]['D'][0] is "device" options
#temp_tplot['tv'][0][1]['P'][0] is "plot" options
#temp_tplot['tv'][0][1]['X'][0] is x axis options
#temp_tplot['tv'][0][1]['Y'][0] is y axis options
####################################################################
else:
in_file = open(filename, "rb")
temp = pickle.load(in_file)
num_data_quants = temp[0]
for i in range(0, num_data_quants):
if isinstance(temp[i + 1], dict):
# NRV variable
pytplot.data_quants[temp[i + 1]['name']] = temp[i + 1]
else:
pytplot.data_quants[temp[i + 1].name] = temp[i + 1]
pytplot.tplot_opt_glob = temp[num_data_quants + 1]
in_file.close()
return
def cdf_to_tplot(filenames, varformat=None, get_support_data=False,
prefix='', suffix='', plot=False, merge=False,
center_measurement=False, notplot=False):
"""
This function will automatically create tplot variables from CDF files.
.. note::
Variables must have an attribute named "VAR_TYPE". If the attribute entry
is "data" (or "support_data"), then they will be added as tplot variables.
Additionally, data variables should have attributes named "DEPEND_TIME" or
"DEPEND_0" that describes which variable is x axis. If the data is 2D,
then an attribute "DEPEND_1" must describe which variable contains the
secondary axis.
Parameters:
filenames : str/list of str
The file names and full paths of CDF files.
varformat : str
The file variable formats to load into tplot. Wildcard character
"*" is accepted. By default, all variables are loaded in.
get_support_data: bool
Data with an attribute "VAR_TYPE" with a value of "support_data"
will be loaded into tplot. By default, only loads in data with a
"VAR_TYPE" attribute of "data".
prefix: str
The tplot variable names will be given this prefix. By default,
no prefix is added.
suffix: str
The tplot variable names will be given this suffix. By default,
no suffix is added.
plot: bool
The data is plotted immediately after being generated. All tplot
variables generated from this function will be on the same plot.
merge: bool
If True, then data from different cdf files will be merged into
a single pytplot variable.
center_measurement: bool
If True, the CDF epoch variables are time-shifted to the middle
of the accumulation interval by their DELTA_PLUS_VAR and
DELTA_MINUS_VAR variable attributes
notplot: bool
If True, then data are returned in a hash table instead of
being stored in tplot variables (useful for debugging, and
access to multi-dimensional data products)
Returns:
List of tplot variables created (unless notplot keyword is used).
"""
stored_variables = []
epoch_cache = {}
output_table = {}
metadata = {}
data_quants = {}
if isinstance(filenames, str):
filenames = [filenames]
elif isinstance(filenames, list):
filenames = filenames
else:
print("Invalid filenames input.")
return stored_variables
var_type = ['data']
if varformat is None:
varformat = ".*"
if get_support_data:
var_type.append('support_data')
varformat = varformat.replace("*", ".*")
var_regex = re.compile(varformat)
for filename in filenames:
cdf_file = cdflib.CDF(filename)
cdf_info = cdf_file.cdf_info()
all_cdf_variables = cdf_info['rVariables'] + cdf_info['zVariables']
# Find the data variables
for var in all_cdf_variables:
if not re.match(var_regex, var):
continue
var_atts = cdf_file.varattsget(var)
if 'VAR_TYPE' not in var_atts:
continue
if var_atts['VAR_TYPE'] in var_type:
var_atts = cdf_file.varattsget(var)
var_properties = cdf_file.varinq(var)
if "DEPEND_TIME" in var_atts:
x_axis_var = var_atts["DEPEND_TIME"]
elif "DEPEND_0" in var_atts:
x_axis_var = var_atts["DEPEND_0"]
else:
if var_atts['VAR_TYPE'].lower() == 'data':
print("Cannot find x axis.")
print("No attribute named DEPEND_TIME or DEPEND_0 in \
variable " + var)
continue
data_type_description \
= cdf_file.varinq(x_axis_var)['Data_Type_Description']
# Find data name and if it is already in stored variables
var_name = prefix + var + suffix
if epoch_cache.get(filename+x_axis_var) is None:
delta_plus_var = 0.0
delta_minus_var = 0.0
delta_time = 0.0
xdata = cdf_file.varget(x_axis_var)
epoch_var_atts = cdf_file.varattsget(x_axis_var)
# check for DELTA_PLUS_VAR/DELTA_MINUS_VAR attributes
if center_measurement:
if 'DELTA_PLUS_VAR' in epoch_var_atts:
delta_plus_var = cdf_file.varget(epoch_var_atts['DELTA_PLUS_VAR'])
delta_plus_var_att = cdf_file.varattsget(epoch_var_atts['DELTA_PLUS_VAR'])
# check if a conversion to seconds is required
if 'SI_CONVERSION' in delta_plus_var_att:
si_conv = delta_plus_var_att['SI_CONVERSION']
delta_plus_var = delta_plus_var.astype(float)*np.float(si_conv.split('>')[0])
elif 'SI_CONV' in delta_plus_var_att:
si_conv = delta_plus_var_att['SI_CONV']
delta_plus_var = delta_plus_var.astype(float)*np.float(si_conv.split('>')[0])
if 'DELTA_MINUS_VAR' in epoch_var_atts:
delta_minus_var = cdf_file.varget(epoch_var_atts['DELTA_MINUS_VAR'])
delta_minus_var_att = cdf_file.varattsget(epoch_var_atts['DELTA_MINUS_VAR'])
# check if a conversion to seconds is required
if 'SI_CONVERSION' in delta_minus_var_att:
si_conv = delta_minus_var_att['SI_CONVERSION']
delta_minus_var = delta_minus_var.astype(float)*np.float(si_conv.split('>')[0])
elif 'SI_CONV' in delta_minus_var_att:
si_conv = delta_minus_var_att['SI_CONV']
delta_minus_var = delta_minus_var.astype(float)*np.float(si_conv.split('>')[0])
# sometimes these are specified as arrays
if isinstance(delta_plus_var, np.ndarray) and isinstance(delta_minus_var, np.ndarray):
delta_time = (delta_plus_var-delta_minus_var)/2.0
else: # and sometimes constants
if delta_plus_var != 0.0 or delta_minus_var != 0.0:
delta_time = (delta_plus_var-delta_minus_var)/2.0
if epoch_cache.get(filename + x_axis_var) is None:
if ('CDF_TIME' in data_type_description) or \
('CDF_EPOCH' in data_type_description):
xdata = cdfepoch.unixtime(xdata)
epoch_cache[filename+x_axis_var] = np.array(xdata)+delta_time
else:
xdata = epoch_cache[filename + x_axis_var]
try:
ydata = cdf_file.varget(var)
except:
continue
if ydata is None:
continue
if "FILLVAL" in var_atts:
if (var_properties['Data_Type_Description'] ==
'CDF_FLOAT' or
var_properties['Data_Type_Description'] ==
'CDF_REAL4' or
var_properties['Data_Type_Description'] ==
'CDF_DOUBLE' or
var_properties['Data_Type_Description'] ==
'CDF_REAL8'):
if ydata[ydata == var_atts["FILLVAL"]].size != 0:
ydata[ydata == var_atts["FILLVAL"]] = np.nan
tplot_data = {'x': xdata, 'y': ydata}
depend_1 = None
depend_2 = None
depend_3 = None
if "DEPEND_1" in var_atts:
if var_atts["DEPEND_1"] in all_cdf_variables:
depend_1 = np.array(cdf_file.varget(var_atts["DEPEND_1"]))
if "DEPEND_2" in var_atts:
if var_atts["DEPEND_2"] in all_cdf_variables:
depend_2 = np.array(cdf_file.varget(var_atts["DEPEND_2"]))
if "DEPEND_3" in var_atts:
if var_atts["DEPEND_3"] in all_cdf_variables:
depend_3 = np.array(cdf_file.varget(var_atts["DEPEND_3"]))
nontime_varying_depends = []
if depend_1 is not None and depend_2 is not None and depend_3 is not None:
tplot_data['v1'] = depend_1
tplot_data['v2'] = depend_2
tplot_data['v3'] = depend_3
if len(depend_1.shape) == 1:
nontime_varying_depends.append('v1')
if len(depend_2.shape) == 1:
nontime_varying_depends.append('v2')
if len(depend_3.shape) == 1:
nontime_varying_depends.append('v3')
elif depend_1 is not None and depend_2 is not None:
tplot_data['v1'] = depend_1
tplot_data['v2'] = depend_2
if len(depend_1.shape) == 1:
nontime_varying_depends.append('v1')
if len(depend_2.shape) == 1:
nontime_varying_depends.append('v2')
elif depend_1 is not None:
tplot_data['v'] = depend_1
if len(depend_1.shape) == 1:
nontime_varying_depends.append('v')
elif depend_2 is not None:
tplot_data['v'] = depend_2
if len(depend_2.shape) == 1:
nontime_varying_depends.append('v')
metadata[var_name] = {'display_type': var_atts.get("DISPLAY_TYPE", "time_series"),
'scale_type': var_atts.get("SCALE_TYP", "linear")}
if var_name not in output_table:
output_table[var_name] = tplot_data
else:
var_data = output_table[var_name]
for output_var in var_data:
if output_var not in nontime_varying_depends:
var_data[output_var] = np.concatenate((var_data[output_var], tplot_data[output_var]))
if notplot:
return output_table
for var_name in output_table.keys():
to_merge = False
if var_name in data_quants.keys() and merge:
prev_data_quant = data_quants[var_name]
to_merge = True
try:
store_data(var_name, data=output_table[var_name])
except ValueError:
continue
if var_name not in stored_variables:
stored_variables.append(var_name)
if metadata.get(var_name) is not None:
if metadata[var_name]['display_type'] == "spectrogram":
options(var_name, 'spec', 1)
if metadata[var_name]['scale_type'] == 'log':
options(var_name, 'ylog', 1)
if to_merge is True:
cur_data_quant = data_quants[var_name]
plot_options = copy.deepcopy(data_quants[var_name].attrs['plot_options'])
data_quants[var_name] = xr.concat([prev_data_quant, cur_data_quant], dim='time')
data_quants[var_name].attrs['plot_options'] = plot_options
if notplot:
return output_table
if plot:
tplot(stored_variables)
return stored_variables
def cdf_to_tplot(filenames,
varformat=None,
get_support_data=False,
get_ignore_data=False,
string_encoding='ascii',
prefix='',
suffix='',
plot=False,
merge=False,
center_measurement=False,
notplot=False,
varnames=[]):
"""
This function will automatically create tplot variables from CDF files. In general, the files should be
ISTP compliant for this importer to work. Each variable is read into a new tplot variable (a.k.a an xarray DataArray),
and all associated file/variable metadata is read into the attrs dictionary.
.. note::
Variables must have an attribute named "VAR_TYPE". If the attribute entry
is "data" (or "support_data"), then they will be added as tplot variables.
Additionally, data variables should have attributes named "DEPEND_TIME" or
"DEPEND_0" that describes which variable is x axis. If the data is 2D,
then an attribute "DEPEND_1" must describe which variable contains the
secondary axis.
Parameters:
filenames : str/list of str
The file names and full paths of CDF files.
varformat : str
The file variable formats to load into tplot. Wildcard character
"*" is accepted. By default, all variables are loaded in.
get_support_data: bool
Data with an attribute "VAR_TYPE" with a value of "support_data"
will be loaded into tplot. By default, only loads in data with a
"VAR_TYPE" attribute of "data".
prefix: str
The tplot variable names will be given this prefix. By default,
no prefix is added.
suffix: str
The tplot variable names will be given this suffix. By default,
no suffix is added.
plot: bool
The data is plotted immediately after being generated. All tplot
variables generated from this function will be on the same plot.
merge: bool
If True, then data from different cdf files will be merged into
a single pytplot variable.
get_ignore_data: bool
Data with an attribute "VAR_TYPE" with a value of "ignore_data"
will be loaded into tplot. By default, only loads in data with a
"VAR_TYPE" attribute of "data".
center_measurement: bool
If True, the CDF epoch variables are time-shifted to the middle
of the accumulation interval by their DELTA_PLUS_VAR and
DELTA_MINUS_VAR variable attributes
notplot: bool
If True, then data are returned in a hash table instead of
being stored in tplot variables (useful for debugging, and
access to multi-dimensional data products)
varnames: str or list of str
Load these variables only. If [] or ['*'], then load everything.
Returns:
List of tplot variables created (unless notplot keyword is used).
"""
stored_variables = []
epoch_cache = {}
output_table = {}
metadata = {}
if not isinstance(varnames, list):
varnames = [varnames]
if len(varnames) > 0:
if '*' in varnames:
varnames = []
# pytplot.data_quants = {}
if isinstance(filenames, str):
filenames = [filenames]
elif isinstance(filenames, list):
filenames = filenames
else:
print("Invalid filenames input.")
return stored_variables
var_type = ['data']
if varformat is None:
varformat = ".*"
if get_support_data:
var_type.append('support_data')
if get_ignore_data:
var_type.append('ignore_data')
varformat = varformat.replace("*", ".*")
var_regex = re.compile(varformat)
filenames.sort()
for filename in filenames:
cdf_file = cdflib.CDF(filename)
cdf_file.string_encoding = string_encoding
cdf_info = cdf_file.cdf_info()
all_cdf_variables = cdf_info['rVariables'] + cdf_info['zVariables']
# User defined variables.
if len(varnames) > 0:
load_cdf_variables = [
value for value in varnames if value in all_cdf_variables
]
else:
load_cdf_variables = all_cdf_variables
try:
gatt = cdf_file.globalattsget()
except:
gatt = {}
for var in load_cdf_variables:
if not re.match(var_regex, var):
continue
var_atts = cdf_file.varattsget(var)
if 'VAR_TYPE' in var_atts:
this_var_type = var_atts['VAR_TYPE'].lower()
elif 'PARAMETER_TYPE' in var_atts:
this_var_type = var_atts['PARAMETER_TYPE'].lower()
else:
# 'VAR_TYPE' and 'PARAMETER_TYPE' not found in the variable attributes
continue
if this_var_type in var_type:
var_atts = cdf_file.varattsget(var)
var_properties = cdf_file.varinq(var)
# Find data name and if it is already in stored variables
if 'TPLOT_NAME' in var_atts:
var_name = prefix + var_atts['TPLOT_NAME'] + suffix
else:
var_name = prefix + var + suffix
if "DEPEND_TIME" in var_atts:
x_axis_var = var_atts["DEPEND_TIME"]
elif "DEPEND_0" in var_atts:
x_axis_var = var_atts["DEPEND_0"]
else:
# non-record varying variables (NRVs)
# added by egrimes, 13Jan2021
# here we assume if there isn't a DEPEND_TIME or DEPEND_0, there are no other depends
try:
ydata = cdf_file.varget(var)
except:
continue
if ydata is None:
continue
# since NRVs don't vary with time, they shouldn't vary across files
output_table[var_name] = {'y': ydata}
continue
data_type_description \
= cdf_file.varinq(x_axis_var)['Data_Type_Description']
if epoch_cache.get(filename + x_axis_var) is None:
delta_plus_var = 0.0
delta_minus_var = 0.0
delta_time = 0.0
# Skip variables with ValueErrors.
try:
xdata = cdf_file.varget(x_axis_var)
epoch_var_atts = cdf_file.varattsget(x_axis_var)
except ValueError:
continue
# check for DELTA_PLUS_VAR/DELTA_MINUS_VAR attributes
if center_measurement:
if 'DELTA_PLUS_VAR' in epoch_var_atts:
delta_plus_var = cdf_file.varget(
epoch_var_atts['DELTA_PLUS_VAR'])
delta_plus_var_att = cdf_file.varattsget(
epoch_var_atts['DELTA_PLUS_VAR'])
# check if a conversion to seconds is required
if 'SI_CONVERSION' in delta_plus_var_att:
si_conv = delta_plus_var_att['SI_CONVERSION']
delta_plus_var = delta_plus_var.astype(
float) * np.float(si_conv.split('>')[0])
elif 'SI_CONV' in delta_plus_var_att:
si_conv = delta_plus_var_att['SI_CONV']
delta_plus_var = delta_plus_var.astype(
float) * np.float(si_conv.split('>')[0])
if 'DELTA_MINUS_VAR' in epoch_var_atts:
delta_minus_var = cdf_file.varget(
epoch_var_atts['DELTA_MINUS_VAR'])
delta_minus_var_att = cdf_file.varattsget(
epoch_var_atts['DELTA_MINUS_VAR'])
# check if a conversion to seconds is required
if 'SI_CONVERSION' in delta_minus_var_att:
si_conv = delta_minus_var_att['SI_CONVERSION']
delta_minus_var = delta_minus_var.astype(
float) * np.float(si_conv.split('>')[0])
elif 'SI_CONV' in delta_minus_var_att:
si_conv = delta_minus_var_att['SI_CONV']
delta_minus_var = delta_minus_var.astype(
float) * np.float(si_conv.split('>')[0])
# sometimes these are specified as arrays
if isinstance(delta_plus_var,
np.ndarray) and isinstance(
delta_minus_var, np.ndarray):
delta_time = (delta_plus_var -
delta_minus_var) / 2.0
else: # and sometimes constants
if delta_plus_var != 0.0 or delta_minus_var != 0.0:
delta_time = (delta_plus_var -
delta_minus_var) / 2.0
if epoch_cache.get(filename + x_axis_var) is None:
if ('CDF_TIME' in data_type_description) or \
('CDF_EPOCH' in data_type_description):
xdata = cdfepoch.unixtime(xdata)
epoch_cache[filename +
x_axis_var] = np.array(xdata) + delta_time
else:
xdata = epoch_cache[filename + x_axis_var]
try:
ydata = cdf_file.varget(var)
except:
continue
if ydata is None:
continue
if "FILLVAL" in var_atts:
if (var_properties['Data_Type_Description'] == 'CDF_FLOAT'
or var_properties['Data_Type_Description']
== 'CDF_REAL4'
or var_properties['Data_Type_Description']
== 'CDF_DOUBLE'
or var_properties['Data_Type_Description']
== 'CDF_REAL8'):
if ydata[ydata == var_atts["FILLVAL"]].size != 0:
ydata[ydata == var_atts["FILLVAL"]] = np.nan
elif var_properties[
'Data_Type_Description'][:7] == 'CDF_INT':
# NaN is only valid for floating point data
# but we still need to handle FILLVAL's for
# integer data, so we'll just set those to 0
ydata[ydata == var_atts["FILLVAL"]] = 0
tplot_data = {'x': xdata, 'y': ydata}
# Data may depend on other data in the CDF.
depend_1 = None
depend_2 = None
depend_3 = None
if "DEPEND_1" in var_atts:
if var_atts["DEPEND_1"] in all_cdf_variables:
depend_1 = np.array(
cdf_file.varget(var_atts["DEPEND_1"]))
# Ignore the depend types if they are strings
if depend_1.dtype.type is np.str_:
depend_1 = None
if "DEPEND_2" in var_atts:
if var_atts["DEPEND_2"] in all_cdf_variables:
depend_2 = np.array(
cdf_file.varget(var_atts["DEPEND_2"]))
# Ignore the depend types if they are strings
if depend_2.dtype.type is np.str_:
depend_2 = None
if "DEPEND_3" in var_atts:
if var_atts["DEPEND_3"] in all_cdf_variables:
depend_3 = np.array(
cdf_file.varget(var_atts["DEPEND_3"]))
# Ignore the depend types if they are strings
if depend_3.dtype.type is np.str_:
depend_3 = None
nontime_varying_depends = []
if depend_1 is not None and depend_2 is not None and depend_3 is not None:
tplot_data['v1'] = depend_1
tplot_data['v2'] = depend_2
tplot_data['v3'] = depend_3
if len(depend_1.shape) == 1:
nontime_varying_depends.append('v1')
if len(depend_2.shape) == 1:
nontime_varying_depends.append('v2')
if len(depend_3.shape) == 1:
nontime_varying_depends.append('v3')
elif depend_1 is not None and depend_2 is not None:
tplot_data['v1'] = depend_1
tplot_data['v2'] = depend_2
if len(depend_1.shape) == 1:
nontime_varying_depends.append('v1')
if len(depend_2.shape) == 1:
nontime_varying_depends.append('v2')
elif depend_1 is not None:
tplot_data['v'] = depend_1
if len(depend_1.shape) == 1:
nontime_varying_depends.append('v')
elif depend_2 is not None:
tplot_data['v'] = depend_2
if len(depend_2.shape) == 1:
nontime_varying_depends.append('v')
metadata[var_name] = {
'display_type': var_atts.get("DISPLAY_TYPE",
"time_series"),
'scale_type': var_atts.get("SCALE_TYP", "linear"),
'var_attrs': var_atts,
'file_name': filename,
'global_attrs': gatt
}
# Check if the variable already exists in the for loop output
if var_name not in output_table:
output_table[var_name] = tplot_data
else:
# If it does, loop though the existing variable's x,y,v,v2,v3,etc
var_data = output_table[var_name]
for output_var in var_data:
if output_var not in nontime_varying_depends:
if np.asarray(tplot_data[output_var]
).ndim == 0 and np.equal(
tplot_data[output_var], None):
# If there is nothing in the new variable, then pass
pass
elif np.asarray(var_data[output_var]
).ndim == 0 and np.equal(
var_data[output_var], None):
# If there is nothing in the old variable, then replace
var_data[output_var] = tplot_data[output_var]
else: # If they both have something, then concatenate
var_data[output_var] = np.concatenate(
(var_data[output_var],
tplot_data[output_var]))
if notplot:
return output_table
for var_name in output_table.keys():
to_merge = False
if var_name in pytplot.data_quants.keys() and merge:
prev_data_quant = pytplot.data_quants[var_name]
to_merge = True
try:
attr_dict = {}
if metadata.get(var_name) is not None:
attr_dict["CDF"] = {}
attr_dict["CDF"]["VATT"] = metadata[var_name]['var_attrs']
attr_dict["CDF"]["GATT"] = metadata[var_name]['global_attrs']
attr_dict["CDF"]["FILENAME"] = metadata[var_name]['file_name']
# extract the coordinate system, if available
vatt_keys = list(attr_dict["CDF"]["VATT"].keys())
vatt_lower = [k.lower() for k in vatt_keys]
if 'coordinate_system' in vatt_lower:
attr_dict['data_att'] = {
'coord_sys':
attr_dict["CDF"]["VATT"][vatt_keys[vatt_lower.index(
'coordinate_system')]]
}
store_data(var_name,
data=output_table[var_name],
attr_dict=attr_dict)
except ValueError:
continue
if var_name not in stored_variables:
stored_variables.append(var_name)
if metadata.get(var_name) is not None:
if metadata[var_name]['display_type'] == "spectrogram":
options(var_name, 'spec', 1)
if metadata[var_name]['scale_type'] == 'log':
options(var_name, 'ylog', 1)
if metadata[var_name].get('var_attrs') is not None:
if metadata[var_name]['var_attrs'].get('LABLAXIS') is not None:
options(var_name, 'ytitle',
metadata[var_name]['var_attrs']['LABLAXIS'])
if metadata[var_name]['var_attrs'].get('UNITS') is not None:
if metadata[var_name]['display_type'] == 'spectrogram':
options(
var_name, 'ztitle', '[' +
metadata[var_name]['var_attrs']['UNITS'] + ']')
else:
options(
var_name, 'ysubtitle', '[' +
metadata[var_name]['var_attrs']['UNITS'] + ']')
# Gather up all options in the variable attribute section, toss them into options and see what sticks
options(var_name, opt_dict=metadata[var_name]['var_attrs'])
if to_merge is True:
cur_data_quant = pytplot.data_quants[var_name]
plot_options = copy.deepcopy(pytplot.data_quants[var_name].attrs)
pytplot.data_quants[var_name] = xr.concat(
[prev_data_quant, cur_data_quant], dim='time').sortby('time')
pytplot.data_quants[var_name].attrs = plot_options
if notplot:
return output_table
if plot:
tplot(stored_variables)
return stored_variables
