def mms_feeps_active_eyes(trange, probe, data_rate, species, level):
if data_rate.lower() == 'brst' and species.lower() == 'electron': return {'top': [1, 2, 3, 4, 5, 9, 10, 11, 12], 'bottom': [1, 2, 3, 4, 5, 9, 10, 11, 12]}
if data_rate.lower() == 'brst' and species.lower() == 'ion': return {'top': [6, 7, 8], 'bottom': [6, 7, 8]}
# old eyes, srvy mode, prior to 16 August 2017
if species.lower() == 'electron':
sensors = {'top': [3, 4, 5, 11, 12], 'bottom': [3, 4, 5, 11, 12]}
else:
sensors = {'top': [6, 7, 8], 'bottom': [6, 7, 8]}
if isinstance(trange[0], str):
start_time = time_double(trange[0])
else:
start_time = trange[0]
# srvy mode, after 16 August 2017
if start_time >= time_double('2017-08-16') and data_rate.lower() == 'srvy':
active_table = {'1-electron': {'top': [3, 5, 9, 10, 12], 'bottom': [2, 4, 5, 9, 10]},
'1-ion': {'top': [6, 7, 8], 'bottom': [6, 7, 8]},
'2-electron': {'top': [1, 2, 3, 5, 10, 11], 'bottom': [1, 4, 5, 9, 11]},
'2-ion': {'top': [6, 8], 'bottom': [6, 7, 8]},
'3-electron': {'top': [3, 5, 9, 10, 12], 'bottom': [1, 2, 3, 9, 10]},
'3-ion': {'top': [6, 7, 8], 'bottom': [6, 7, 8]},
'4-electron': {'top': [3, 4, 5, 9, 10, 11], 'bottom': [3, 5, 9, 10, 12]},
'4-ion': {'top': [6, 8], 'bottom': [6, 7, 8]}}
sensors = active_table[probe.lower()+'-'+species.lower()]
if level.lower() == 'sitl':
return {'top': list(set(sensors['top']) & set([5, 11, 12])), 'bottom': []}
if level.lower() == 'sitl':
return {'top': [5, 11, 12], 'bottom': []}
return sensors
def test_time_double(self):
self.assertTrue(
time_double('2015-12-15 12:07:23.767000') == 1450181243.767)
self.assertTrue(
time_double([
'2015-12-15 12:07:23.767000', '2015-12-15 12:07:43.767000'
]) == [1450181243.767, 1450181263.767])
def get_mms_srois(start_time=None, end_time=None, sc_id=None):
if start_time is None:
logging.error('Error, start time not specified')
return
if end_time is None:
logging.error('Error, end time not specified')
return
if sc_id is None:
logging.error('Error, sc_id not specified')
return
# public unauthenticated path
path = 'mms/sdc/public/service/latis/mms_events_view.csv'
query = '?start_time_utc,end_time_utc,sc_id,start_orbit&event_type=SROI'
# convert to standard ISO format as accepted by LaTiS
query += '&start_time_utc>=' + 'T'.join(start_time.split(' '))
query += '&start_time_utc<' + 'T'.join(end_time.split(' '))
query += '&sc_id=' + sc_id.lower()
qreq = requests.get('https://lasp.colorado.edu/' + path + query)
if qreq.status_code != 200:
logging.error('Error downloading SRoI segments')
return
sroi_data = csv.reader(qreq.content.decode('utf-8').splitlines())
count = 0
unix_start = []
unix_end = []
for line in sroi_data:
if count == 0:
# ignore the first line
count += 1
continue
if len(line) != 4:
continue
unix_start.append(time_double(line[0]))
unix_end.append(time_double(line[1]))
count += 1
return (np.array(unix_start), np.array(unix_end))
def dailynames(directory='',
trange=None,
res=24 * 3600.,
hour_res=False,
file_format='%Y%m%d',
prefix='',
suffix=''):
if trange == None:
print('No trange specified')
return
if hour_res == True:
res = 3600.
file_format = '%Y%m%d%H'
# allows the user to pass in trange as list of datetime objects
if type(trange[0]) == datetime and type(trange[1]) == datetime:
trange = [
time_string(trange[0].timestamp()),
time_string(trange[1].timestamp())
]
tr = [trange[0], trange[1]]
if isinstance(trange[0], str):
tr[0] = time_double(trange[0])
if isinstance(trange[1], str):
tr[1] = time_double(trange[1])
# Davin's magic heisted from file_dailynames in IDL
mmtr = [np.floor(tr[0] / res), np.ceil(tr[1] / res)]
if mmtr[1] - mmtr[0] < 1:
n = 1
else:
n = int(mmtr[1] - mmtr[0])
times = [(float(num) + mmtr[0]) * res for num in range(n)]
dates = []
files = []
for time in times:
if time_string(time, fmt=file_format) not in dates:
dates.append(time_string(time, fmt=file_format))
for date in dates:
files.append(directory + prefix + date + suffix)
return files
def get_params(model):
support_trange = [
time_double(trange[0]) - 60 * 60 * 24,
time_double(trange[1]) + 60 * 60 * 24
]
pyspedas.kyoto.dst(trange=support_trange)
pyspedas.omni.data(trange=trange)
join_vec(['BX_GSE', 'BY_GSM', 'BZ_GSM'])
return get_tsy_params('kyoto_dst',
'BX_GSE-BY_GSM-BZ_GSM_joined',
'proton_density',
'flow_speed',
model,
pressure_tvar='Pressure',
speed=True)
def mms_read_feeps_sector_masks_csv(trange):
"""
This function returns the FEEPS sectors to mask due to sunlight contamination
Parameters:
trange : list of str
time range of interest [starttime, endtime] with the format
'YYYY-MM-DD','YYYY-MM-DD'] or to specify more or less than a day
['YYYY-MM-DD/hh:mm:ss','YYYY-MM-DD/hh:mm:ss']
Returns:
Hash table containing the sectors to mask for each spacecraft and sensor ID
"""
masks = {}
dates = [
1447200000.0000000, # 11/11/2015
1468022400.0000000, # 7/9/2016
1477612800.0000000, # 10/28/2016
1496188800.0000000, # 5/31/2017
1506988800.0000000, # 10/3/2017
1538697600.0000000
] # 10/5/2018
# find the file closest to the start time
nearest_date = dates[(np.abs(np.array(dates) -
time_double(trange[0]))).argmin()]
for mms_sc in [1, 2, 3, 4]:
csv_file = os.sep.join([
os.path.dirname(os.path.abspath(__file__)), 'sun',
'MMS' + str(mms_sc) + '_FEEPS_ContaminatedSectors_' +
time_string(nearest_date, fmt='%Y%m%d') + '.csv'
])
csv_file = open(csv_file, 'r')
csv_reader = csv.reader(csv_file)
csv_data = []
for line in csv_reader:
csv_data.append([float(l) for l in line])
csv_file.close()
csv_data = np.array(csv_data)
for i in range(0, 12):
mask_vals = []
for val_idx in range(0, len(csv_data[:, i])):
if csv_data[val_idx, i] == 1: mask_vals.append(val_idx)
masks['mms' + str(mms_sc) + 'imaskt' + str(i + 1)] = mask_vals
for i in range(0, 12):
mask_vals = []
for val_idx in range(0, len(csv_data[:, i + 12])):
if csv_data[val_idx, i + 12] == 1: mask_vals.append(val_idx)
masks['mms' + str(mms_sc) + 'imaskb' + str(i + 1)] = mask_vals
return masks
def spd_mms_load_bss(trange=['2015-10-16', '2015-10-17'], datatype=['fast', 'burst'],
include_labels=False, probe='1', nodownload=False):
"""
Creates tplot variables which allow you to display horizontal color bars
indicating burst data availability.
Parameters
-----------
trange: list of str
Time frame for the bars
datatype: str or list of str
Type of BSS data (current valid options: 'fast', 'burst')
include_labels: bool
Flag to have horizontal bars labeled
probe: str or int
S/C probe # for SRoI bars (used as fast survey segments after 6Nov15; default: 1)
"""
if not isinstance(datatype, list):
datatype = [datatype]
if include_labels:
burst_label = 'Burst'
fast_label = 'Fast'
else:
burst_label = ''
fast_label = ''
for dtype in datatype:
if dtype == 'fast':
if time_double(trange[0]) <= time_double('2015-11-06'):
# use the old fast segments code for dates before 6Nov15
out = mms_load_fast_segments(trange=trange)
else:
# use SRoI code for dates on and after 6Nov15
out = mms_load_sroi_segments(trange=trange, probe=probe)
elif dtype == 'burst':
out = mms_load_brst_segments(trange=trange)
else:
print('Unsupported datatype: ' + dtype + '; valid options: "fast" and "burst"')
continue
if out is None:
print('Problem loading segments for ' + dtype)
def mms_load_fast_segments(trange=None, suffix=''):
'''
This function loads the fast segment intervals
Parameters:
trange : list of str
time range of interest [starttime, endtime] with the format
'YYYY-MM-DD','YYYY-MM-DD'] or to specify more or less than a day
['YYYY-MM-DD/hh:mm:ss','YYYY-MM-DD/hh:mm:ss']
Returns:
Tuple containing (start_times, end_times)
'''
if trange == None:
logging.error('Error; no trange specified.')
return None
tr = time_double(trange)
save_file = os.path.join(CONFIG['local_data_dir'], 'mms_fast_intervals.sav')
fast_file = download(remote_file='http://www.spedas.org/mms/mms_fast_intervals.sav',
local_file=save_file)
try:
intervals = readsav(save_file)
except FileNotFoundError:
logging.error('Error loading fast intervals sav file: ' + save_file)
return None
unix_start = np.flip(intervals['fast_intervals'].start_times[0])
unix_end = np.flip(intervals['fast_intervals'].end_times[0])
times_in_range = (unix_start >= tr[0]-2*86400.0) & (unix_start <= tr[1]+2*86400.0)
unix_start = unix_start[times_in_range]
unix_end = unix_end[times_in_range]
bar_x = []
bar_y = []
for start_time, end_time in zip(unix_start, unix_end):
if end_time >= tr[0] and start_time <= tr[1]:
bar_x.extend([start_time, start_time, end_time, end_time])
bar_y.extend([np.nan, 0., 0., np.nan])
vars_created = store_data('mms_bss_fast'+suffix, data={'x': bar_x, 'y': bar_y})
options('mms_bss_fast'+suffix, 'panel_size', 0.09)
options('mms_bss_fast'+suffix, 'thick', 20)
options('mms_bss_fast'+suffix, 'Color', 'green')
options('mms_bss_fast'+suffix, 'border', False)
options('mms_bss_fast'+suffix, 'yrange', [-0.001,0.001])
options('mms_bss_fast'+suffix, 'legend_names', ['Fast'])
options('mms_bss_fast'+suffix, 'ytitle', '')
return (unix_start, unix_end)
def dailynames(directory='',
trange=None,
res=24 * 3600.,
hour_res=False,
file_format='%Y%m%f',
dir_format='',
YYYY_MM_DIR=False,
prefix='',
suffix=''):
if trange == None:
print('No trange specified')
return
if hour_res == True:
res = 3600.
file_format = '%Y%m%d%H'
if YYYY_MM_DIR:
dir_format = '%Y/%m/'
tr = [time_double(trange[0]), time_double(trange[1])]
# Davin's magic heisted from file_dailynames in IDL
mmtr = [np.floor(tr[0] / res), np.ceil(tr[1] / res)]
if mmtr[1] - mmtr[0] < 1:
n = 1
else:
n = int(mmtr[1] - mmtr[0])
times = [(float(num) + mmtr[0]) * res for num in range(n)]
dates = []
files = []
for time in times:
if time_string(time, fmt=file_format) not in dates:
dates.append(time_string(time, fmt=file_format))
for date in dates:
files.append(directory + prefix + date + suffix)
return files
def parse_html(html_text, year=None, month=None):
times = []
data = []
# remove all of the HTML before the table
html_data = html_text[html_text.find('Hourly Equatorial Dst Values'):]
# remove all of the HTML after the table
html_data = html_data[:html_data.
find('<!-- vvvvv S yyyymm_part3.html vvvvv -->')]
html_lines = html_data.split('\n')
data_strs = html_lines[5:]
# loop over days
for day_str in data_strs:
# the first element of hourly_data is the day, the rest are the hourly Dst values
hourly_data = day_str.split()
if len(hourly_data[1:]) != 24:
continue
for idx, dst_value in enumerate(hourly_data[1:]):
times.append(
time_double(year + '-' + month + '-' + hourly_data[0] + '/' +
str(idx) + ':30'))
data.append(float(dst_value))
return (times, data)
def mms_feeps_remove_bad_data(probe='1',
data_rate='srvy',
datatype='electron',
level='l2',
suffix=''):
data_rate_level = data_rate + '_' + level
# electrons first, remove bad eyes
#;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1. BAD EYES ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# First, here is a list of the EYES that are bad, we need to make sure these
# data are not usable (i.e., make all of the counts/rate/flux data from these eyes NAN).
# These are for all modes, burst and survey:
bad_data_table = {}
bad_data_table['2017-10-01'] = {}
bad_data_table['2017-10-01']['mms1'] = {'top': [1], 'bottom': [1, 11]}
bad_data_table['2017-10-01']['mms2'] = {'top': [5, 7, 12], 'bottom': [7]}
bad_data_table['2017-10-01']['mms3'] = {
'top': [2, 12],
'bottom': [2, 5, 11]
}
bad_data_table['2017-10-01']['mms4'] = {
'top': [1, 2, 7],
'bottom': [2, 4, 5, 10, 11]
}
bad_data_table['2018-10-01'] = {}
bad_data_table['2018-10-01']['mms1'] = {'top': [1], 'bottom': [1, 11]}
bad_data_table['2018-10-01']['mms2'] = {'top': [7, 12], 'bottom': [2, 12]}
bad_data_table['2018-10-01']['mms3'] = {'top': [1, 2], 'bottom': [5, 11]}
bad_data_table['2018-10-01']['mms4'] = {'top': [1, 7], 'bottom': [4, 11]}
dates = np.asarray(time_double(list(bad_data_table.keys())))
closest_table_tm = (np.abs(dates - dt.datetime.now().timestamp())).argmin()
closest_table = time_string(dates[closest_table_tm], '%Y-%m-%d')
bad_data = bad_data_table[closest_table]['mms' + probe]
bad_vars = []
# top electrons
for bad_var in bad_data['top']:
if bad_var in [6, 7, 8]: continue # ion eyes
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_top_count_rate_sensorid_' + str(bad_var) +
suffix))
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_top_intensity_sensorid_' + str(bad_var) +
suffix))
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_top_counts_sensorid_' + str(bad_var) + suffix))
# bottom electrons
for bad_var in bad_data['bottom']:
if bad_var in [6, 7, 8]: continue # ion eyes
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_bottom_count_rate_sensorid_' + str(bad_var) +
suffix))
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_bottom_intensity_sensorid_' + str(bad_var) +
suffix))
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_bottom_counts_sensorid_' + str(bad_var) +
suffix))
# top ions
for bad_var in bad_data['top']:
if bad_var not in [6, 7, 8]: continue # ion eyes
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_top_count_rate_sensorid_' + str(bad_var) + suffix))
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_top_intensity_sensorid_' + str(bad_var) + suffix))
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_top_counts_sensorid_' + str(bad_var) + suffix))
# bottom ions
for bad_var in bad_data['top']:
if bad_var not in [6, 7, 8]: continue # ion eyes
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_bottom_count_rate_sensorid_' + str(bad_var) + suffix))
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_bottom_intensity_sensorid_' + str(bad_var) + suffix))
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_bottom_counts_sensorid_' + str(bad_var) + suffix))
for bad_var in bad_vars:
if bad_var == []: continue
bad_var_data = pytplot.get_data(bad_var[0])
if bad_var_data is not None:
times, data = bad_var_data
energies = pytplot.data_quants[bad_var[0]].spec_bins.values
# check if the energy table contains all names
if np.isnan(np.sum(energies)): continue
data[:] = np.nan
pytplot.store_data(bad_var[0],
data={
'x': times,
'y': data,
'v': energies.reshape(energies.size)
})
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2. BAD LOWEST E-CHANNELS ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
# ; Next, these eyes have bad first channels (i.e., lowest energy channel, E-channel 0 in IDL indexing).
# ; Again, these data (just the counts/rate/flux from the lowest energy channel ONLY!!!)
# ; should be hardwired to be NAN for all modes (burst and both types of survey).
# ; The eyes not listed here or above are ok though... so once we do this, we can actually start
# ; showing the data down to the lowest levels (~33 keV), meaning we'll have to adjust the hard-coded
# ; ylim settings in SPEDAS and the SITL software:
# ; from Drew Turner, 5Oct18:
# ;Bad Channels (0 and 1):
# ;Update: All channels 0 (Ch0) on MMS-2, -3, and -4 electron eyes (1, 2, 3, 4, 5, 9, 10, 11, 12) should be NaN
# ;Additionally, the second channels (Ch1) on the following should also be made NaN:
# ;MMS-1: Top: Ch0 on Eyes 6, 7
# ;Bot: Ch0 on Eyes 6, 7, 8
# ;MMS-2: Top:
# ;Bot: Ch0 on Eyes 6, 8
# ;MMS-3: Top: Ch0 on Eye 8
# ;Bot: Ch0 on Eyes 6, 7
# ;MMS-4: Top: Ch1 on Eye 1; Ch0 on Eye 8
# ;Bot: Ch0 on Eyes 6, 7, 8; Ch1 on Eye 9
bad_vars = []
bad_vars_both_chans = []
bad_ch0 = {}
bad_ch0['mms1'] = {
'top': [2, 5, 6, 7],
'bottom': [2, 3, 4, 5, 6, 7, 8, 9, 11, 12]
}
bad_ch0['mms2'] = {
'top': [1, 2, 3, 4, 5, 9, 10, 11, 12],
'bottom': [1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12]
}
bad_ch0['mms3'] = {
'top': [1, 2, 3, 4, 5, 8, 9, 10, 11, 12],
'bottom': [1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12]
}
bad_ch0['mms4'] = {
'top': [1, 2, 3, 4, 5, 8, 9, 10, 11, 12],
'bottom': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
}
bad_ch1 = {}
bad_ch1['mms1'] = {'top': [], 'bottom': [11]}
bad_ch1['mms2'] = {'top': [8], 'bottom': [12]}
bad_ch1['mms3'] = {'top': [1], 'bottom': []}
bad_ch1['mms4'] = {'top': [1], 'bottom': [6, 9]}
bad_ch0 = bad_ch0['mms' + str(probe)]
bad_ch1 = bad_ch1['mms' + str(probe)]
#### bottom channel
# top electrons
for bad_var in bad_ch0['top']:
if bad_var in [6, 7, 8]: continue # ion eyes
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_top_count_rate_sensorid_' + str(bad_var) +
suffix))
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_top_intensity_sensorid_' + str(bad_var) +
suffix))
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_top_counts_sensorid_' + str(bad_var) + suffix))
# bottom electrons
for bad_var in bad_ch0['bottom']:
if bad_var in [6, 7, 8]: continue # ion eyes
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_bottom_count_rate_sensorid_' + str(bad_var) +
suffix))
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_bottom_intensity_sensorid_' + str(bad_var) +
suffix))
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_bottom_counts_sensorid_' + str(bad_var) +
suffix))
# top ions
for bad_var in bad_ch0['top']:
if bad_var not in [6, 7, 8]: continue # ion eyes
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_top_count_rate_sensorid_' + str(bad_var) + suffix))
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_top_intensity_sensorid_' + str(bad_var) + suffix))
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_top_counts_sensorid_' + str(bad_var) + suffix))
# bottom ions
for bad_var in bad_ch0['bottom']:
if bad_var not in [6, 7, 8]: continue # ion eyes
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_bottom_count_rate_sensorid_' + str(bad_var) + suffix))
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_bottom_intensity_sensorid_' + str(bad_var) + suffix))
bad_vars.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_bottom_counts_sensorid_' + str(bad_var) + suffix))
#### bottom 2 channels
# top electrons
for bad_var in bad_ch1['top']:
if bad_var in [6, 7, 8]: continue # ion eyes
bad_vars_both_chans.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_top_count_rate_sensorid_' + str(bad_var) +
suffix))
bad_vars_both_chans.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_top_intensity_sensorid_' + str(bad_var) +
suffix))
bad_vars_both_chans.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_top_counts_sensorid_' + str(bad_var) + suffix))
# bottom electrons
for bad_var in bad_ch1['bottom']:
if bad_var in [6, 7, 8]: continue # ion eyes
bad_vars_both_chans.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_bottom_count_rate_sensorid_' + str(bad_var) +
suffix))
bad_vars_both_chans.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_bottom_intensity_sensorid_' + str(bad_var) +
suffix))
bad_vars_both_chans.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_electron_bottom_counts_sensorid_' + str(bad_var) +
suffix))
# top ions
for bad_var in bad_ch1['top']:
if bad_var not in [6, 7, 8]: continue # ion eyes
bad_vars_both_chans.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_top_count_rate_sensorid_' + str(bad_var) + suffix))
bad_vars_both_chans.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_top_intensity_sensorid_' + str(bad_var) + suffix))
bad_vars_both_chans.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_top_counts_sensorid_' + str(bad_var) + suffix))
# bottom ions
for bad_var in bad_ch1['bottom']:
if bad_var not in [6, 7, 8]: continue # ion eyes
bad_vars_both_chans.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_bottom_count_rate_sensorid_' + str(bad_var) + suffix))
bad_vars_both_chans.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_bottom_intensity_sensorid_' + str(bad_var) + suffix))
bad_vars_both_chans.append(
tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level +
'_ion_bottom_counts_sensorid_' + str(bad_var) + suffix))
# set the first energy channel to NaN
for bad_var in bad_vars:
if bad_var == []: continue
bad_var_data = pytplot.get_data(bad_var[0])
if bad_var_data is not None:
times, data = bad_var_data
energies = pytplot.data_quants[bad_var[0]].spec_bins.values
# check if the energy table contains all names
if np.isnan(np.sum(energies)): continue
data[:, 0] = np.nan
pytplot.store_data(bad_var[0],
data={
'x': times,
'y': data,
'v': energies.reshape(energies.size)
})
# set the first and second energy channels to NaN
for bad_var in bad_vars_both_chans:
if bad_var == []: continue
bad_var_data = pytplot.get_data(bad_var[0])
if bad_var_data is not None:
times, data = bad_var_data
energies = pytplot.data_quants[bad_var[0]].spec_bins.values
# check if the energy table contains all names
if np.isnan(np.sum(energies)): continue
data[:, 0] = np.nan
data[:, 1] = np.nan
pytplot.store_data(bad_var[0],
data={
'x': times,
'y': data,
'v': energies.reshape(energies.size)
})
# times are unix time (seconds since 1 January 1970)
print(times[0])
# FGM data include the magnitude
fgm_data[0]
# you can convert the unix time to a string with time_string
from pyspedas import time_string
print(time_string(times[0]))
# and convert back to unix time using time_double
from pyspedas import time_double
print(time_double('2015-10-16 13:06:00.00451'))
# create new tplot variables with store_data
from pytplot import store_data
# save the B-field vector
store_data('b_vector', data={'x': times, 'y': fgm_data[:, 0:3]})
# save the B-field magnitude
store_data('b_mag', data={'x': times, 'y': fgm_data[:, 3]})
# the keywords are very flexible, e.g.,
from pyspedas.mms import mms_load_hpca, mms_load_eis, mms_load_feeps, mms_load_aspoc
# specify multiple probes as integers, and multiple datatypes
mms_load_hpca(probe=[1, 2, 4],
data_rate='brst',
def mms_load_data(trange=['2015-10-16', '2015-10-17'],
probe='1',
data_rate='srvy',
level='l2',
instrument='fgm',
datatype='',
varformat=None,
prefix='',
suffix='',
get_support_data=False,
time_clip=False,
no_update=False,
center_measurement=False,
available=False,
notplot=False,
latest_version=False,
major_version=False,
min_version=None,
cdf_version=None,
spdf=False):
"""
This function loads MMS data into pyTplot variables
This function is not meant to be called directly. Please see the individual load routines for documentation and use.
"""
if not isinstance(probe, list): probe = [probe]
if not isinstance(data_rate, list): data_rate = [data_rate]
if not isinstance(level, list): level = [level]
if not isinstance(datatype, list): datatype = [datatype]
probe = [str(p) for p in probe]
# allows the user to pass in trange as list of datetime objects
if type(trange[0]) == datetime and type(trange[1]) == datetime:
trange = [
time_string(trange[0].timestamp()),
time_string(trange[1].timestamp())
]
# allows the user to pass in trange as a list of floats (unix times)
if isinstance(trange[0], float):
trange[0] = time_string(trange[0])
if isinstance(trange[1], float):
trange[1] = time_string(trange[1])
start_date = parse(trange[0]).strftime(
'%Y-%m-%d') # need to request full day, then parse out later
end_date = parse(time_string(time_double(trange[1]) - 0.1)).strftime(
'%Y-%m-%d-%H-%M-%S'
) # -1 second to avoid getting data for the next day
download_only = CONFIG['download_only']
no_download = False
if no_update or CONFIG['no_download']: no_download = True
if spdf:
return mms_load_data_spdf(trange=trange,
probe=probe,
data_rate=data_rate,
level=level,
instrument=instrument,
datatype=datatype,
varformat=varformat,
suffix=suffix,
get_support_data=get_support_data,
time_clip=time_clip,
no_update=no_update,
center_measurement=center_measurement,
notplot=notplot,
latest_version=latest_version,
major_version=major_version,
min_version=min_version,
cdf_version=cdf_version)
user = None
if not no_download:
sdc_session, user = mms_login_lasp()
out_files = []
available_files = []
for prb in probe:
for drate in data_rate:
for lvl in level:
for dtype in datatype:
if user is None:
url = 'https://lasp.colorado.edu/mms/sdc/public/files/api/v1/file_info/science?start_date=' + start_date + '&end_date=' + end_date + '&sc_id=mms' + prb + '&instrument_id=' + instrument + '&data_rate_mode=' + drate + '&data_level=' + lvl
else:
url = 'https://lasp.colorado.edu/mms/sdc/sitl/files/api/v1/file_info/science?start_date=' + start_date + '&end_date=' + end_date + '&sc_id=mms' + prb + '&instrument_id=' + instrument + '&data_rate_mode=' + drate + '&data_level=' + lvl
if dtype != '':
url = url + '&descriptor=' + dtype
if CONFIG['debug_mode']: logging.info('Fetching: ' + url)
if no_download == False:
# query list of available files
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore",
category=ResourceWarning)
http_json = sdc_session.get(
url, verify=True).json()
if CONFIG['debug_mode']:
logging.info(
'Filtering the results down to your trange'
)
files_in_interval = mms_files_in_interval(
http_json['files'], trange)
if available:
for file in files_in_interval:
logging.info(
file['file_name'] + ' (' + str(
np.round(file['file_size'] /
(1024. * 1024),
decimals=1)) + ' MB)')
available_files.append(file['file_name'])
continue
for file in files_in_interval:
file_date = parse(file['timetag'])
if dtype == '':
out_dir = os.sep.join([
CONFIG['local_data_dir'], 'mms' + prb,
instrument, drate, lvl,
file_date.strftime('%Y'),
file_date.strftime('%m')
])
else:
out_dir = os.sep.join([
CONFIG['local_data_dir'], 'mms' + prb,
instrument, drate, lvl, dtype,
file_date.strftime('%Y'),
file_date.strftime('%m')
])
if drate.lower() == 'brst':
out_dir = os.sep.join(
[out_dir,
file_date.strftime('%d')])
out_file = os.sep.join(
[out_dir, file['file_name']])
if CONFIG['debug_mode']:
logging.info('File: ' + file['file_name'] +
' / ' + file['timetag'])
if os.path.exists(out_file) and str(
os.stat(out_file).st_size) == str(
file['file_size']):
if not download_only:
logging.info('Loading ' + out_file)
out_files.append(out_file)
continue
if user is None:
download_url = 'https://lasp.colorado.edu/mms/sdc/public/files/api/v1/download/science?file=' + file[
'file_name']
else:
download_url = 'https://lasp.colorado.edu/mms/sdc/sitl/files/api/v1/download/science?file=' + file[
'file_name']
logging.info('Downloading ' +
file['file_name'] + ' to ' +
out_dir)
with warnings.catch_warnings():
warnings.simplefilter(
"ignore", category=ResourceWarning)
fsrc = sdc_session.get(download_url,
stream=True,
verify=True)
ftmp = NamedTemporaryFile(delete=False)
with open(ftmp.name, 'wb') as f:
copyfileobj(fsrc.raw, f)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# if the download was successful, copy to data directory
copy(ftmp.name, out_file)
out_files.append(out_file)
fsrc.close()
ftmp.close()
except requests.exceptions.ConnectionError:
# No/bad internet connection; try loading the files locally
logging.error('No internet connection!')
if out_files == []:
if not download_only:
logging.info('Searching for local files...')
out_files = mms_get_local_files(
prb, instrument, drate, lvl, dtype, trange)
if out_files == [] and CONFIG[
'mirror_data_dir'] != None:
# check for network mirror; note: network mirrors are assumed to be read-only
# and we always copy the files from the mirror to the local data directory
# before trying to load into tplot variables
logging.info(
'No local files found; checking network mirror...'
)
out_files = mms_get_local_files(prb,
instrument,
drate,
lvl,
dtype,
trange,
mirror=True)
if not no_download:
sdc_session.close()
if available:
return available_files
if not download_only:
out_files = sorted(out_files)
filtered_out_files = mms_file_filter(out_files,
latest_version=latest_version,
major_version=major_version,
min_version=min_version,
version=cdf_version)
if filtered_out_files == []:
logging.info('No matching CDF versions found.')
return
new_variables = cdf_to_tplot(filtered_out_files,
varformat=varformat,
merge=True,
get_support_data=get_support_data,
prefix=prefix,
suffix=suffix,
center_measurement=center_measurement,
notplot=notplot)
if notplot:
return new_variables
if new_variables == []:
logging.warning('No data loaded.')
return
if time_clip:
for new_var in new_variables:
tclip(new_var, trange[0], trange[1], suffix='')
return new_variables
else:
return out_files
def mms_load_brst_segments(trange=None, suffix=''):
'''
This function loads the burst segment intervals
Parameters:
trange : list of str
time range of interest [starttime, endtime] with the format
'YYYY-MM-DD','YYYY-MM-DD'] or to specify more or less than a day
['YYYY-MM-DD/hh:mm:ss','YYYY-MM-DD/hh:mm:ss']
Returns:
Tuple containing (start_times, end_times)
'''
if trange is None:
logging.error('Error; no trange specified.')
return None
tr = time_double(trange)
save_file = os.path.join(CONFIG['local_data_dir'],
'mms_brst_intervals.sav')
brst_file = download(
remote_file='http://www.spedas.org/mms/mms_brst_intervals.sav',
local_file=save_file)
if len(brst_file) == 0:
logging.error('Error downloading burst intervals sav file')
return None
try:
intervals = readsav(save_file)
except FileNotFoundError:
logging.error('Error loading burst intervals sav file: ' + save_file)
return None
unix_start = intervals['brst_intervals'].start_times[0]
unix_end = intervals['brst_intervals'].end_times[0]
sorted_idxs = np.argsort(unix_start)
unix_start = unix_start[sorted_idxs]
unix_end = unix_end[sorted_idxs]
times_in_range = (unix_start >= tr[0] - 300.0) & (unix_start <=
tr[1] + 300.0)
unix_start = unix_start[times_in_range]
unix_end = unix_end[times_in_range]
# +10 second offset added; there appears to be an extra 10
# seconds of data, consistently, not included in the range here
unix_end = [end_time + 10.0 for end_time in unix_end]
bar_x = []
bar_y = []
for start_time, end_time in zip(unix_start, unix_end):
if end_time >= tr[0] and start_time <= tr[1]:
bar_x.extend([start_time, start_time, end_time, end_time])
bar_y.extend([np.nan, 0., 0., np.nan])
vars_created = store_data('mms_bss_burst' + suffix,
data={
'x': bar_x,
'y': bar_y
})
if vars_created == False:
logging.error('Error creating burst segment intervals tplot variable')
return None
options('mms_bss_burst' + suffix, 'panel_size', 0.09)
options('mms_bss_burst' + suffix, 'thick', 2)
options('mms_bss_burst' + suffix, 'Color', 'green')
options('mms_bss_burst' + suffix, 'border', False)
options('mms_bss_burst' + suffix, 'yrange', [-0.001, 0.001])
options('mms_bss_burst' + suffix, 'legend_names', ['Burst'])
options('mms_bss_burst' + suffix, 'ytitle', '')
return (unix_start, unix_end)
def mms_feeps_active_eyes(trange, probe, data_rate, species, level):
"""
This function returns the FEEPS active eyes, based on date/probe/species/rate
Parameters:
trange: list of str
time range
probe: str
probe #, e.g., '4' for MMS4
data_rate: str
instrument data rate, e.g., 'srvy' or 'brst'
species: str
'electron' or 'ion'
level: str
data level
Returns:
Hash table containing 2 keys:
output['top'] -> maps to the active top eyes
output['bottom'] -> maps to the active bottom eyes
Notes:
1) Burst mode should include all sensors (TOP and BOTTOM):
electrons: [1, 2, 3, 4, 5, 9, 10, 11, 12]
ions: [6, 7, 8]
2) SITL should return (TOP only):
electrons: set_intersection([5, 11, 12], active_eyes)
ions: None
3) From Drew Turner, 9/7/2017, srvy mode:
- before 16 August 2017:
electrons: [3, 4, 5, 11, 12]
iond: [6, 7, 8]
- after 16 August 2017:
MMS1
Top Eyes: 3, 5, 6, 7, 8, 9, 10, 12
Bot Eyes: 2, 4, 5, 6, 7, 8, 9, 10
MMS2
Top Eyes: 1, 2, 3, 5, 6, 8, 10, 11
Bot Eyes: 1, 4, 5, 6, 7, 8, 9, 11
MMS3
Top Eyes: 3, 5, 6, 7, 8, 9, 10, 12
Bot Eyes: 1, 2, 3, 6, 7, 8, 9, 10
MMS4
Top Eyes: 3, 4, 5, 6, 8, 9, 10, 11
Bot Eyes: 3, 5, 6, 7, 8, 9, 10, 12
"""
if data_rate.lower() == 'brst' and species.lower() == 'electron':
return {
'top': [1, 2, 3, 4, 5, 9, 10, 11, 12],
'bottom': [1, 2, 3, 4, 5, 9, 10, 11, 12]
}
if data_rate.lower() == 'brst' and species.lower() == 'ion':
return {'top': [6, 7, 8], 'bottom': [6, 7, 8]}
# old eyes, srvy mode, prior to 16 August 2017
if species.lower() == 'electron':
sensors = {'top': [3, 4, 5, 11, 12], 'bottom': [3, 4, 5, 11, 12]}
else:
sensors = {'top': [6, 7, 8], 'bottom': [6, 7, 8]}
if isinstance(trange[0], str):
start_time = time_double(trange[0])
else:
start_time = trange[0]
# srvy mode, after 16 August 2017
if start_time >= time_double('2017-08-16') and data_rate.lower() == 'srvy':
active_table = {
'1-electron': {
'top': [3, 5, 9, 10, 12],
'bottom': [2, 4, 5, 9, 10]
},
'1-ion': {
'top': [6, 7, 8],
'bottom': [6, 7, 8]
},
'2-electron': {
'top': [1, 2, 3, 5, 10, 11],
'bottom': [1, 4, 5, 9, 11]
},
'2-ion': {
'top': [6, 8],
'bottom': [6, 7, 8]
},
'3-electron': {
'top': [3, 5, 9, 10, 12],
'bottom': [1, 2, 3, 9, 10]
},
'3-ion': {
'top': [6, 7, 8],
'bottom': [6, 7, 8]
},
'4-electron': {
'top': [3, 4, 5, 9, 10, 11],
'bottom': [3, 5, 9, 10, 12]
},
'4-ion': {
'top': [6, 8],
'bottom': [6, 7, 8]
}
}
sensors = active_table[probe.lower() + '-' + species.lower()]
if level.lower() == 'sitl':
return {
'top': list(set(sensors['top']) & set([5, 11, 12])),
'bottom': []
}
if level.lower() == 'sitl':
return {'top': [5, 11, 12], 'bottom': []}
return sensors
def mms_load_data(trange=['2015-10-16', '2015-10-17'],
probe='1',
data_rate='srvy',
level='l2',
instrument='fgm',
datatype='',
prefix='',
suffix='',
get_support_data=False,
time_clip=False):
"""
This function loads MMS data into pyTplot variables
"""
if not isinstance(probe, list): probe = [probe]
if not isinstance(data_rate, list): data_rate = [data_rate]
if not isinstance(level, list): level = [level]
if not isinstance(datatype, list): datatype = [datatype]
probe = [str(p) for p in probe]
# allows the user to pass in trange as list of datetime objects
if type(trange[0]) == datetime and type(trange[1]) == datetime:
trange = [
time_string(trange[0].timestamp()),
time_string(trange[1].timestamp())
]
start_date = parse(trange[0]).strftime(
'%Y-%m-%d') # need to request full day, then parse out later
end_date = parse(time_string(time_double(trange[1]) - 0.1)).strftime(
'%Y-%m-%d-%H-%M-%S'
) # -1 second to avoid getting data for the next day
download_only = CONFIG['download_only']
if not CONFIG['no_download']:
sdc_session, user = mms_login_lasp()
out_files = []
for prb in probe:
for drate in data_rate:
for lvl in level:
for dtype in datatype:
if user is None:
url = 'https://lasp.colorado.edu/mms/sdc/public/files/api/v1/file_info/science?start_date=' + start_date + '&end_date=' + end_date + '&sc_id=mms' + prb + '&instrument_id=' + instrument + '&data_rate_mode=' + drate + '&data_level=' + lvl
else:
url = 'https://lasp.colorado.edu/mms/sdc/sitl/files/api/v1/file_info/science?start_date=' + start_date + '&end_date=' + end_date + '&sc_id=mms' + prb + '&instrument_id=' + instrument + '&data_rate_mode=' + drate + '&data_level=' + lvl
if dtype != '':
url = url + '&descriptor=' + dtype
if CONFIG['debug_mode']: logging.info('Fetching: ' + url)
if CONFIG['no_download'] == False:
# query list of available files
try:
http_json = sdc_session.get(url,
verify=True).json()
if CONFIG['debug_mode']:
logging.info(
'Filtering the results down to your trange'
)
files_in_interval = mms_files_in_interval(
http_json['files'], trange)
for file in files_in_interval:
file_date = parse(file['timetag'])
if dtype == '':
out_dir = os.sep.join([
CONFIG['local_data_dir'], 'mms' + prb,
instrument, drate, lvl,
file_date.strftime('%Y'),
file_date.strftime('%m')
])
else:
out_dir = os.sep.join([
CONFIG['local_data_dir'], 'mms' + prb,
instrument, drate, lvl, dtype,
file_date.strftime('%Y'),
file_date.strftime('%m')
])
if drate.lower() == 'brst':
out_dir = os.sep.join(
[out_dir,
file_date.strftime('%d')])
out_file = os.sep.join(
[out_dir, file['file_name']])
if CONFIG['debug_mode']:
logging.info('File: ' + file['file_name'] +
' / ' + file['timetag'])
#if os.path.exists(out_file) and str(os.stat(out_file).st_size) == str(file['file_size']):
if os.path.exists(out_file):
if not download_only:
logging.info('Loading ' + out_file)
out_files.append(out_file)
continue
if user is None:
download_url = 'https://lasp.colorado.edu/mms/sdc/public/files/api/v1/download/science?file=' + file[
'file_name']
else:
download_url = 'https://lasp.colorado.edu/mms/sdc/sitl/files/api/v1/download/science?file=' + file[
'file_name']
logging.info('Downloading ' +
file['file_name'] + ' to ' +
out_dir)
fsrc = sdc_session.get(download_url,
stream=True,
verify=True)
ftmp = NamedTemporaryFile(delete=False)
copyfileobj(fsrc.raw, ftmp)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# if the download was successful, copy to data directory
copy(ftmp.name, out_file)
out_files.append(out_file)
ftmp.close()
fsrc.close()
except requests.exceptions.ConnectionError:
# No/bad internet connection; try loading the files locally
logging.error('No internet connection!')
if out_files == []:
if not download_only:
logging.info('Searching for local files...')
out_files = mms_get_local_files(
prb, instrument, drate, lvl, dtype, trange)
if not CONFIG['no_download']:
sdc_session.close()
if not download_only:
out_files = sorted(out_files)
new_variables = cdf_to_tplot(out_files,
merge=True,
get_support_data=get_support_data,
prefix=prefix,
suffix=suffix)
if new_variables == []:
logging.warning('No data loaded.')
return
logging.info('Loaded variables:')
for new_var in new_variables:
print(new_var)
if time_clip:
tclip(new_var, trange[0], trange[1], suffix='')
return new_variables
def get_w(trange=None, create_tvar=False, newname=None):
"""
This routine downloads the 6 Tsygeneko (TS05) model
driving variables W1, W2, W3, W4, W5, W6; from:
http://geo.phys.spbu.ru/~tsyganenko/TS05_data_and_stuff
"""
if trange is None:
print('trange keyword must be specified.')
return
years = dailynames(trange=trange, file_format='%Y')
tmpdir = mkdtemp()
if newname is None:
newname = 'Tsy_W_vars_' + '-'.join(years)
ut_out = np.empty(0)
w1_out = np.empty(0)
w2_out = np.empty(0)
w3_out = np.empty(0)
w4_out = np.empty(0)
w5_out = np.empty(0)
w6_out = np.empty(0)
for year in years:
file = download(
remote_path=
'http://geo.phys.spbu.ru/~tsyganenko/TS05_data_and_stuff/',
remote_file=year + '_OMNI_5m_with_TS05_variables.???',
local_path=tmpdir)
if file[0][-3:] == 'zip':
with zipfile.ZipFile(file[0], 'r') as zip_ref:
zip_ref.extractall(tmpdir)
rows = pd.read_csv(tmpdir + '/' + year +
'_OMNI_5m_with_TS05_variables.dat',
delim_whitespace=True,
header=None)
# extract the W variables
w1 = rows.to_numpy()[:, -6]
w2 = rows.to_numpy()[:, -5]
w3 = rows.to_numpy()[:, -4]
w4 = rows.to_numpy()[:, -3]
w5 = rows.to_numpy()[:, -2]
w6 = rows.to_numpy()[:, -1]
# extract the times
years = rows.to_numpy()[:, 0]
doys = rows.to_numpy()[:, 1]
hours = rows.to_numpy()[:, 2]
minutes = rows.to_numpy()[:, 3]
time_strings = [
str(int(year)) + '-' + str(int(doy)).zfill(3) + ' ' +
str(int(hour)).zfill(2) + ':' + str(int(minute)).zfill(2)
for year, doy, hour, minute in zip(years, doys, hours, minutes)
]
unix_times = np.array(time_double(time_strings))
ut_out = np.append(ut_out, unix_times)
w1_out = np.append(w1_out, w1)
w2_out = np.append(w2_out, w2)
w3_out = np.append(w3_out, w3)
w4_out = np.append(w4_out, w4)
w5_out = np.append(w5_out, w5)
w6_out = np.append(w6_out, w6)
in_range = np.argwhere((ut_out >= time_double(trange[0]))
& (ut_out < time_double(trange[1]))).squeeze()
if len(in_range) == 0:
print('No data found in the trange.')
return
if create_tvar:
out = np.array((w1_out[in_range], w2_out[in_range], w3_out[in_range],
w4_out[in_range], w5_out[in_range], w6_out[in_range]))
store_data(newname, data={'x': ut_out[in_range], 'y': out.T})
return newname
return {
'times': ut_out[in_range],
'w1': w1_out[in_range],
'w2': w2_out[in_range],
'w3': w3_out[in_range],
'w4': w4_out[in_range],
'w5': w5_out[in_range],
'w6': w6_out[in_range]
}
def mms_fpi_make_compressionlossbars(tname, lossy=False):
"""
Creates FPI's compressionloss flag bars
Parameters
-----------
tname: str
Tplot variable name of DIS or DES compressionloss data
lossy: bool
The value for lossy compression (use this keyword only for special case)
Flag
-----------
0: Lossless compression
1: Lossy compression
In old files (v2.1.0 or older until the end of Phase 1A)
1: Lossless compression
3: Lossy compression
Notes
-----------
In cases when data had been lossy compressed, some artifacts may appear in the data due to the compression.
Since all of fast survey data are lossy compressed, it is not necessary to make this bar for fast survey data.
Returns
------------
List of the tplot variables created.
"""
if fnmatch(tname, 'mms?_dis*'):
instrument = 'DIS'
elif fnmatch(tname, 'mms?_des*'):
instrument = 'DES'
else:
print('Unable to determine instrument from variable name.')
return
if instrument == 'DES':
colors = 'red'
else:
colors = 'blue'
if fnmatch(tname, '*_fast*'):
print('All fast survey data are lossy compressed, so there is no need to create this bar.')
return
elif fnmatch(tname, '*_brst*'):
data_rate = 'Brst'
else:
print('Unable to determine data rate from variable name.')
return
data = get_data(tname)
metadata = get_data(tname, metadata=True)
if data is None:
print('Problem reading the variable: ' + tname)
return
flagline = np.zeros(len(data.times))
if not lossy:
file_id = metadata['CDF']['GATT']['Logical_file_id']
version = file_id.split('_v')[1].split('.')
if version[0] == '2':
if version[1] == '1':
if data.times[0] < time_double('2016-04-01'):
lossy = 3
else:
lossy = 1
else:
if float(version[1]) > 1:
lossy = 1
else:
lossy = 3
else:
if float(version[0]) > 2:
lossy = 1
else:
lossy = 3
for j in range(len(data.times)):
if data.y[j] != lossy:
flagline[j] = np.nan
else:
flagline[j] = 0.5
store_data(tname + '_flagbars', data={'x': data.times, 'y': flagline})
options(tname + '_flagbars', 'yrange', [0, 1])
options(tname + '_flagbars', 'panel_size', 0.2)
options(tname + '_flagbars', 'symbols', True)
options(tname + '_flagbars', 'markers', 's')
options(tname + '_flagbars', 'thick', 4)
options(tname + '_flagbars', 'border', False)
return [tname + '_flagbars']
def mms_part_getspec(instrument='fpi',
probe='1',
species='e',
data_rate='fast',
trange=None,
output=['energy', 'theta', 'phi', 'pa', 'gyro'],
units='eflux',
energy=None,
phi=None,
theta=None,
pitch=None,
gyro=None,
mag_data_rate=None,
scpot_data_rate=None,
fac_type='mphigeo',
center_measurement=False,
spdf=False,
correct_photoelectrons=False,
internal_photoelectron_corrections=False,
disable_photoelectron_corrections=False,
zero_negative_values=False,
regrid=[32, 16],
no_regrid=False):
"""
Generate spectra and moments from 3D MMS particle data
Parameters
----------
trange: list of str
Time range
units: str
Specify units of output variables; must be 'eflux' to calculate moments
valid options:
'flux' - # / (cm^2 * s * sr * eV)
'eflux' - eV / (cm^2 * s * sr * eV) <default>
'df_cm' - s^3 / cm^6
'df_km' - s^3 / km^6
species: str
Specify the species of the input tplot variable
data_rate: str
Data rate of the input data
instrument: str
Instrument (FPI or HPCA)
probe: int or str
Spacecraft probe #
output: str or list of str
Output variables; options:
'energy': energy spectrograms
'theta': theta spectrograms
'phi': phi spectrograms
'pa': pitch-angle spectrograms
'gyro': gyro-phase spectrograms
'moments': plasma moments
energy: list of float
Energy range [min, max], in eV
phi: list of float
Phi range [min, max], in degrees
theta: list of float
Theta range [min, max], in degrees
pitch: list of float
Pitch-angle range [min, max], in degrees
gyro: list of float
Gyro-angle range [min, max], in degrees
mag_name: str
Tplot variable containing magnetic field data for
moments and FAC transformations
pos_name: str
Tplot variable containing spacecraft position for
FAC transformations
sc_pot_name: str
Tplot variable containing spacecraft potential data
for moments corrections
fac_type: str
Field aligned coordinate system variant; default: 'mphigeo'
options: 'phigeo', 'mphigeo', 'xgse'
correct_photoelectrons: bool
Flag to correct FPI data for photoelectrons
(defaults to True for FPI electron data - disable with the parameter below)
disable_photoelectron_corrections: bool
Flag to disable FPI photoelectron corrections
internal_photoelectron_corrections: bool
Apply internal photoelectron corrections
zero_negative_values: bool
Turn negative values to 0 after doing the photoelectron corrections (DES)
Returns
----------
Creates tplot variables containing spectrograms and moments
"""
start_time = time()
if trange is None:
# test data for development
trange = ['2015-10-16/13:06', '2015-10-16/13:07']
# data_rate = 'brst'
if mag_data_rate is None:
if data_rate == 'brst':
mag_data_rate = 'brst'
else:
mag_data_rate = 'srvy'
if scpot_data_rate is None:
if data_rate == 'brst':
scpot_data_rate = 'brst'
else:
scpot_data_rate = 'fast'
instrument = instrument.lower()
# HPCA is required to be at the center of the accumulation interval
# due to assumptions made in mms_get_hpca_dist
if instrument == 'hpca' and center_measurement == False:
center_measurement = True
if instrument == 'fpi':
data_vars = pyspedas.mms.fpi(datatype='d' + species + 's-dist',
probe=probe,
data_rate=data_rate,
trange=trange,
time_clip=True,
center_measurement=center_measurement,
spdf=spdf)
elif instrument == 'hpca':
# for HPCA, 'fast' should be 'srvy'
if data_rate == 'fast':
data_rate = 'srvy'
# 'i' and 'e' are only valid for FPI
if species in ['i', 'e']:
species = 'hplus'
data_vars = pyspedas.mms.hpca(datatype='ion',
probe=probe,
data_rate=data_rate,
trange=trange,
time_clip=True,
center_measurement=center_measurement,
get_support_data=True,
spdf=spdf)
else:
logging.error('Error, unknown instrument: ' + instrument +
'; valid options: fpi, hpca')
return
if data_vars is None or len(data_vars) == 0:
logging.error('Error, no data loaded.')
return None
if not isinstance(probe, list):
probe = [probe]
if instrument == 'fpi' and species == 'e' and 'moments' in output and not disable_photoelectron_corrections:
correct_photoelectrons = True
support_trange = [
time_double(trange[0]) - 60.0,
time_double(trange[1]) + 60.0
]
# load state data (needed for coordinate transformations and field-aligned coordinates)
pos_vars = pyspedas.mms.mec(probe=probe,
trange=support_trange,
time_clip=True,
spdf=spdf)
if len(pos_vars) == 0:
logging.error('Error, no state data loaded.')
return
mag_vars = pyspedas.mms.fgm(probe=probe,
trange=support_trange,
data_rate=mag_data_rate,
time_clip=True,
spdf=spdf)
if len(mag_vars) == 0:
logging.error('Error, no magnetic field data loaded.')
return
scpot_vars = pyspedas.mms.edp(probe=probe,
trange=support_trange,
level='l2',
spdf=spdf,
data_rate=scpot_data_rate,
datatype='scpot',
varformat='*_edp_scpot_*')
out_vars = []
for prb in probe:
prb_str = str(prb)
mag_name = 'mms' + prb_str + '_fgm_b_gse_' + mag_data_rate + '_l2_bvec'
pos_name = 'mms' + prb_str + '_mec_r_gse'
if instrument == 'fpi':
tname = 'mms' + prb_str + '_d' + species + 's_dist_' + data_rate
elif instrument == 'hpca':
tname = 'mms' + prb_str + '_hpca_' + species + '_phase_space_density'
scpot_variable = 'mms' + prb_str + '_edp_scpot_' + scpot_data_rate + '_l2'
new_vars = mms_part_products(
tname,
species=species,
instrument=instrument,
probe=prb,
data_rate=data_rate,
output=output,
units=units,
energy=energy,
phi=phi,
theta=theta,
pitch=pitch,
gyro=gyro,
mag_name=mag_name,
pos_name=pos_name,
fac_type=fac_type,
sc_pot_name=scpot_variable,
correct_photoelectrons=correct_photoelectrons,
zero_negative_values=zero_negative_values,
internal_photoelectron_corrections=
internal_photoelectron_corrections,
disable_photoelectron_corrections=disable_photoelectron_corrections,
regrid=regrid,
no_regrid=no_regrid)
if new_vars is None:
continue
out_vars = out_vars + new_vars
logging.info('Finished; time to run: ' +
str(round(time() - start_time, 1)) + ' seconds.')
return out_vars
def mms_load_sroi_segments(trange=None, probe=1, suffix=''):
"""
This function loads the Science Region of Interest (SRoI) segment intervals
Parameters:
trange: list of str
time range of interest [starttime, endtime] with the format
'YYYY-MM-DD','YYYY-MM-DD'] or to specify more or less than a day
['YYYY-MM-DD/hh:mm:ss','YYYY-MM-DD/hh:mm:ss']
probe: str or int
Spacecraft probe # (default: 1)
suffix: str
Suffix to append to the end of the tplot variables
Returns:
Tuple containing (start_times, end_times)
"""
if not isinstance(probe, str):
probe = str(probe)
if trange is None:
logging.error('Error; no trange specified.')
return None
tr = time_double(trange)
unix_start, unix_end = get_mms_srois(
start_time=time_string(tr[0] - 2 * 86400.0, fmt='%Y-%m-%d %H:%M:%S'),
end_time=time_string(tr[1] + 2 * 86400.0, fmt='%Y-%m-%d %H:%M:%S'),
sc_id='mms' + str(probe))
if len(unix_start) == 0:
return ([], [])
times_in_range = (unix_start >= tr[0] - 2 * 86400.0) & (
unix_start <= tr[1] + 2 * 86400.0)
unix_start = unix_start[times_in_range]
unix_end = unix_end[times_in_range]
start_out = []
end_out = []
bar_x = []
bar_y = []
for start_time, end_time in zip(unix_start, unix_end):
if end_time >= tr[0] and start_time <= tr[1]:
bar_x.extend([start_time, start_time, end_time, end_time])
bar_y.extend([np.nan, 0., 0., np.nan])
start_out.append(start_time)
end_out.append(end_time)
vars_created = store_data('mms' + probe + '_bss_sroi' + suffix,
data={
'x': bar_x,
'y': bar_y
})
if not vars_created:
logging.error('Error creating SRoI segment intervals tplot variable')
return None
options('mms' + probe + '_bss_sroi' + suffix, 'panel_size', 0.09)
options('mms' + probe + '_bss_sroi' + suffix, 'thick', 2)
options('mms' + probe + '_bss_sroi' + suffix, 'Color', 'green')
options('mms' + probe + '_bss_sroi' + suffix, 'border', False)
options('mms' + probe + '_bss_sroi' + suffix, 'yrange', [-0.001, 0.001])
options('mms' + probe + '_bss_sroi' + suffix, 'legend_names', ['Fast'])
options('mms' + probe + '_bss_sroi' + suffix, 'ytitle', '')
return (start_out, end_out)
def dailynames(directory='',
trange=None,
res=24 * 3600.,
hour_res=False,
file_format='%Y%m%d',
prefix='',
suffix=''):
'''
Creates a list of file names using a time range, resoution and file format
Based on Davin Larson's file_dailynames in IDL SPEDAS
Parameters:
directory: str
String containing the directory for the generated file names
trange: list of str, list of datetime or list of floats
Two-element list containing the start and end times for the file names
res: float
File name resolution in seconds (default: 24*3600., i.e., daily)
file_format: str
Format of the file names using strftime directives; for reference: https://strftime.org
(default: %Y%m%d, i.e., daily)
prefix: str
file name prefix
suffix: str
file name suffix
Returns:
List containing filenames
'''
if trange is None:
print('No trange specified')
return
if hour_res == True:
res = 3600.
file_format = '%Y%m%d%H'
# allows the user to pass in trange as list of datetime objects
if type(trange[0]) == datetime and type(trange[1]) == datetime:
trange = [
time_string(trange[0].timestamp()),
time_string(trange[1].timestamp())
]
tr = [trange[0], trange[1]]
if isinstance(trange[0], str):
tr[0] = time_double(trange[0])
if isinstance(trange[1], str):
tr[1] = time_double(trange[1])
# Davin's magic heisted from file_dailynames in IDL
mmtr = [np.floor(tr[0] / res), np.ceil(tr[1] / res)]
if mmtr[1] - mmtr[0] < 1:
n = 1
else:
n = int(mmtr[1] - mmtr[0])
times = [(float(num) + mmtr[0]) * res for num in range(n)]
dates = []
files = []
for time in times:
if time_string(time, fmt=file_format) not in dates:
dates.append(time_string(time, fmt=file_format))
for date in dates:
files.append(directory + prefix + date + suffix)
return files
def mms_load_data_spdf(trange=['2015-10-16', '2015-10-17'],
probe='1',
data_rate='srvy',
level='l2',
instrument='fgm',
datatype='',
varformat=None,
suffix='',
get_support_data=False,
time_clip=False,
no_update=False,
center_measurement=False,
available=False,
notplot=False,
latest_version=False,
major_version=False,
min_version=None,
cdf_version=None,
varnames=[]):
"""
This function loads MMS data from NASA SPDF into pyTplot variables
This function is not meant to be called directly. Please see the individual load routines for documentation and use.
"""
tvars_created = []
if not isinstance(probe, list): probe = [probe]
if not isinstance(data_rate, list): data_rate = [data_rate]
if not isinstance(level, list): level = [level]
if not isinstance(datatype, list): datatype = [datatype]
for prb in probe:
for lvl in level:
for drate in data_rate:
if drate == 'brst':
time_format = '%Y%m%d%H%M??'
file_res = 60.
else:
time_format = '%Y%m%d'
file_res = 24 * 3600.
for dtype in datatype:
remote_path = 'mms' + prb + '/' + instrument + '/' + drate + '/' + lvl + '/'
if instrument == 'fgm':
pathformat = remote_path + '%Y/%m/mms' + prb + '_fgm_' + drate + '_' + lvl + '_' + time_format + '_v*.cdf'
elif instrument == 'aspoc':
pathformat = remote_path + '%Y/%m/mms' + prb + '_aspoc_' + drate + '_' + lvl + '_' + time_format + '_v*.cdf'
elif instrument == 'edi':
pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_edi_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf'
elif instrument == 'fpi':
if drate != 'brst':
time_format = '%Y%m%d??0000'
pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_fpi_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf'
elif instrument == 'epd-eis':
pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_epd-eis_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf'
elif instrument == 'feeps':
if drate != 'brst':
time_format = '%Y%m%d000000'
pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_feeps_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf'
elif instrument == 'hpca':
time_format = '%Y%m%d??????'
pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_hpca_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf'
elif instrument == 'mec':
pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_mec_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf'
elif instrument == 'scm':
pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_scm_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf'
elif instrument == 'dsp':
pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_dsp_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf'
elif instrument == 'edp':
pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_edp_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf'
if drate == 'brst':
if isinstance(trange[0], float):
trange = [trange[0] - 300., trange[1]]
else:
trange = [time_double(trange[0]) - 300., trange[1]]
# find the full remote path names using the trange
remote_names = dailynames(file_format=pathformat,
trange=trange,
res=file_res)
out_files = []
files = download(remote_file=remote_names,
remote_path=CONFIG['remote_data_dir'],
local_path=CONFIG['local_data_dir'])
if files is not None:
for file in files:
out_files.append(file)
out_files = sorted(out_files)
filtered_out_files = mms_file_filter(
out_files,
latest_version=latest_version,
major_version=major_version,
min_version=min_version,
version=cdf_version)
tvars = cdf_to_tplot(filtered_out_files,
varformat=varformat,
varnames=varnames,
get_support_data=get_support_data,
suffix=suffix,
center_measurement=center_measurement,
notplot=notplot)
if tvars is not None:
tvars_created.extend(tvars)
if time_clip:
for new_var in tvars_created:
tclip(new_var, trange[0], trange[1], suffix='')
return tvars_created
def mms_get_local_state_files(probe='1',
level='def',
filetype='eph',
trange=None):
"""
Search for local state MMS files in case a list cannot be retrieved from the
remote server. Returns a sorted list of file paths.
Parameters:
trange : list of str
time range of interest [starttime, endtime] with the format
'YYYY-MM-DD','YYYY-MM-DD'] or to specify more or less than a day
['YYYY-MM-DD/hh:mm:ss','YYYY-MM-DD/hh:mm:ss']
probe: str
probe #, e.g., '4' for MMS4
level: str
state data level; either 'def' (for definitive) or 'pred' (for predicted)
filetype: str
state file type, e.g. 'eph' (for ephemeris) or 'att' (for attitude)
Returns:
List of local files found matching the input parameters.
"""
if trange is None:
logging.info('No trange specified in mms_get_local_state_files')
return
# directory and file name search patterns
# For now
# -all ancillary data is in one directory:
# mms\ancillary
# -assume file names are of the form:
# SPACECRAFT_FILETYPE_startDate_endDate.version
# where SPACECRAFT is [MMS1, MMS2, MMS3, MMS4] in uppercase
# and FILETYPE is either DEFATT, PREDATT, DEFEPH, PREDEPH in uppercase
# and start/endDate is YYYYDOY
# and version is Vnn (.V00, .V01, etc..)
dir_pattern = os.sep.join([
CONFIG['local_data_dir'], 'ancillary', 'mms' + probe, level + filetype
])
file_pattern = 'MMS' + probe + '_' + level.upper() + filetype.upper(
) + '_' + '???????_???????.V??'
files_in_trange = []
out_files = []
files = glob.glob(os.sep.join([dir_pattern, file_pattern]))
# find the files within the trange
file_regex = re.compile(
os.sep.join([
dir_pattern, 'MMS' + probe + '_' + level.upper() +
filetype.upper() + '_([0-9]{7})_([0-9]{7}).V[0-9]{2}'
]))
for file in files:
time_match = file_regex.match(file)
if time_match != None:
start_time = pd.to_datetime(time_match.group(1),
format='%Y%j').timestamp()
end_time = pd.to_datetime(time_match.group(2),
format='%Y%j').timestamp()
if start_time < time_double(trange[1]) and end_time >= time_double(
trange[0]):
files_in_trange.append(file)
# ensure only the latest version of each file is loaded
for file in files_in_trange:
this_file = file[0:-3] + 'V??'
versions = fnmatch.filter(files_in_trange, this_file)
if len(versions) > 1:
out_files.append(
sorted(versions)[-1]) # only grab the latest version
else:
out_files.append(versions[0])
return list(set(out_files))
def mms_get_state_data(probe='1',
trange=['2015-10-16', '2015-10-17'],
datatypes=['pos', 'vel'],
level='def',
no_download=False,
pred_or_def=True,
suffix='',
always_prompt=False):
"""
Helper routine for loading state data (ASCII files from the SDC); not meant to be called directly; see pyspedas.mms.state instead
"""
if not isinstance(probe, list): probe = [probe]
local_data_dir = CONFIG['local_data_dir']
download_only = CONFIG['download_only']
start_time = time_double(trange[0]) - 60 * 60 * 24.
end_time = time_double(trange[1])
# check if end date is anything other than 00:00:00, if so
# add a day to the end time to ensure that all data is downloaded
if type(trange[1]) == str:
endtime_day = time_double(
time_string(time_double(trange[1]), fmt='%Y-%m-%d'))
else:
endtime_day = time_double(time_string(trange[1], fmt='%Y-%m-%d'))
if end_time > endtime_day:
add_day = 60 * 60 * 24.
else:
add_day = 0
start_time_str = time_string(start_time, fmt='%Y-%m-%d')
end_time_str = time_string(end_time + add_day, fmt='%Y-%m-%d')
filetypes = []
if 'pos' in datatypes or 'vel' in datatypes:
filetypes.append('eph')
if 'spinras' in datatypes or 'spindec' in datatypes:
filetypes.append('att')
user = None
if not no_download:
sdc_session, user = mms_login_lasp(always_prompt=always_prompt)
for probe_id in probe:
# probe will need to be a string from now on
probe_id = str(probe_id)
for filetype in filetypes:
file_dir = local_data_dir + 'ancillary/' + 'mms' + probe_id + '/' + level + filetype + '/'
product = level + filetype
# predicted doesn't support start_date/end_date
if level == 'def':
dates_for_query = '&start_date=' + start_time_str + '&end_date=' + end_time_str
else:
dates_for_query = ''
if user == None:
url = 'https://lasp.colorado.edu/mms/sdc/public/files/api/v1/file_info/ancillary?sc_id=mms' + probe_id + '&product=' + product + dates_for_query
else:
url = 'https://lasp.colorado.edu/mms/sdc/sitl/files/api/v1/file_info/ancillary?sc_id=mms' + probe_id + '&product=' + product + dates_for_query
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=ResourceWarning)
http_request = sdc_session.get(url, verify=True)
http_json = http_request.json()
out_dir = os.sep.join([
local_data_dir, 'ancillary', 'mms' + probe_id, level + filetype
])
files_in_interval = []
out_files = []
# since predicted doesn't support start_date/end_date, we'll need to parse the correct dates
if level != 'def':
for file in http_json['files']:
# first, remove the dates that start after the end of the trange
if time_double(file['start_date']) > endtime_day:
continue
# now remove files that end before the start of the trange
if start_time > time_double(file['end_date']):
continue
files_in_interval.append(file)
break
else:
files_in_interval = http_json['files']
for file in files_in_interval:
out_file = os.sep.join([out_dir, file['file_name']])
if os.path.exists(out_file) and str(
os.stat(out_file).st_size) == str(file['file_size']):
if not download_only: logging.info('Loading ' + out_file)
out_files.append(out_file)
http_request.close()
continue
if user == None:
download_url = 'https://lasp.colorado.edu/mms/sdc/public/files/api/v1/download/ancillary?file=' + file[
'file_name']
else:
download_url = 'https://lasp.colorado.edu/mms/sdc/sitl/files/api/v1/download/ancillary?file=' + file[
'file_name']
logging.info('Downloading ' + file['file_name'] + ' to ' +
out_dir)
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=ResourceWarning)
fsrc = sdc_session.get(download_url,
stream=True,
verify=True)
ftmp = NamedTemporaryFile(delete=False)
with open(ftmp.name, 'wb') as f:
copyfileobj(fsrc.raw, f)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# if the download was successful, copy to data directory
copy(ftmp.name, out_file)
out_files.append(out_file)
fsrc.close()
ftmp.close()
if download_only:
continue
if filetype == 'eph':
mms_load_eph_tplot(sorted(out_files),
level=level,
probe=probe_id,
datatypes=datatypes,
suffix=suffix,
trange=trange)
elif filetype == 'att':
mms_load_att_tplot(sorted(out_files),
level=level,
probe=probe_id,
datatypes=datatypes,
suffix=suffix,
trange=trange)
if not no_download:
sdc_session.close()
def mms_feeps_getgyrophase(trange=['2017-07-11/22:30', '2017-07-11/22:35'],
probe='2',
data_rate='brst',
level='l2',
datatype='electron'):
"""
Calculates FEEPS gyrophase angles for electron burst data
Notes: This is intended for use with burst mode data, but the output gyrophase product,
mms*_feeps_gyrophase, can be interpolated onto FEEPS burst or survey cadence
to produce spectra
Based on the IDL code writen by Drew Turner (10 Oct 2017): mms_feeps_getgyrophase.pro
"""
mec_vars = mms.mec(trange=trange, probe=probe, data_rate=data_rate)
if mec_vars is None:
print(
'Problem loading MEC data for calculating FEEPS gyrophase angles')
qeci2sm = get_data('mms' + probe + '_mec_quat_eci_to_sm')
qeci2bcs = get_data('mms' + probe + '_mec_quat_eci_to_bcs')
rsun = get_data('mms' + probe + '_mec_r_sun_de421_eci')
rsunbcs = np.zeros((len(rsun.times), 3))
rduskbcs = np.zeros((len(rsun.times), 3))
rdusksm = [0, 1, 0]
for i in range(len(rsun.times)):
q = qeci2bcs.y[i, :]
# Quaternion rotation matrix:
s = 1 # these quaternions are unit-qs
R = np.array([
[
1 - 2 * s * (q[2]**2 + q[3]**2),
2 * s * (q[1] * q[2] - q[3] * q[0]),
2 * s * (q[1] * q[3] + q[2] * q[0])
], # ECI to BCS
[
2 * s * (q[1] * q[2] + q[3] * q[0]),
1 - 2 * s * (q[1]**2 + q[3]**2),
2 * s * (q[2] * q[3] - q[1] * q[0])
],
[
2 * s * (q[1] * q[3] - q[2] * q[0]),
2 * s * (q[2] * q[3] + q[1] * q[0]),
1 - 2 * s * (q[1]**2 + q[2]**2)
]
])
R = R.T
rsunbcs[i, :] = np.array([
R[0, 0] * rsun.y[i, 0] + R[1, 0] * rsun.y[i, 1] +
R[2, 0] * rsun.y[i, 2], R[0, 1] * rsun.y[i, 0] +
R[1, 1] * rsun.y[i, 1] + R[2, 1] * rsun.y[i, 2],
R[0, 2] * rsun.y[i, 0] + R[1, 2] * rsun.y[i, 1] +
R[2, 2] * rsun.y[i, 2]
])
# now make second vector for gyroplane reference, dusk direction (+Y in SM)
q = qeci2sm.y[i, :]
# Quaternion rotation matrix:
s = 1 # these quaternions are unit-qs
R2 = np.array([
[
1 - 2 * s * (q[2]**2 + q[3]**2),
2 * s * (q[1] * q[2] - q[3] * q[0]),
2 * s * (q[1] * q[3] + q[2] * q[0])
], # ECI to SM
[
2 * s * (q[1] * q[2] + q[3] * q[0]),
1 - 2 * s * (q[1]**2 + q[3]**2),
2 * s * (q[2] * q[3] - q[1] * q[0])
],
[
2 * s * (q[1] * q[3] - q[2] * q[0]),
2 * s * (q[2] * q[3] + q[1] * q[0]),
1 - 2 * s * (q[1]**2 + q[2]**2)
]
])
# going from SM to ECI, so invert R:
R2 = np.linalg.inv(R2) # SM to ECI
R2 = R2.T
rduskeci = [
R2[0, 0] * rdusksm[0] + R2[1, 0] * rdusksm[1] +
R2[2, 0] * rdusksm[2], R2[0, 1] * rdusksm[0] +
R2[1, 1] * rdusksm[1] + R2[2, 1] * rdusksm[2],
R2[0, 2] * rdusksm[0] + R2[1, 2] * rdusksm[1] +
R2[2, 2] * rdusksm[2]
]
# Now convert to BCS:
rduskbcs[i, :] = np.array([
R[0, 0] * rduskeci[0] + R[1, 0] * rduskeci[1] +
R[2, 0] * rduskeci[2], R[0, 1] * rduskeci[0] +
R[1, 1] * rduskeci[1] + R[2, 1] * rduskeci[2],
R[0, 2] * rduskeci[0] + R[1, 2] * rduskeci[1] +
R[2, 2] * rduskeci[2]
])
saved = store_data('mms' + probe + '_mec_r_sun_bcs',
data={
'x': rsun.times,
'y': rsunbcs
})
if not saved:
print('Problem saving r_sun_bcs')
saved = store_data('mms' + probe + '_mec_r_dusk_bcs',
data={
'x': rsun.times,
'y': rduskbcs
})
if not saved:
print('Problem saving r_dusk_bcs')
# Rotation matrices for FEEPS coord system (FCS) into body coordinate system (BCS):
Ttop = np.array([[1. / np.sqrt(2.), -1. / np.sqrt(2.), 0],
[1. / np.sqrt(2.), 1. / np.sqrt(2.), 0], [0, 0, 1]]).T
Tbot = np.array([[-1. / np.sqrt(2.), -1. / np.sqrt(2.), 0],
[-1. / np.sqrt(2.), 1. / np.sqrt(2.), 0], [0, 0, -1]]).T
# Telescope vectors in FCS:
# Electrons
V1fcs = [0.347, -0.837, 0.423]
V2fcs = [0.347, -0.837, -0.423]
V3fcs = [0.837, -0.347, 0.423]
V4fcs = [0.837, -0.347, -0.423]
V5fcs = [-0.087, 0.000, 0.996]
V9fcs = [0.837, 0.347, 0.423]
V10fcs = [0.837, 0.347, -0.423]
V11fcs = [0.347, 0.837, 0.423]
V12fcs = [0.347, 0.837, -0.423]
# Ions
V6fcs = [0.104, 0.180, 0.978]
V7fcs = [0.654, -0.377, 0.656]
V8fcs = [0.654, -0.377, -0.656]
# Now telescope vectors in Body Coordinate System:
# Factors of -1 account for 180 deg shift between particle velocity and telescope normal direction:
# Top:
Vt1bcs = [
-1. * (Ttop[0, 0] * V1fcs[0] + Ttop[1, 0] * V1fcs[1] +
Ttop[2, 0] * V1fcs[2]),
-1. * (Ttop[0, 1] * V1fcs[0] + Ttop[1, 1] * V1fcs[1] +
Ttop[2, 1] * V1fcs[2]), -1. *
(Ttop[0, 2] * V1fcs[0] + Ttop[1, 2] * V1fcs[1] + Ttop[2, 2] * V1fcs[2])
]
Vt2bcs = [
-1. * (Ttop[0, 0] * V2fcs[0] + Ttop[1, 0] * V2fcs[1] +
Ttop[2, 0] * V2fcs[2]),
-1. * (Ttop[0, 1] * V2fcs[0] + Ttop[1, 1] * V2fcs[1] +
Ttop[2, 1] * V2fcs[2]), -1. *
(Ttop[0, 2] * V2fcs[0] + Ttop[1, 2] * V2fcs[1] + Ttop[2, 2] * V2fcs[2])
]
Vt3bcs = [
-1. * (Ttop[0, 0] * V3fcs[0] + Ttop[1, 0] * V3fcs[1] +
Ttop[2, 0] * V3fcs[2]),
-1. * (Ttop[0, 1] * V3fcs[0] + Ttop[1, 1] * V3fcs[1] +
Ttop[2, 1] * V3fcs[2]), -1. *
(Ttop[0, 2] * V3fcs[0] + Ttop[1, 2] * V3fcs[1] + Ttop[2, 2] * V3fcs[2])
]
Vt4bcs = [
-1. * (Ttop[0, 0] * V4fcs[0] + Ttop[1, 0] * V4fcs[1] +
Ttop[2, 0] * V4fcs[2]),
-1. * (Ttop[0, 1] * V4fcs[0] + Ttop[1, 1] * V4fcs[1] +
Ttop[2, 1] * V4fcs[2]), -1. *
(Ttop[0, 2] * V4fcs[0] + Ttop[1, 2] * V4fcs[1] + Ttop[2, 2] * V4fcs[2])
]
Vt5bcs = [
-1. * (Ttop[0, 0] * V5fcs[0] + Ttop[1, 0] * V5fcs[1] +
Ttop[2, 0] * V5fcs[2]),
-1. * (Ttop[0, 1] * V5fcs[0] + Ttop[1, 1] * V5fcs[1] +
Ttop[2, 1] * V5fcs[2]), -1. *
(Ttop[0, 2] * V5fcs[0] + Ttop[1, 2] * V5fcs[1] + Ttop[2, 2] * V5fcs[2])
]
Vt6bcs = [
-1. * (Ttop[0, 0] * V6fcs[0] + Ttop[1, 0] * V6fcs[1] +
Ttop[2, 0] * V6fcs[2]),
-1. * (Ttop[0, 1] * V6fcs[0] + Ttop[1, 1] * V6fcs[1] +
Ttop[2, 1] * V6fcs[2]), -1. *
(Ttop[0, 2] * V6fcs[0] + Ttop[1, 2] * V6fcs[1] + Ttop[2, 2] * V6fcs[2])
]
Vt7bcs = [
-1. * (Ttop[0, 0] * V7fcs[0] + Ttop[1, 0] * V7fcs[1] +
Ttop[2, 0] * V7fcs[2]),
-1. * (Ttop[0, 1] * V7fcs[0] + Ttop[1, 1] * V7fcs[1] +
Ttop[2, 1] * V7fcs[2]), -1. *
(Ttop[0, 2] * V7fcs[0] + Ttop[1, 2] * V7fcs[1] + Ttop[2, 2] * V7fcs[2])
]
Vt8bcs = [
-1. * (Ttop[0, 0] * V8fcs[0] + Ttop[1, 0] * V8fcs[1] +
Ttop[2, 0] * V8fcs[2]),
-1. * (Ttop[0, 1] * V8fcs[0] + Ttop[1, 1] * V8fcs[1] +
Ttop[2, 1] * V8fcs[2]), -1. *
(Ttop[0, 2] * V8fcs[0] + Ttop[1, 2] * V8fcs[1] + Ttop[2, 2] * V8fcs[2])
]
Vt9bcs = [
-1. * (Ttop[0, 0] * V9fcs[0] + Ttop[1, 0] * V9fcs[1] +
Ttop[2, 0] * V9fcs[2]),
-1. * (Ttop[0, 1] * V9fcs[0] + Ttop[1, 1] * V9fcs[1] +
Ttop[2, 1] * V9fcs[2]), -1. *
(Ttop[0, 2] * V9fcs[0] + Ttop[1, 2] * V9fcs[1] + Ttop[2, 2] * V9fcs[2])
]
Vt10bcs = [
-1. * (Ttop[0, 0] * V10fcs[0] + Ttop[1, 0] * V10fcs[1] +
Ttop[2, 0] * V10fcs[2]),
-1. * (Ttop[0, 1] * V10fcs[0] + Ttop[1, 1] * V10fcs[1] +
Ttop[2, 1] * V10fcs[2]),
-1. * (Ttop[0, 2] * V10fcs[0] + Ttop[1, 2] * V10fcs[1] +
Ttop[2, 2] * V10fcs[2])
]
Vt11bcs = [
-1. * (Ttop[0, 0] * V11fcs[0] + Ttop[1, 0] * V11fcs[1] +
Ttop[2, 0] * V11fcs[2]),
-1. * (Ttop[0, 1] * V11fcs[0] + Ttop[1, 1] * V11fcs[1] +
Ttop[2, 1] * V11fcs[2]),
-1. * (Ttop[0, 2] * V11fcs[0] + Ttop[1, 2] * V11fcs[1] +
Ttop[2, 2] * V11fcs[2])
]
Vt12bcs = [
-1. * (Ttop[0, 0] * V12fcs[0] + Ttop[1, 0] * V12fcs[1] +
Ttop[2, 0] * V12fcs[2]),
-1. * (Ttop[0, 1] * V12fcs[0] + Ttop[1, 1] * V12fcs[1] +
Ttop[2, 1] * V12fcs[2]),
-1. * (Ttop[0, 2] * V12fcs[0] + Ttop[1, 2] * V12fcs[1] +
Ttop[2, 2] * V12fcs[2])
]
# Bottom:
Vb1bcs = [
-1. * (Tbot[0, 0] * V1fcs[0] + Tbot[1, 0] * V1fcs[1] +
Tbot[2, 0] * V1fcs[2]),
-1. * (Tbot[0, 1] * V1fcs[0] + Tbot[1, 1] * V1fcs[1] +
Tbot[2, 1] * V1fcs[2]), -1. *
(Tbot[0, 2] * V1fcs[0] + Tbot[1, 2] * V1fcs[1] + Tbot[2, 2] * V1fcs[2])
]
Vb2bcs = [
-1. * (Tbot[0, 0] * V2fcs[0] + Tbot[1, 0] * V2fcs[1] +
Tbot[2, 0] * V2fcs[2]),
-1. * (Tbot[0, 1] * V2fcs[0] + Tbot[1, 1] * V2fcs[1] +
Tbot[2, 1] * V2fcs[2]), -1. *
(Tbot[0, 2] * V2fcs[0] + Tbot[1, 2] * V2fcs[1] + Tbot[2, 2] * V2fcs[2])
]
Vb3bcs = [
-1. * (Tbot[0, 0] * V3fcs[0] + Tbot[1, 0] * V3fcs[1] +
Tbot[2, 0] * V3fcs[2]),
-1. * (Tbot[0, 1] * V3fcs[0] + Tbot[1, 1] * V3fcs[1] +
Tbot[2, 1] * V3fcs[2]), -1. *
(Tbot[0, 2] * V3fcs[0] + Tbot[1, 2] * V3fcs[1] + Tbot[2, 2] * V3fcs[2])
]
Vb4bcs = [
-1. * (Tbot[0, 0] * V4fcs[0] + Tbot[1, 0] * V4fcs[1] +
Tbot[2, 0] * V4fcs[2]),
-1. * (Tbot[0, 1] * V4fcs[0] + Tbot[1, 1] * V4fcs[1] +
Tbot[2, 1] * V4fcs[2]), -1. *
(Tbot[0, 2] * V4fcs[0] + Tbot[1, 2] * V4fcs[1] + Tbot[2, 2] * V4fcs[2])
]
Vb5bcs = [
-1. * (Tbot[0, 0] * V5fcs[0] + Tbot[1, 0] * V5fcs[1] +
Tbot[2, 0] * V5fcs[2]),
-1. * (Tbot[0, 1] * V5fcs[0] + Tbot[1, 1] * V5fcs[1] +
Tbot[2, 1] * V5fcs[2]), -1. *
(Tbot[0, 2] * V5fcs[0] + Tbot[1, 2] * V5fcs[1] + Tbot[2, 2] * V5fcs[2])
]
Vb6bcs = [
-1. * (Tbot[0, 0] * V6fcs[0] + Tbot[1, 0] * V6fcs[1] +
Tbot[2, 0] * V6fcs[2]),
-1. * (Tbot[0, 1] * V6fcs[0] + Tbot[1, 1] * V6fcs[1] +
Tbot[2, 1] * V6fcs[2]), -1. *
(Tbot[0, 2] * V6fcs[0] + Tbot[1, 2] * V6fcs[1] + Tbot[2, 2] * V6fcs[2])
]
Vb7bcs = [
-1. * (Tbot[0, 0] * V7fcs[0] + Tbot[1, 0] * V7fcs[1] +
Tbot[2, 0] * V7fcs[2]),
-1. * (Tbot[0, 1] * V7fcs[0] + Tbot[1, 1] * V7fcs[1] +
Tbot[2, 1] * V7fcs[2]), -1. *
(Tbot[0, 2] * V7fcs[0] + Tbot[1, 2] * V7fcs[1] + Tbot[2, 2] * V7fcs[2])
]
Vb8bcs = [
-1. * (Tbot[0, 0] * V8fcs[0] + Tbot[1, 0] * V8fcs[1] +
Tbot[2, 0] * V8fcs[2]),
-1. * (Tbot[0, 1] * V8fcs[0] + Tbot[1, 1] * V8fcs[1] +
Tbot[2, 1] * V8fcs[2]), -1. *
(Tbot[0, 2] * V8fcs[0] + Tbot[1, 2] * V8fcs[1] + Tbot[2, 2] * V8fcs[2])
]
Vb9bcs = [
-1. * (Tbot[0, 0] * V9fcs[0] + Tbot[1, 0] * V9fcs[1] +
Tbot[2, 0] * V9fcs[2]),
-1. * (Tbot[0, 1] * V9fcs[0] + Tbot[1, 1] * V9fcs[1] +
Tbot[2, 1] * V9fcs[2]), -1. *
(Tbot[0, 2] * V9fcs[0] + Tbot[1, 2] * V9fcs[1] + Tbot[2, 2] * V9fcs[2])
]
Vb10bcs = [
-1. * (Tbot[0, 0] * V10fcs[0] + Tbot[1, 0] * V10fcs[1] +
Tbot[2, 0] * V10fcs[2]),
-1. * (Tbot[0, 1] * V10fcs[0] + Tbot[1, 1] * V10fcs[1] +
Tbot[2, 1] * V10fcs[2]),
-1. * (Tbot[0, 2] * V10fcs[0] + Tbot[1, 2] * V10fcs[1] +
Tbot[2, 2] * V10fcs[2])
]
Vb11bcs = [
-1. * (Tbot[0, 0] * V11fcs[0] + Tbot[1, 0] * V11fcs[1] +
Tbot[2, 0] * V11fcs[2]),
-1. * (Tbot[0, 1] * V11fcs[0] + Tbot[1, 1] * V11fcs[1] +
Tbot[2, 1] * V11fcs[2]),
-1. * (Tbot[0, 2] * V11fcs[0] + Tbot[1, 2] * V11fcs[1] +
Tbot[2, 2] * V11fcs[2])
]
Vb12bcs = [
-1. * (Tbot[0, 0] * V12fcs[0] + Tbot[1, 0] * V12fcs[1] +
Tbot[2, 0] * V12fcs[2]),
-1. * (Tbot[0, 1] * V12fcs[0] + Tbot[1, 1] * V12fcs[1] +
Tbot[2, 1] * V12fcs[2]),
-1. * (Tbot[0, 2] * V12fcs[0] + Tbot[1, 2] * V12fcs[1] +
Tbot[2, 2] * V12fcs[2])
]
fgm_vars = mms.fgm(
trange=[time_double(trange[0]) - 600,
time_double(trange[1]) + 600],
probe=probe,
data_rate='srvy')
if fgm_vars is None:
print(
'Problem loading FGM vars for calculating FEEPS gyrophase angles')
# interpolate the FGM var to the MEC var timestamps
tinterpol('mms' + probe + '_fgm_b_bcs_srvy_l2_bvec',
'mms' + probe + '_mec_r_sun_bcs',
newname='mms' + probe + '_fgm_b_bcs_srvy_l2_bvec_int')
B = get_data('mms' + probe + '_fgm_b_bcs_srvy_l2_bvec_int')
# Now calculate gyrophase
# Telescope vectors perp to B:
Tperp = np.zeros((len(rsunbcs[:, 0]), 3, 24))
# Gyrophase:
phi = np.zeros((len(rsunbcs[:, 0]), 24))
for i in range(len(rsunbcs[:, 0])):
uB = B.y[i, :] / np.sqrt(B.y[i, 0]**2 + B.y[i, 1]**2 + B.y[i, 2]**2)
# Sun vector perp to B:
Sperp = np.cross(
np.cross(uB, rsunbcs[i, :] / np.sqrt(np.nansum(rsunbcs[i, :]**2))),
uB)
# Dusk vector perp to B:
Dperp = np.cross(
np.cross(uB,
rduskbcs[i, :] / np.sqrt(np.nansum(rduskbcs[i, :]**2))),
uB)
Tperp[i, :, 0] = np.cross(np.cross(uB, Vt1bcs), uB)
Tperp[i, :, 1] = np.cross(np.cross(uB, Vt2bcs), uB)
Tperp[i, :, 2] = np.cross(np.cross(uB, Vt3bcs), uB)
Tperp[i, :, 3] = np.cross(np.cross(uB, Vt4bcs), uB)
Tperp[i, :, 4] = np.cross(np.cross(uB, Vt5bcs), uB)
Tperp[i, :, 5] = np.cross(np.cross(uB, Vt6bcs), uB)
Tperp[i, :, 6] = np.cross(np.cross(uB, Vt7bcs), uB)
Tperp[i, :, 7] = np.cross(np.cross(uB, Vt8bcs), uB)
Tperp[i, :, 8] = np.cross(np.cross(uB, Vt9bcs), uB)
Tperp[i, :, 9] = np.cross(np.cross(uB, Vt10bcs), uB)
Tperp[i, :, 10] = np.cross(np.cross(uB, Vt11bcs), uB)
Tperp[i, :, 11] = np.cross(np.cross(uB, Vt12bcs), uB)
Tperp[i, :, 12] = np.cross(np.cross(uB, Vb1bcs), uB)
Tperp[i, :, 13] = np.cross(np.cross(uB, Vb2bcs), uB)
Tperp[i, :, 14] = np.cross(np.cross(uB, Vb3bcs), uB)
Tperp[i, :, 15] = np.cross(np.cross(uB, Vb4bcs), uB)
Tperp[i, :, 16] = np.cross(np.cross(uB, Vb5bcs), uB)
Tperp[i, :, 17] = np.cross(np.cross(uB, Vb6bcs), uB)
Tperp[i, :, 18] = np.cross(np.cross(uB, Vb7bcs), uB)
Tperp[i, :, 19] = np.cross(np.cross(uB, Vb8bcs), uB)
Tperp[i, :, 20] = np.cross(np.cross(uB, Vb9bcs), uB)
Tperp[i, :, 21] = np.cross(np.cross(uB, Vb10bcs), uB)
Tperp[i, :, 22] = np.cross(np.cross(uB, Vb11bcs), uB)
Tperp[i, :, 23] = np.cross(np.cross(uB, Vb12bcs), uB)
for j in range(24):
th1 = np.arccos(
np.nansum(Tperp[i, :, j] * Sperp) /
(np.sqrt(np.nansum(Tperp[i, :, j]**2)) *
np.sqrt(np.nansum(Sperp**2))))
th2 = np.arccos(
np.nansum(Tperp[i, :, j] * Dperp) /
(np.sqrt(np.nansum(Tperp[i, :, j]**2)) *
np.sqrt(np.nansum(Dperp**2))))
if th1 <= np.pi / 2.0 and th2 < np.pi / 2:
phi[i, j] = 2 * np.pi - th1
if th1 < np.pi / 2.0 and th2 >= np.pi / 2.0:
phi[i, j] = th1
if th1 > np.pi / 2.0 and th2 <= np.pi / 2.0:
phi[i, j] = 270.0 * np.pi / 180.0 - th2
if th1 >= np.pi / 2.0 and th2 > np.pi / 2.0:
phi[i, j] = th1
saved = store_data('mms' + probe + '_epd_feeps_' + data_rate +
'_gyrophase',
data={
'x': rsun.times,
'y': phi * 180. / np.pi
})
if not saved:
print('Problem saving gyrophase angles')
return
options('mms' + probe + '_epd_feeps_' + data_rate + '_gyrophase', 'yrange',
[0, 360.0])
# Gyrophase always returns on time stamps from MEC data, get those closest to FEEPS time stamps:
eyes = mms_feeps_active_eyes(trange, probe, data_rate, datatype, level)
sensor_types = ['top', 'bottom']
feepst = get_data('mms' + probe + '_epd_feeps_' + data_rate + '_' + level +
'_' + datatype + '_spinsectnum')
indt = np.zeros(len(feepst.times), dtype='int32')
gpd = get_data('mms' + probe + '_epd_feeps_' + data_rate + '_gyrophase')
for i in range(len(feepst.times)):
indt[i] = np.argwhere(
np.abs(gpd.times - feepst.times[i]) == np.min(
np.abs(gpd.times - feepst.times[i]))).flatten()[0]
# Gyrophase always returns all 24 FEEPS telescopes, downselect based on species:
iT = np.array([
np.array(eyes[sensor_types[0]]) - 1,
np.array(eyes[sensor_types[0]]) + 11
]).flatten().tolist()
gp_data = np.zeros((len(gpd.times[indt]), len(iT)))
#return (iT, gp_data, gpd)
for i in range(len(iT)):
gp_data[:, i] = gpd.y[indt, iT[i]]
saved = store_data('mms' + probe + '_epd_feeps_' + data_rate + '_' +
level + '_' + datatype + '_gyrophase',
data={
'x': gpd.times[indt],
'y': gp_data
})
if saved:
options(
'mms' + probe + '_epd_feeps_' + data_rate + '_' + level + '_' +
datatype + '_gyrophase', 'yrange', [0.0, 360.0])
return 'mms' + probe + '_epd_feeps_' + data_rate + '_' + level + '_' + datatype + '_gyrophase'
def hapi(trange=None, server=None, dataset=None, parameters='', suffix='',
catalog=False):
"""
Loads data from a HAPI server into pytplot variables
Parameters
-----------
trange: list of str or list of float
Time range to load the data for
server: str
HAPI server to load the data from
dataset: str
HAPI dataset to load
parameters: str or list of str
Parameters in the dataset to load; default
is to load them all
suffix: str
Suffix to append to the tplot variables
catalog: bool
If True, returns the server's catalog of datasets
Returns
-------
List of tplot variables created.
"""
if server is None:
print('Error, no server specified; example servers include:')
print('- https://cdaweb.gsfc.nasa.gov/hapi')
print('- https://pds-ppi.igpp.ucla.edu/hapi')
print('- http://planet.physics.uiowa.edu/das/das2Server/hapi')
print('- https://iswa.gsfc.nasa.gov/IswaSystemWebApp/hapi')
print('- http://lasp.colorado.edu/lisird/hapi')
return
if catalog:
catalog = load_hapi(server)
items = []
if 'catalog' in catalog.keys():
items = catalog['catalog']
print('Available datasets: ')
for item in items:
if 'title' in item.keys():
print(item['id'] + ': ' + item['title'])
else:
print(item['id'])
return
if dataset is None:
print('Error, no dataset specified; please see the catalog for a list of available data sets.')
return
if trange is None:
print('Error, no trange specified')
return
if isinstance(parameters, list):
parameters = ','.join(parameters)
opts = {'logging': False}
data, hapi_metadata = load_hapi(server, dataset, parameters, trange[0], trange[1], **opts)
out_vars = []
# loop through the parameters in this dataset
params = hapi_metadata['parameters']
for param in params[1:]:
spec = False
param_name = param.get('name')
print('Loading ' + param_name)
# load the data only for this parameter
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=ResourceWarning)
data, hapi_metadata = load_hapi(server, dataset, param_name, trange[0], trange[1], **opts)
except:
breakpoint()
print('Error! 95')
continue
timestamps = [datapoint[0] for datapoint in data]
unixtimes = [time_double(timestamp.decode('utf-8')) for timestamp in timestamps]
param_type = hapi_metadata['parameters'][1].get('type')
if param_type is None:
param_type = 'double'
data_size = hapi_metadata['parameters'][1].get('size')
if data_size is None:
single_line = True
try:
if param_type == 'double':
single_line = isinstance(data[0][1], np.float64)
elif param_type == 'integer':
single_line = isinstance(data[0][1], np.int32)
except IndexError:
breakpoint()
print('Error! 103')
continue
if single_line:
data_out = np.zeros((len(data)))
else:
try:
data_out = np.zeros((len(data), len(data[0][1])))
except TypeError:
print('Error! 112')
breakpoint()
continue
for idx, datapoint in enumerate(data):
if single_line:
data_out[idx] = datapoint[1]
else:
data_out[idx, :] = datapoint[1]
data_out = data_out.squeeze()
# check for fill values
fill_value = hapi_metadata['parameters'][1].get('fill')
if fill_value is not None:
if param_type == 'double':
fill_value = float(fill_value)
data_out[data_out == fill_value] = np.nan
elif param_type == 'integer':
# NaN is only floating point, so we replace integer fill
# values with 0 instead of NaN
fill_value = int(fill_value)
data_out[data_out == fill_value] = 0
bins = param.get('bins')
if bins is not None:
centers = bins[0].get('centers')
if centers is not None:
spec = True
data_table = {'x': unixtimes, 'y': data_out}
if spec:
data_table['v'] = centers
saved = store_data(param_name + suffix, data=data_table)
metadata = get_data(param_name + suffix, metadata=True)
metadata['HAPI'] = hapi_metadata
if spec:
options(param_name + suffix, 'spec', True)
if saved:
out_vars.append(param_name + suffix)
# wait for a second before going to the next variable
# to avoid hitting the server too quickly
sleep(1)
return out_vars
