def mms_feeps_active_eyes(trange, probe, data_rate, species, level): if data_rate.lower() == 'brst' and species.lower() == 'electron': return {'top': [1, 2, 3, 4, 5, 9, 10, 11, 12], 'bottom': [1, 2, 3, 4, 5, 9, 10, 11, 12]} if data_rate.lower() == 'brst' and species.lower() == 'ion': return {'top': [6, 7, 8], 'bottom': [6, 7, 8]} # old eyes, srvy mode, prior to 16 August 2017 if species.lower() == 'electron': sensors = {'top': [3, 4, 5, 11, 12], 'bottom': [3, 4, 5, 11, 12]} else: sensors = {'top': [6, 7, 8], 'bottom': [6, 7, 8]} if isinstance(trange[0], str): start_time = time_double(trange[0]) else: start_time = trange[0] # srvy mode, after 16 August 2017 if start_time >= time_double('2017-08-16') and data_rate.lower() == 'srvy': active_table = {'1-electron': {'top': [3, 5, 9, 10, 12], 'bottom': [2, 4, 5, 9, 10]}, '1-ion': {'top': [6, 7, 8], 'bottom': [6, 7, 8]}, '2-electron': {'top': [1, 2, 3, 5, 10, 11], 'bottom': [1, 4, 5, 9, 11]}, '2-ion': {'top': [6, 8], 'bottom': [6, 7, 8]}, '3-electron': {'top': [3, 5, 9, 10, 12], 'bottom': [1, 2, 3, 9, 10]}, '3-ion': {'top': [6, 7, 8], 'bottom': [6, 7, 8]}, '4-electron': {'top': [3, 4, 5, 9, 10, 11], 'bottom': [3, 5, 9, 10, 12]}, '4-ion': {'top': [6, 8], 'bottom': [6, 7, 8]}} sensors = active_table[probe.lower()+'-'+species.lower()] if level.lower() == 'sitl': return {'top': list(set(sensors['top']) & set([5, 11, 12])), 'bottom': []} if level.lower() == 'sitl': return {'top': [5, 11, 12], 'bottom': []} return sensors
def test_time_double(self): self.assertTrue( time_double('2015-12-15 12:07:23.767000') == 1450181243.767) self.assertTrue( time_double([ '2015-12-15 12:07:23.767000', '2015-12-15 12:07:43.767000' ]) == [1450181243.767, 1450181263.767])
def get_mms_srois(start_time=None, end_time=None, sc_id=None): if start_time is None: logging.error('Error, start time not specified') return if end_time is None: logging.error('Error, end time not specified') return if sc_id is None: logging.error('Error, sc_id not specified') return # public unauthenticated path path = 'mms/sdc/public/service/latis/mms_events_view.csv' query = '?start_time_utc,end_time_utc,sc_id,start_orbit&event_type=SROI' # convert to standard ISO format as accepted by LaTiS query += '&start_time_utc>=' + 'T'.join(start_time.split(' ')) query += '&start_time_utc<' + 'T'.join(end_time.split(' ')) query += '&sc_id=' + sc_id.lower() qreq = requests.get('https://lasp.colorado.edu/' + path + query) if qreq.status_code != 200: logging.error('Error downloading SRoI segments') return sroi_data = csv.reader(qreq.content.decode('utf-8').splitlines()) count = 0 unix_start = [] unix_end = [] for line in sroi_data: if count == 0: # ignore the first line count += 1 continue if len(line) != 4: continue unix_start.append(time_double(line[0])) unix_end.append(time_double(line[1])) count += 1 return (np.array(unix_start), np.array(unix_end))
def dailynames(directory='', trange=None, res=24 * 3600., hour_res=False, file_format='%Y%m%d', prefix='', suffix=''): if trange == None: print('No trange specified') return if hour_res == True: res = 3600. file_format = '%Y%m%d%H' # allows the user to pass in trange as list of datetime objects if type(trange[0]) == datetime and type(trange[1]) == datetime: trange = [ time_string(trange[0].timestamp()), time_string(trange[1].timestamp()) ] tr = [trange[0], trange[1]] if isinstance(trange[0], str): tr[0] = time_double(trange[0]) if isinstance(trange[1], str): tr[1] = time_double(trange[1]) # Davin's magic heisted from file_dailynames in IDL mmtr = [np.floor(tr[0] / res), np.ceil(tr[1] / res)] if mmtr[1] - mmtr[0] < 1: n = 1 else: n = int(mmtr[1] - mmtr[0]) times = [(float(num) + mmtr[0]) * res for num in range(n)] dates = [] files = [] for time in times: if time_string(time, fmt=file_format) not in dates: dates.append(time_string(time, fmt=file_format)) for date in dates: files.append(directory + prefix + date + suffix) return files
def get_params(model): support_trange = [ time_double(trange[0]) - 60 * 60 * 24, time_double(trange[1]) + 60 * 60 * 24 ] pyspedas.kyoto.dst(trange=support_trange) pyspedas.omni.data(trange=trange) join_vec(['BX_GSE', 'BY_GSM', 'BZ_GSM']) return get_tsy_params('kyoto_dst', 'BX_GSE-BY_GSM-BZ_GSM_joined', 'proton_density', 'flow_speed', model, pressure_tvar='Pressure', speed=True)
def mms_read_feeps_sector_masks_csv(trange): """ This function returns the FEEPS sectors to mask due to sunlight contamination Parameters: trange : list of str time range of interest [starttime, endtime] with the format 'YYYY-MM-DD','YYYY-MM-DD'] or to specify more or less than a day ['YYYY-MM-DD/hh:mm:ss','YYYY-MM-DD/hh:mm:ss'] Returns: Hash table containing the sectors to mask for each spacecraft and sensor ID """ masks = {} dates = [ 1447200000.0000000, # 11/11/2015 1468022400.0000000, # 7/9/2016 1477612800.0000000, # 10/28/2016 1496188800.0000000, # 5/31/2017 1506988800.0000000, # 10/3/2017 1538697600.0000000 ] # 10/5/2018 # find the file closest to the start time nearest_date = dates[(np.abs(np.array(dates) - time_double(trange[0]))).argmin()] for mms_sc in [1, 2, 3, 4]: csv_file = os.sep.join([ os.path.dirname(os.path.abspath(__file__)), 'sun', 'MMS' + str(mms_sc) + '_FEEPS_ContaminatedSectors_' + time_string(nearest_date, fmt='%Y%m%d') + '.csv' ]) csv_file = open(csv_file, 'r') csv_reader = csv.reader(csv_file) csv_data = [] for line in csv_reader: csv_data.append([float(l) for l in line]) csv_file.close() csv_data = np.array(csv_data) for i in range(0, 12): mask_vals = [] for val_idx in range(0, len(csv_data[:, i])): if csv_data[val_idx, i] == 1: mask_vals.append(val_idx) masks['mms' + str(mms_sc) + 'imaskt' + str(i + 1)] = mask_vals for i in range(0, 12): mask_vals = [] for val_idx in range(0, len(csv_data[:, i + 12])): if csv_data[val_idx, i + 12] == 1: mask_vals.append(val_idx) masks['mms' + str(mms_sc) + 'imaskb' + str(i + 1)] = mask_vals return masks
def spd_mms_load_bss(trange=['2015-10-16', '2015-10-17'], datatype=['fast', 'burst'], include_labels=False, probe='1', nodownload=False): """ Creates tplot variables which allow you to display horizontal color bars indicating burst data availability. Parameters ----------- trange: list of str Time frame for the bars datatype: str or list of str Type of BSS data (current valid options: 'fast', 'burst') include_labels: bool Flag to have horizontal bars labeled probe: str or int S/C probe # for SRoI bars (used as fast survey segments after 6Nov15; default: 1) """ if not isinstance(datatype, list): datatype = [datatype] if include_labels: burst_label = 'Burst' fast_label = 'Fast' else: burst_label = '' fast_label = '' for dtype in datatype: if dtype == 'fast': if time_double(trange[0]) <= time_double('2015-11-06'): # use the old fast segments code for dates before 6Nov15 out = mms_load_fast_segments(trange=trange) else: # use SRoI code for dates on and after 6Nov15 out = mms_load_sroi_segments(trange=trange, probe=probe) elif dtype == 'burst': out = mms_load_brst_segments(trange=trange) else: print('Unsupported datatype: ' + dtype + '; valid options: "fast" and "burst"') continue if out is None: print('Problem loading segments for ' + dtype)
def mms_load_fast_segments(trange=None, suffix=''): ''' This function loads the fast segment intervals Parameters: trange : list of str time range of interest [starttime, endtime] with the format 'YYYY-MM-DD','YYYY-MM-DD'] or to specify more or less than a day ['YYYY-MM-DD/hh:mm:ss','YYYY-MM-DD/hh:mm:ss'] Returns: Tuple containing (start_times, end_times) ''' if trange == None: logging.error('Error; no trange specified.') return None tr = time_double(trange) save_file = os.path.join(CONFIG['local_data_dir'], 'mms_fast_intervals.sav') fast_file = download(remote_file='http://www.spedas.org/mms/mms_fast_intervals.sav', local_file=save_file) try: intervals = readsav(save_file) except FileNotFoundError: logging.error('Error loading fast intervals sav file: ' + save_file) return None unix_start = np.flip(intervals['fast_intervals'].start_times[0]) unix_end = np.flip(intervals['fast_intervals'].end_times[0]) times_in_range = (unix_start >= tr[0]-2*86400.0) & (unix_start <= tr[1]+2*86400.0) unix_start = unix_start[times_in_range] unix_end = unix_end[times_in_range] bar_x = [] bar_y = [] for start_time, end_time in zip(unix_start, unix_end): if end_time >= tr[0] and start_time <= tr[1]: bar_x.extend([start_time, start_time, end_time, end_time]) bar_y.extend([np.nan, 0., 0., np.nan]) vars_created = store_data('mms_bss_fast'+suffix, data={'x': bar_x, 'y': bar_y}) options('mms_bss_fast'+suffix, 'panel_size', 0.09) options('mms_bss_fast'+suffix, 'thick', 20) options('mms_bss_fast'+suffix, 'Color', 'green') options('mms_bss_fast'+suffix, 'border', False) options('mms_bss_fast'+suffix, 'yrange', [-0.001,0.001]) options('mms_bss_fast'+suffix, 'legend_names', ['Fast']) options('mms_bss_fast'+suffix, 'ytitle', '') return (unix_start, unix_end)
def dailynames(directory='', trange=None, res=24 * 3600., hour_res=False, file_format='%Y%m%f', dir_format='', YYYY_MM_DIR=False, prefix='', suffix=''): if trange == None: print('No trange specified') return if hour_res == True: res = 3600. file_format = '%Y%m%d%H' if YYYY_MM_DIR: dir_format = '%Y/%m/' tr = [time_double(trange[0]), time_double(trange[1])] # Davin's magic heisted from file_dailynames in IDL mmtr = [np.floor(tr[0] / res), np.ceil(tr[1] / res)] if mmtr[1] - mmtr[0] < 1: n = 1 else: n = int(mmtr[1] - mmtr[0]) times = [(float(num) + mmtr[0]) * res for num in range(n)] dates = [] files = [] for time in times: if time_string(time, fmt=file_format) not in dates: dates.append(time_string(time, fmt=file_format)) for date in dates: files.append(directory + prefix + date + suffix) return files
def parse_html(html_text, year=None, month=None): times = [] data = [] # remove all of the HTML before the table html_data = html_text[html_text.find('Hourly Equatorial Dst Values'):] # remove all of the HTML after the table html_data = html_data[:html_data. find('<!-- vvvvv S yyyymm_part3.html vvvvv -->')] html_lines = html_data.split('\n') data_strs = html_lines[5:] # loop over days for day_str in data_strs: # the first element of hourly_data is the day, the rest are the hourly Dst values hourly_data = day_str.split() if len(hourly_data[1:]) != 24: continue for idx, dst_value in enumerate(hourly_data[1:]): times.append( time_double(year + '-' + month + '-' + hourly_data[0] + '/' + str(idx) + ':30')) data.append(float(dst_value)) return (times, data)
def mms_feeps_remove_bad_data(probe='1', data_rate='srvy', datatype='electron', level='l2', suffix=''): data_rate_level = data_rate + '_' + level # electrons first, remove bad eyes #;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 1. BAD EYES ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; # First, here is a list of the EYES that are bad, we need to make sure these # data are not usable (i.e., make all of the counts/rate/flux data from these eyes NAN). # These are for all modes, burst and survey: bad_data_table = {} bad_data_table['2017-10-01'] = {} bad_data_table['2017-10-01']['mms1'] = {'top': [1], 'bottom': [1, 11]} bad_data_table['2017-10-01']['mms2'] = {'top': [5, 7, 12], 'bottom': [7]} bad_data_table['2017-10-01']['mms3'] = { 'top': [2, 12], 'bottom': [2, 5, 11] } bad_data_table['2017-10-01']['mms4'] = { 'top': [1, 2, 7], 'bottom': [2, 4, 5, 10, 11] } bad_data_table['2018-10-01'] = {} bad_data_table['2018-10-01']['mms1'] = {'top': [1], 'bottom': [1, 11]} bad_data_table['2018-10-01']['mms2'] = {'top': [7, 12], 'bottom': [2, 12]} bad_data_table['2018-10-01']['mms3'] = {'top': [1, 2], 'bottom': [5, 11]} bad_data_table['2018-10-01']['mms4'] = {'top': [1, 7], 'bottom': [4, 11]} dates = np.asarray(time_double(list(bad_data_table.keys()))) closest_table_tm = (np.abs(dates - dt.datetime.now().timestamp())).argmin() closest_table = time_string(dates[closest_table_tm], '%Y-%m-%d') bad_data = bad_data_table[closest_table]['mms' + probe] bad_vars = [] # top electrons for bad_var in bad_data['top']: if bad_var in [6, 7, 8]: continue # ion eyes bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_top_count_rate_sensorid_' + str(bad_var) + suffix)) bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_top_intensity_sensorid_' + str(bad_var) + suffix)) bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_top_counts_sensorid_' + str(bad_var) + suffix)) # bottom electrons for bad_var in bad_data['bottom']: if bad_var in [6, 7, 8]: continue # ion eyes bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_bottom_count_rate_sensorid_' + str(bad_var) + suffix)) bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_bottom_intensity_sensorid_' + str(bad_var) + suffix)) bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_bottom_counts_sensorid_' + str(bad_var) + suffix)) # top ions for bad_var in bad_data['top']: if bad_var not in [6, 7, 8]: continue # ion eyes bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_top_count_rate_sensorid_' + str(bad_var) + suffix)) bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_top_intensity_sensorid_' + str(bad_var) + suffix)) bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_top_counts_sensorid_' + str(bad_var) + suffix)) # bottom ions for bad_var in bad_data['top']: if bad_var not in [6, 7, 8]: continue # ion eyes bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_bottom_count_rate_sensorid_' + str(bad_var) + suffix)) bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_bottom_intensity_sensorid_' + str(bad_var) + suffix)) bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_bottom_counts_sensorid_' + str(bad_var) + suffix)) for bad_var in bad_vars: if bad_var == []: continue bad_var_data = pytplot.get_data(bad_var[0]) if bad_var_data is not None: times, data = bad_var_data energies = pytplot.data_quants[bad_var[0]].spec_bins.values # check if the energy table contains all names if np.isnan(np.sum(energies)): continue data[:] = np.nan pytplot.store_data(bad_var[0], data={ 'x': times, 'y': data, 'v': energies.reshape(energies.size) }) # ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 2. BAD LOWEST E-CHANNELS ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; # ; Next, these eyes have bad first channels (i.e., lowest energy channel, E-channel 0 in IDL indexing). # ; Again, these data (just the counts/rate/flux from the lowest energy channel ONLY!!!) # ; should be hardwired to be NAN for all modes (burst and both types of survey). # ; The eyes not listed here or above are ok though... so once we do this, we can actually start # ; showing the data down to the lowest levels (~33 keV), meaning we'll have to adjust the hard-coded # ; ylim settings in SPEDAS and the SITL software: # ; from Drew Turner, 5Oct18: # ;Bad Channels (0 and 1): # ;Update: All channels 0 (Ch0) on MMS-2, -3, and -4 electron eyes (1, 2, 3, 4, 5, 9, 10, 11, 12) should be NaN # ;Additionally, the second channels (Ch1) on the following should also be made NaN: # ;MMS-1: Top: Ch0 on Eyes 6, 7 # ;Bot: Ch0 on Eyes 6, 7, 8 # ;MMS-2: Top: # ;Bot: Ch0 on Eyes 6, 8 # ;MMS-3: Top: Ch0 on Eye 8 # ;Bot: Ch0 on Eyes 6, 7 # ;MMS-4: Top: Ch1 on Eye 1; Ch0 on Eye 8 # ;Bot: Ch0 on Eyes 6, 7, 8; Ch1 on Eye 9 bad_vars = [] bad_vars_both_chans = [] bad_ch0 = {} bad_ch0['mms1'] = { 'top': [2, 5, 6, 7], 'bottom': [2, 3, 4, 5, 6, 7, 8, 9, 11, 12] } bad_ch0['mms2'] = { 'top': [1, 2, 3, 4, 5, 9, 10, 11, 12], 'bottom': [1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12] } bad_ch0['mms3'] = { 'top': [1, 2, 3, 4, 5, 8, 9, 10, 11, 12], 'bottom': [1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12] } bad_ch0['mms4'] = { 'top': [1, 2, 3, 4, 5, 8, 9, 10, 11, 12], 'bottom': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] } bad_ch1 = {} bad_ch1['mms1'] = {'top': [], 'bottom': [11]} bad_ch1['mms2'] = {'top': [8], 'bottom': [12]} bad_ch1['mms3'] = {'top': [1], 'bottom': []} bad_ch1['mms4'] = {'top': [1], 'bottom': [6, 9]} bad_ch0 = bad_ch0['mms' + str(probe)] bad_ch1 = bad_ch1['mms' + str(probe)] #### bottom channel # top electrons for bad_var in bad_ch0['top']: if bad_var in [6, 7, 8]: continue # ion eyes bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_top_count_rate_sensorid_' + str(bad_var) + suffix)) bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_top_intensity_sensorid_' + str(bad_var) + suffix)) bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_top_counts_sensorid_' + str(bad_var) + suffix)) # bottom electrons for bad_var in bad_ch0['bottom']: if bad_var in [6, 7, 8]: continue # ion eyes bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_bottom_count_rate_sensorid_' + str(bad_var) + suffix)) bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_bottom_intensity_sensorid_' + str(bad_var) + suffix)) bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_bottom_counts_sensorid_' + str(bad_var) + suffix)) # top ions for bad_var in bad_ch0['top']: if bad_var not in [6, 7, 8]: continue # ion eyes bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_top_count_rate_sensorid_' + str(bad_var) + suffix)) bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_top_intensity_sensorid_' + str(bad_var) + suffix)) bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_top_counts_sensorid_' + str(bad_var) + suffix)) # bottom ions for bad_var in bad_ch0['bottom']: if bad_var not in [6, 7, 8]: continue # ion eyes bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_bottom_count_rate_sensorid_' + str(bad_var) + suffix)) bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_bottom_intensity_sensorid_' + str(bad_var) + suffix)) bad_vars.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_bottom_counts_sensorid_' + str(bad_var) + suffix)) #### bottom 2 channels # top electrons for bad_var in bad_ch1['top']: if bad_var in [6, 7, 8]: continue # ion eyes bad_vars_both_chans.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_top_count_rate_sensorid_' + str(bad_var) + suffix)) bad_vars_both_chans.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_top_intensity_sensorid_' + str(bad_var) + suffix)) bad_vars_both_chans.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_top_counts_sensorid_' + str(bad_var) + suffix)) # bottom electrons for bad_var in bad_ch1['bottom']: if bad_var in [6, 7, 8]: continue # ion eyes bad_vars_both_chans.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_bottom_count_rate_sensorid_' + str(bad_var) + suffix)) bad_vars_both_chans.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_bottom_intensity_sensorid_' + str(bad_var) + suffix)) bad_vars_both_chans.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_electron_bottom_counts_sensorid_' + str(bad_var) + suffix)) # top ions for bad_var in bad_ch1['top']: if bad_var not in [6, 7, 8]: continue # ion eyes bad_vars_both_chans.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_top_count_rate_sensorid_' + str(bad_var) + suffix)) bad_vars_both_chans.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_top_intensity_sensorid_' + str(bad_var) + suffix)) bad_vars_both_chans.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_top_counts_sensorid_' + str(bad_var) + suffix)) # bottom ions for bad_var in bad_ch1['bottom']: if bad_var not in [6, 7, 8]: continue # ion eyes bad_vars_both_chans.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_bottom_count_rate_sensorid_' + str(bad_var) + suffix)) bad_vars_both_chans.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_bottom_intensity_sensorid_' + str(bad_var) + suffix)) bad_vars_both_chans.append( tnames('mms' + str(probe) + '_epd_feeps_' + data_rate_level + '_ion_bottom_counts_sensorid_' + str(bad_var) + suffix)) # set the first energy channel to NaN for bad_var in bad_vars: if bad_var == []: continue bad_var_data = pytplot.get_data(bad_var[0]) if bad_var_data is not None: times, data = bad_var_data energies = pytplot.data_quants[bad_var[0]].spec_bins.values # check if the energy table contains all names if np.isnan(np.sum(energies)): continue data[:, 0] = np.nan pytplot.store_data(bad_var[0], data={ 'x': times, 'y': data, 'v': energies.reshape(energies.size) }) # set the first and second energy channels to NaN for bad_var in bad_vars_both_chans: if bad_var == []: continue bad_var_data = pytplot.get_data(bad_var[0]) if bad_var_data is not None: times, data = bad_var_data energies = pytplot.data_quants[bad_var[0]].spec_bins.values # check if the energy table contains all names if np.isnan(np.sum(energies)): continue data[:, 0] = np.nan data[:, 1] = np.nan pytplot.store_data(bad_var[0], data={ 'x': times, 'y': data, 'v': energies.reshape(energies.size) })
# times are unix time (seconds since 1 January 1970) print(times[0]) # FGM data include the magnitude fgm_data[0] # you can convert the unix time to a string with time_string from pyspedas import time_string print(time_string(times[0])) # and convert back to unix time using time_double from pyspedas import time_double print(time_double('2015-10-16 13:06:00.00451')) # create new tplot variables with store_data from pytplot import store_data # save the B-field vector store_data('b_vector', data={'x': times, 'y': fgm_data[:, 0:3]}) # save the B-field magnitude store_data('b_mag', data={'x': times, 'y': fgm_data[:, 3]}) # the keywords are very flexible, e.g., from pyspedas.mms import mms_load_hpca, mms_load_eis, mms_load_feeps, mms_load_aspoc # specify multiple probes as integers, and multiple datatypes mms_load_hpca(probe=[1, 2, 4], data_rate='brst',
def mms_load_data(trange=['2015-10-16', '2015-10-17'], probe='1', data_rate='srvy', level='l2', instrument='fgm', datatype='', varformat=None, prefix='', suffix='', get_support_data=False, time_clip=False, no_update=False, center_measurement=False, available=False, notplot=False, latest_version=False, major_version=False, min_version=None, cdf_version=None, spdf=False): """ This function loads MMS data into pyTplot variables This function is not meant to be called directly. Please see the individual load routines for documentation and use. """ if not isinstance(probe, list): probe = [probe] if not isinstance(data_rate, list): data_rate = [data_rate] if not isinstance(level, list): level = [level] if not isinstance(datatype, list): datatype = [datatype] probe = [str(p) for p in probe] # allows the user to pass in trange as list of datetime objects if type(trange[0]) == datetime and type(trange[1]) == datetime: trange = [ time_string(trange[0].timestamp()), time_string(trange[1].timestamp()) ] # allows the user to pass in trange as a list of floats (unix times) if isinstance(trange[0], float): trange[0] = time_string(trange[0]) if isinstance(trange[1], float): trange[1] = time_string(trange[1]) start_date = parse(trange[0]).strftime( '%Y-%m-%d') # need to request full day, then parse out later end_date = parse(time_string(time_double(trange[1]) - 0.1)).strftime( '%Y-%m-%d-%H-%M-%S' ) # -1 second to avoid getting data for the next day download_only = CONFIG['download_only'] no_download = False if no_update or CONFIG['no_download']: no_download = True if spdf: return mms_load_data_spdf(trange=trange, probe=probe, data_rate=data_rate, level=level, instrument=instrument, datatype=datatype, varformat=varformat, suffix=suffix, get_support_data=get_support_data, time_clip=time_clip, no_update=no_update, center_measurement=center_measurement, notplot=notplot, latest_version=latest_version, major_version=major_version, min_version=min_version, cdf_version=cdf_version) user = None if not no_download: sdc_session, user = mms_login_lasp() out_files = [] available_files = [] for prb in probe: for drate in data_rate: for lvl in level: for dtype in datatype: if user is None: url = 'https://lasp.colorado.edu/mms/sdc/public/files/api/v1/file_info/science?start_date=' + start_date + '&end_date=' + end_date + '&sc_id=mms' + prb + '&instrument_id=' + instrument + '&data_rate_mode=' + drate + '&data_level=' + lvl else: url = 'https://lasp.colorado.edu/mms/sdc/sitl/files/api/v1/file_info/science?start_date=' + start_date + '&end_date=' + end_date + '&sc_id=mms' + prb + '&instrument_id=' + instrument + '&data_rate_mode=' + drate + '&data_level=' + lvl if dtype != '': url = url + '&descriptor=' + dtype if CONFIG['debug_mode']: logging.info('Fetching: ' + url) if no_download == False: # query list of available files try: with warnings.catch_warnings(): warnings.simplefilter("ignore", category=ResourceWarning) http_json = sdc_session.get( url, verify=True).json() if CONFIG['debug_mode']: logging.info( 'Filtering the results down to your trange' ) files_in_interval = mms_files_in_interval( http_json['files'], trange) if available: for file in files_in_interval: logging.info( file['file_name'] + ' (' + str( np.round(file['file_size'] / (1024. * 1024), decimals=1)) + ' MB)') available_files.append(file['file_name']) continue for file in files_in_interval: file_date = parse(file['timetag']) if dtype == '': out_dir = os.sep.join([ CONFIG['local_data_dir'], 'mms' + prb, instrument, drate, lvl, file_date.strftime('%Y'), file_date.strftime('%m') ]) else: out_dir = os.sep.join([ CONFIG['local_data_dir'], 'mms' + prb, instrument, drate, lvl, dtype, file_date.strftime('%Y'), file_date.strftime('%m') ]) if drate.lower() == 'brst': out_dir = os.sep.join( [out_dir, file_date.strftime('%d')]) out_file = os.sep.join( [out_dir, file['file_name']]) if CONFIG['debug_mode']: logging.info('File: ' + file['file_name'] + ' / ' + file['timetag']) if os.path.exists(out_file) and str( os.stat(out_file).st_size) == str( file['file_size']): if not download_only: logging.info('Loading ' + out_file) out_files.append(out_file) continue if user is None: download_url = 'https://lasp.colorado.edu/mms/sdc/public/files/api/v1/download/science?file=' + file[ 'file_name'] else: download_url = 'https://lasp.colorado.edu/mms/sdc/sitl/files/api/v1/download/science?file=' + file[ 'file_name'] logging.info('Downloading ' + file['file_name'] + ' to ' + out_dir) with warnings.catch_warnings(): warnings.simplefilter( "ignore", category=ResourceWarning) fsrc = sdc_session.get(download_url, stream=True, verify=True) ftmp = NamedTemporaryFile(delete=False) with open(ftmp.name, 'wb') as f: copyfileobj(fsrc.raw, f) if not os.path.exists(out_dir): os.makedirs(out_dir) # if the download was successful, copy to data directory copy(ftmp.name, out_file) out_files.append(out_file) fsrc.close() ftmp.close() except requests.exceptions.ConnectionError: # No/bad internet connection; try loading the files locally logging.error('No internet connection!') if out_files == []: if not download_only: logging.info('Searching for local files...') out_files = mms_get_local_files( prb, instrument, drate, lvl, dtype, trange) if out_files == [] and CONFIG[ 'mirror_data_dir'] != None: # check for network mirror; note: network mirrors are assumed to be read-only # and we always copy the files from the mirror to the local data directory # before trying to load into tplot variables logging.info( 'No local files found; checking network mirror...' ) out_files = mms_get_local_files(prb, instrument, drate, lvl, dtype, trange, mirror=True) if not no_download: sdc_session.close() if available: return available_files if not download_only: out_files = sorted(out_files) filtered_out_files = mms_file_filter(out_files, latest_version=latest_version, major_version=major_version, min_version=min_version, version=cdf_version) if filtered_out_files == []: logging.info('No matching CDF versions found.') return new_variables = cdf_to_tplot(filtered_out_files, varformat=varformat, merge=True, get_support_data=get_support_data, prefix=prefix, suffix=suffix, center_measurement=center_measurement, notplot=notplot) if notplot: return new_variables if new_variables == []: logging.warning('No data loaded.') return if time_clip: for new_var in new_variables: tclip(new_var, trange[0], trange[1], suffix='') return new_variables else: return out_files
def mms_load_brst_segments(trange=None, suffix=''): ''' This function loads the burst segment intervals Parameters: trange : list of str time range of interest [starttime, endtime] with the format 'YYYY-MM-DD','YYYY-MM-DD'] or to specify more or less than a day ['YYYY-MM-DD/hh:mm:ss','YYYY-MM-DD/hh:mm:ss'] Returns: Tuple containing (start_times, end_times) ''' if trange is None: logging.error('Error; no trange specified.') return None tr = time_double(trange) save_file = os.path.join(CONFIG['local_data_dir'], 'mms_brst_intervals.sav') brst_file = download( remote_file='http://www.spedas.org/mms/mms_brst_intervals.sav', local_file=save_file) if len(brst_file) == 0: logging.error('Error downloading burst intervals sav file') return None try: intervals = readsav(save_file) except FileNotFoundError: logging.error('Error loading burst intervals sav file: ' + save_file) return None unix_start = intervals['brst_intervals'].start_times[0] unix_end = intervals['brst_intervals'].end_times[0] sorted_idxs = np.argsort(unix_start) unix_start = unix_start[sorted_idxs] unix_end = unix_end[sorted_idxs] times_in_range = (unix_start >= tr[0] - 300.0) & (unix_start <= tr[1] + 300.0) unix_start = unix_start[times_in_range] unix_end = unix_end[times_in_range] # +10 second offset added; there appears to be an extra 10 # seconds of data, consistently, not included in the range here unix_end = [end_time + 10.0 for end_time in unix_end] bar_x = [] bar_y = [] for start_time, end_time in zip(unix_start, unix_end): if end_time >= tr[0] and start_time <= tr[1]: bar_x.extend([start_time, start_time, end_time, end_time]) bar_y.extend([np.nan, 0., 0., np.nan]) vars_created = store_data('mms_bss_burst' + suffix, data={ 'x': bar_x, 'y': bar_y }) if vars_created == False: logging.error('Error creating burst segment intervals tplot variable') return None options('mms_bss_burst' + suffix, 'panel_size', 0.09) options('mms_bss_burst' + suffix, 'thick', 2) options('mms_bss_burst' + suffix, 'Color', 'green') options('mms_bss_burst' + suffix, 'border', False) options('mms_bss_burst' + suffix, 'yrange', [-0.001, 0.001]) options('mms_bss_burst' + suffix, 'legend_names', ['Burst']) options('mms_bss_burst' + suffix, 'ytitle', '') return (unix_start, unix_end)
def mms_feeps_active_eyes(trange, probe, data_rate, species, level): """ This function returns the FEEPS active eyes, based on date/probe/species/rate Parameters: trange: list of str time range probe: str probe #, e.g., '4' for MMS4 data_rate: str instrument data rate, e.g., 'srvy' or 'brst' species: str 'electron' or 'ion' level: str data level Returns: Hash table containing 2 keys: output['top'] -> maps to the active top eyes output['bottom'] -> maps to the active bottom eyes Notes: 1) Burst mode should include all sensors (TOP and BOTTOM): electrons: [1, 2, 3, 4, 5, 9, 10, 11, 12] ions: [6, 7, 8] 2) SITL should return (TOP only): electrons: set_intersection([5, 11, 12], active_eyes) ions: None 3) From Drew Turner, 9/7/2017, srvy mode: - before 16 August 2017: electrons: [3, 4, 5, 11, 12] iond: [6, 7, 8] - after 16 August 2017: MMS1 Top Eyes: 3, 5, 6, 7, 8, 9, 10, 12 Bot Eyes: 2, 4, 5, 6, 7, 8, 9, 10 MMS2 Top Eyes: 1, 2, 3, 5, 6, 8, 10, 11 Bot Eyes: 1, 4, 5, 6, 7, 8, 9, 11 MMS3 Top Eyes: 3, 5, 6, 7, 8, 9, 10, 12 Bot Eyes: 1, 2, 3, 6, 7, 8, 9, 10 MMS4 Top Eyes: 3, 4, 5, 6, 8, 9, 10, 11 Bot Eyes: 3, 5, 6, 7, 8, 9, 10, 12 """ if data_rate.lower() == 'brst' and species.lower() == 'electron': return { 'top': [1, 2, 3, 4, 5, 9, 10, 11, 12], 'bottom': [1, 2, 3, 4, 5, 9, 10, 11, 12] } if data_rate.lower() == 'brst' and species.lower() == 'ion': return {'top': [6, 7, 8], 'bottom': [6, 7, 8]} # old eyes, srvy mode, prior to 16 August 2017 if species.lower() == 'electron': sensors = {'top': [3, 4, 5, 11, 12], 'bottom': [3, 4, 5, 11, 12]} else: sensors = {'top': [6, 7, 8], 'bottom': [6, 7, 8]} if isinstance(trange[0], str): start_time = time_double(trange[0]) else: start_time = trange[0] # srvy mode, after 16 August 2017 if start_time >= time_double('2017-08-16') and data_rate.lower() == 'srvy': active_table = { '1-electron': { 'top': [3, 5, 9, 10, 12], 'bottom': [2, 4, 5, 9, 10] }, '1-ion': { 'top': [6, 7, 8], 'bottom': [6, 7, 8] }, '2-electron': { 'top': [1, 2, 3, 5, 10, 11], 'bottom': [1, 4, 5, 9, 11] }, '2-ion': { 'top': [6, 8], 'bottom': [6, 7, 8] }, '3-electron': { 'top': [3, 5, 9, 10, 12], 'bottom': [1, 2, 3, 9, 10] }, '3-ion': { 'top': [6, 7, 8], 'bottom': [6, 7, 8] }, '4-electron': { 'top': [3, 4, 5, 9, 10, 11], 'bottom': [3, 5, 9, 10, 12] }, '4-ion': { 'top': [6, 8], 'bottom': [6, 7, 8] } } sensors = active_table[probe.lower() + '-' + species.lower()] if level.lower() == 'sitl': return { 'top': list(set(sensors['top']) & set([5, 11, 12])), 'bottom': [] } if level.lower() == 'sitl': return {'top': [5, 11, 12], 'bottom': []} return sensors
def mms_load_data(trange=['2015-10-16', '2015-10-17'], probe='1', data_rate='srvy', level='l2', instrument='fgm', datatype='', prefix='', suffix='', get_support_data=False, time_clip=False): """ This function loads MMS data into pyTplot variables """ if not isinstance(probe, list): probe = [probe] if not isinstance(data_rate, list): data_rate = [data_rate] if not isinstance(level, list): level = [level] if not isinstance(datatype, list): datatype = [datatype] probe = [str(p) for p in probe] # allows the user to pass in trange as list of datetime objects if type(trange[0]) == datetime and type(trange[1]) == datetime: trange = [ time_string(trange[0].timestamp()), time_string(trange[1].timestamp()) ] start_date = parse(trange[0]).strftime( '%Y-%m-%d') # need to request full day, then parse out later end_date = parse(time_string(time_double(trange[1]) - 0.1)).strftime( '%Y-%m-%d-%H-%M-%S' ) # -1 second to avoid getting data for the next day download_only = CONFIG['download_only'] if not CONFIG['no_download']: sdc_session, user = mms_login_lasp() out_files = [] for prb in probe: for drate in data_rate: for lvl in level: for dtype in datatype: if user is None: url = 'https://lasp.colorado.edu/mms/sdc/public/files/api/v1/file_info/science?start_date=' + start_date + '&end_date=' + end_date + '&sc_id=mms' + prb + '&instrument_id=' + instrument + '&data_rate_mode=' + drate + '&data_level=' + lvl else: url = 'https://lasp.colorado.edu/mms/sdc/sitl/files/api/v1/file_info/science?start_date=' + start_date + '&end_date=' + end_date + '&sc_id=mms' + prb + '&instrument_id=' + instrument + '&data_rate_mode=' + drate + '&data_level=' + lvl if dtype != '': url = url + '&descriptor=' + dtype if CONFIG['debug_mode']: logging.info('Fetching: ' + url) if CONFIG['no_download'] == False: # query list of available files try: http_json = sdc_session.get(url, verify=True).json() if CONFIG['debug_mode']: logging.info( 'Filtering the results down to your trange' ) files_in_interval = mms_files_in_interval( http_json['files'], trange) for file in files_in_interval: file_date = parse(file['timetag']) if dtype == '': out_dir = os.sep.join([ CONFIG['local_data_dir'], 'mms' + prb, instrument, drate, lvl, file_date.strftime('%Y'), file_date.strftime('%m') ]) else: out_dir = os.sep.join([ CONFIG['local_data_dir'], 'mms' + prb, instrument, drate, lvl, dtype, file_date.strftime('%Y'), file_date.strftime('%m') ]) if drate.lower() == 'brst': out_dir = os.sep.join( [out_dir, file_date.strftime('%d')]) out_file = os.sep.join( [out_dir, file['file_name']]) if CONFIG['debug_mode']: logging.info('File: ' + file['file_name'] + ' / ' + file['timetag']) #if os.path.exists(out_file) and str(os.stat(out_file).st_size) == str(file['file_size']): if os.path.exists(out_file): if not download_only: logging.info('Loading ' + out_file) out_files.append(out_file) continue if user is None: download_url = 'https://lasp.colorado.edu/mms/sdc/public/files/api/v1/download/science?file=' + file[ 'file_name'] else: download_url = 'https://lasp.colorado.edu/mms/sdc/sitl/files/api/v1/download/science?file=' + file[ 'file_name'] logging.info('Downloading ' + file['file_name'] + ' to ' + out_dir) fsrc = sdc_session.get(download_url, stream=True, verify=True) ftmp = NamedTemporaryFile(delete=False) copyfileobj(fsrc.raw, ftmp) if not os.path.exists(out_dir): os.makedirs(out_dir) # if the download was successful, copy to data directory copy(ftmp.name, out_file) out_files.append(out_file) ftmp.close() fsrc.close() except requests.exceptions.ConnectionError: # No/bad internet connection; try loading the files locally logging.error('No internet connection!') if out_files == []: if not download_only: logging.info('Searching for local files...') out_files = mms_get_local_files( prb, instrument, drate, lvl, dtype, trange) if not CONFIG['no_download']: sdc_session.close() if not download_only: out_files = sorted(out_files) new_variables = cdf_to_tplot(out_files, merge=True, get_support_data=get_support_data, prefix=prefix, suffix=suffix) if new_variables == []: logging.warning('No data loaded.') return logging.info('Loaded variables:') for new_var in new_variables: print(new_var) if time_clip: tclip(new_var, trange[0], trange[1], suffix='') return new_variables
def get_w(trange=None, create_tvar=False, newname=None): """ This routine downloads the 6 Tsygeneko (TS05) model driving variables W1, W2, W3, W4, W5, W6; from: http://geo.phys.spbu.ru/~tsyganenko/TS05_data_and_stuff """ if trange is None: print('trange keyword must be specified.') return years = dailynames(trange=trange, file_format='%Y') tmpdir = mkdtemp() if newname is None: newname = 'Tsy_W_vars_' + '-'.join(years) ut_out = np.empty(0) w1_out = np.empty(0) w2_out = np.empty(0) w3_out = np.empty(0) w4_out = np.empty(0) w5_out = np.empty(0) w6_out = np.empty(0) for year in years: file = download( remote_path= 'http://geo.phys.spbu.ru/~tsyganenko/TS05_data_and_stuff/', remote_file=year + '_OMNI_5m_with_TS05_variables.???', local_path=tmpdir) if file[0][-3:] == 'zip': with zipfile.ZipFile(file[0], 'r') as zip_ref: zip_ref.extractall(tmpdir) rows = pd.read_csv(tmpdir + '/' + year + '_OMNI_5m_with_TS05_variables.dat', delim_whitespace=True, header=None) # extract the W variables w1 = rows.to_numpy()[:, -6] w2 = rows.to_numpy()[:, -5] w3 = rows.to_numpy()[:, -4] w4 = rows.to_numpy()[:, -3] w5 = rows.to_numpy()[:, -2] w6 = rows.to_numpy()[:, -1] # extract the times years = rows.to_numpy()[:, 0] doys = rows.to_numpy()[:, 1] hours = rows.to_numpy()[:, 2] minutes = rows.to_numpy()[:, 3] time_strings = [ str(int(year)) + '-' + str(int(doy)).zfill(3) + ' ' + str(int(hour)).zfill(2) + ':' + str(int(minute)).zfill(2) for year, doy, hour, minute in zip(years, doys, hours, minutes) ] unix_times = np.array(time_double(time_strings)) ut_out = np.append(ut_out, unix_times) w1_out = np.append(w1_out, w1) w2_out = np.append(w2_out, w2) w3_out = np.append(w3_out, w3) w4_out = np.append(w4_out, w4) w5_out = np.append(w5_out, w5) w6_out = np.append(w6_out, w6) in_range = np.argwhere((ut_out >= time_double(trange[0])) & (ut_out < time_double(trange[1]))).squeeze() if len(in_range) == 0: print('No data found in the trange.') return if create_tvar: out = np.array((w1_out[in_range], w2_out[in_range], w3_out[in_range], w4_out[in_range], w5_out[in_range], w6_out[in_range])) store_data(newname, data={'x': ut_out[in_range], 'y': out.T}) return newname return { 'times': ut_out[in_range], 'w1': w1_out[in_range], 'w2': w2_out[in_range], 'w3': w3_out[in_range], 'w4': w4_out[in_range], 'w5': w5_out[in_range], 'w6': w6_out[in_range] }
def mms_fpi_make_compressionlossbars(tname, lossy=False): """ Creates FPI's compressionloss flag bars Parameters ----------- tname: str Tplot variable name of DIS or DES compressionloss data lossy: bool The value for lossy compression (use this keyword only for special case) Flag ----------- 0: Lossless compression 1: Lossy compression In old files (v2.1.0 or older until the end of Phase 1A) 1: Lossless compression 3: Lossy compression Notes ----------- In cases when data had been lossy compressed, some artifacts may appear in the data due to the compression. Since all of fast survey data are lossy compressed, it is not necessary to make this bar for fast survey data. Returns ------------ List of the tplot variables created. """ if fnmatch(tname, 'mms?_dis*'): instrument = 'DIS' elif fnmatch(tname, 'mms?_des*'): instrument = 'DES' else: print('Unable to determine instrument from variable name.') return if instrument == 'DES': colors = 'red' else: colors = 'blue' if fnmatch(tname, '*_fast*'): print('All fast survey data are lossy compressed, so there is no need to create this bar.') return elif fnmatch(tname, '*_brst*'): data_rate = 'Brst' else: print('Unable to determine data rate from variable name.') return data = get_data(tname) metadata = get_data(tname, metadata=True) if data is None: print('Problem reading the variable: ' + tname) return flagline = np.zeros(len(data.times)) if not lossy: file_id = metadata['CDF']['GATT']['Logical_file_id'] version = file_id.split('_v')[1].split('.') if version[0] == '2': if version[1] == '1': if data.times[0] < time_double('2016-04-01'): lossy = 3 else: lossy = 1 else: if float(version[1]) > 1: lossy = 1 else: lossy = 3 else: if float(version[0]) > 2: lossy = 1 else: lossy = 3 for j in range(len(data.times)): if data.y[j] != lossy: flagline[j] = np.nan else: flagline[j] = 0.5 store_data(tname + '_flagbars', data={'x': data.times, 'y': flagline}) options(tname + '_flagbars', 'yrange', [0, 1]) options(tname + '_flagbars', 'panel_size', 0.2) options(tname + '_flagbars', 'symbols', True) options(tname + '_flagbars', 'markers', 's') options(tname + '_flagbars', 'thick', 4) options(tname + '_flagbars', 'border', False) return [tname + '_flagbars']
def mms_part_getspec(instrument='fpi', probe='1', species='e', data_rate='fast', trange=None, output=['energy', 'theta', 'phi', 'pa', 'gyro'], units='eflux', energy=None, phi=None, theta=None, pitch=None, gyro=None, mag_data_rate=None, scpot_data_rate=None, fac_type='mphigeo', center_measurement=False, spdf=False, correct_photoelectrons=False, internal_photoelectron_corrections=False, disable_photoelectron_corrections=False, zero_negative_values=False, regrid=[32, 16], no_regrid=False): """ Generate spectra and moments from 3D MMS particle data Parameters ---------- trange: list of str Time range units: str Specify units of output variables; must be 'eflux' to calculate moments valid options: 'flux' - # / (cm^2 * s * sr * eV) 'eflux' - eV / (cm^2 * s * sr * eV) <default> 'df_cm' - s^3 / cm^6 'df_km' - s^3 / km^6 species: str Specify the species of the input tplot variable data_rate: str Data rate of the input data instrument: str Instrument (FPI or HPCA) probe: int or str Spacecraft probe # output: str or list of str Output variables; options: 'energy': energy spectrograms 'theta': theta spectrograms 'phi': phi spectrograms 'pa': pitch-angle spectrograms 'gyro': gyro-phase spectrograms 'moments': plasma moments energy: list of float Energy range [min, max], in eV phi: list of float Phi range [min, max], in degrees theta: list of float Theta range [min, max], in degrees pitch: list of float Pitch-angle range [min, max], in degrees gyro: list of float Gyro-angle range [min, max], in degrees mag_name: str Tplot variable containing magnetic field data for moments and FAC transformations pos_name: str Tplot variable containing spacecraft position for FAC transformations sc_pot_name: str Tplot variable containing spacecraft potential data for moments corrections fac_type: str Field aligned coordinate system variant; default: 'mphigeo' options: 'phigeo', 'mphigeo', 'xgse' correct_photoelectrons: bool Flag to correct FPI data for photoelectrons (defaults to True for FPI electron data - disable with the parameter below) disable_photoelectron_corrections: bool Flag to disable FPI photoelectron corrections internal_photoelectron_corrections: bool Apply internal photoelectron corrections zero_negative_values: bool Turn negative values to 0 after doing the photoelectron corrections (DES) Returns ---------- Creates tplot variables containing spectrograms and moments """ start_time = time() if trange is None: # test data for development trange = ['2015-10-16/13:06', '2015-10-16/13:07'] # data_rate = 'brst' if mag_data_rate is None: if data_rate == 'brst': mag_data_rate = 'brst' else: mag_data_rate = 'srvy' if scpot_data_rate is None: if data_rate == 'brst': scpot_data_rate = 'brst' else: scpot_data_rate = 'fast' instrument = instrument.lower() # HPCA is required to be at the center of the accumulation interval # due to assumptions made in mms_get_hpca_dist if instrument == 'hpca' and center_measurement == False: center_measurement = True if instrument == 'fpi': data_vars = pyspedas.mms.fpi(datatype='d' + species + 's-dist', probe=probe, data_rate=data_rate, trange=trange, time_clip=True, center_measurement=center_measurement, spdf=spdf) elif instrument == 'hpca': # for HPCA, 'fast' should be 'srvy' if data_rate == 'fast': data_rate = 'srvy' # 'i' and 'e' are only valid for FPI if species in ['i', 'e']: species = 'hplus' data_vars = pyspedas.mms.hpca(datatype='ion', probe=probe, data_rate=data_rate, trange=trange, time_clip=True, center_measurement=center_measurement, get_support_data=True, spdf=spdf) else: logging.error('Error, unknown instrument: ' + instrument + '; valid options: fpi, hpca') return if data_vars is None or len(data_vars) == 0: logging.error('Error, no data loaded.') return None if not isinstance(probe, list): probe = [probe] if instrument == 'fpi' and species == 'e' and 'moments' in output and not disable_photoelectron_corrections: correct_photoelectrons = True support_trange = [ time_double(trange[0]) - 60.0, time_double(trange[1]) + 60.0 ] # load state data (needed for coordinate transformations and field-aligned coordinates) pos_vars = pyspedas.mms.mec(probe=probe, trange=support_trange, time_clip=True, spdf=spdf) if len(pos_vars) == 0: logging.error('Error, no state data loaded.') return mag_vars = pyspedas.mms.fgm(probe=probe, trange=support_trange, data_rate=mag_data_rate, time_clip=True, spdf=spdf) if len(mag_vars) == 0: logging.error('Error, no magnetic field data loaded.') return scpot_vars = pyspedas.mms.edp(probe=probe, trange=support_trange, level='l2', spdf=spdf, data_rate=scpot_data_rate, datatype='scpot', varformat='*_edp_scpot_*') out_vars = [] for prb in probe: prb_str = str(prb) mag_name = 'mms' + prb_str + '_fgm_b_gse_' + mag_data_rate + '_l2_bvec' pos_name = 'mms' + prb_str + '_mec_r_gse' if instrument == 'fpi': tname = 'mms' + prb_str + '_d' + species + 's_dist_' + data_rate elif instrument == 'hpca': tname = 'mms' + prb_str + '_hpca_' + species + '_phase_space_density' scpot_variable = 'mms' + prb_str + '_edp_scpot_' + scpot_data_rate + '_l2' new_vars = mms_part_products( tname, species=species, instrument=instrument, probe=prb, data_rate=data_rate, output=output, units=units, energy=energy, phi=phi, theta=theta, pitch=pitch, gyro=gyro, mag_name=mag_name, pos_name=pos_name, fac_type=fac_type, sc_pot_name=scpot_variable, correct_photoelectrons=correct_photoelectrons, zero_negative_values=zero_negative_values, internal_photoelectron_corrections= internal_photoelectron_corrections, disable_photoelectron_corrections=disable_photoelectron_corrections, regrid=regrid, no_regrid=no_regrid) if new_vars is None: continue out_vars = out_vars + new_vars logging.info('Finished; time to run: ' + str(round(time() - start_time, 1)) + ' seconds.') return out_vars
def mms_load_sroi_segments(trange=None, probe=1, suffix=''): """ This function loads the Science Region of Interest (SRoI) segment intervals Parameters: trange: list of str time range of interest [starttime, endtime] with the format 'YYYY-MM-DD','YYYY-MM-DD'] or to specify more or less than a day ['YYYY-MM-DD/hh:mm:ss','YYYY-MM-DD/hh:mm:ss'] probe: str or int Spacecraft probe # (default: 1) suffix: str Suffix to append to the end of the tplot variables Returns: Tuple containing (start_times, end_times) """ if not isinstance(probe, str): probe = str(probe) if trange is None: logging.error('Error; no trange specified.') return None tr = time_double(trange) unix_start, unix_end = get_mms_srois( start_time=time_string(tr[0] - 2 * 86400.0, fmt='%Y-%m-%d %H:%M:%S'), end_time=time_string(tr[1] + 2 * 86400.0, fmt='%Y-%m-%d %H:%M:%S'), sc_id='mms' + str(probe)) if len(unix_start) == 0: return ([], []) times_in_range = (unix_start >= tr[0] - 2 * 86400.0) & ( unix_start <= tr[1] + 2 * 86400.0) unix_start = unix_start[times_in_range] unix_end = unix_end[times_in_range] start_out = [] end_out = [] bar_x = [] bar_y = [] for start_time, end_time in zip(unix_start, unix_end): if end_time >= tr[0] and start_time <= tr[1]: bar_x.extend([start_time, start_time, end_time, end_time]) bar_y.extend([np.nan, 0., 0., np.nan]) start_out.append(start_time) end_out.append(end_time) vars_created = store_data('mms' + probe + '_bss_sroi' + suffix, data={ 'x': bar_x, 'y': bar_y }) if not vars_created: logging.error('Error creating SRoI segment intervals tplot variable') return None options('mms' + probe + '_bss_sroi' + suffix, 'panel_size', 0.09) options('mms' + probe + '_bss_sroi' + suffix, 'thick', 2) options('mms' + probe + '_bss_sroi' + suffix, 'Color', 'green') options('mms' + probe + '_bss_sroi' + suffix, 'border', False) options('mms' + probe + '_bss_sroi' + suffix, 'yrange', [-0.001, 0.001]) options('mms' + probe + '_bss_sroi' + suffix, 'legend_names', ['Fast']) options('mms' + probe + '_bss_sroi' + suffix, 'ytitle', '') return (start_out, end_out)
def dailynames(directory='', trange=None, res=24 * 3600., hour_res=False, file_format='%Y%m%d', prefix='', suffix=''): ''' Creates a list of file names using a time range, resoution and file format Based on Davin Larson's file_dailynames in IDL SPEDAS Parameters: directory: str String containing the directory for the generated file names trange: list of str, list of datetime or list of floats Two-element list containing the start and end times for the file names res: float File name resolution in seconds (default: 24*3600., i.e., daily) file_format: str Format of the file names using strftime directives; for reference: https://strftime.org (default: %Y%m%d, i.e., daily) prefix: str file name prefix suffix: str file name suffix Returns: List containing filenames ''' if trange is None: print('No trange specified') return if hour_res == True: res = 3600. file_format = '%Y%m%d%H' # allows the user to pass in trange as list of datetime objects if type(trange[0]) == datetime and type(trange[1]) == datetime: trange = [ time_string(trange[0].timestamp()), time_string(trange[1].timestamp()) ] tr = [trange[0], trange[1]] if isinstance(trange[0], str): tr[0] = time_double(trange[0]) if isinstance(trange[1], str): tr[1] = time_double(trange[1]) # Davin's magic heisted from file_dailynames in IDL mmtr = [np.floor(tr[0] / res), np.ceil(tr[1] / res)] if mmtr[1] - mmtr[0] < 1: n = 1 else: n = int(mmtr[1] - mmtr[0]) times = [(float(num) + mmtr[0]) * res for num in range(n)] dates = [] files = [] for time in times: if time_string(time, fmt=file_format) not in dates: dates.append(time_string(time, fmt=file_format)) for date in dates: files.append(directory + prefix + date + suffix) return files
def mms_load_data_spdf(trange=['2015-10-16', '2015-10-17'], probe='1', data_rate='srvy', level='l2', instrument='fgm', datatype='', varformat=None, suffix='', get_support_data=False, time_clip=False, no_update=False, center_measurement=False, available=False, notplot=False, latest_version=False, major_version=False, min_version=None, cdf_version=None, varnames=[]): """ This function loads MMS data from NASA SPDF into pyTplot variables This function is not meant to be called directly. Please see the individual load routines for documentation and use. """ tvars_created = [] if not isinstance(probe, list): probe = [probe] if not isinstance(data_rate, list): data_rate = [data_rate] if not isinstance(level, list): level = [level] if not isinstance(datatype, list): datatype = [datatype] for prb in probe: for lvl in level: for drate in data_rate: if drate == 'brst': time_format = '%Y%m%d%H%M??' file_res = 60. else: time_format = '%Y%m%d' file_res = 24 * 3600. for dtype in datatype: remote_path = 'mms' + prb + '/' + instrument + '/' + drate + '/' + lvl + '/' if instrument == 'fgm': pathformat = remote_path + '%Y/%m/mms' + prb + '_fgm_' + drate + '_' + lvl + '_' + time_format + '_v*.cdf' elif instrument == 'aspoc': pathformat = remote_path + '%Y/%m/mms' + prb + '_aspoc_' + drate + '_' + lvl + '_' + time_format + '_v*.cdf' elif instrument == 'edi': pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_edi_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf' elif instrument == 'fpi': if drate != 'brst': time_format = '%Y%m%d??0000' pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_fpi_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf' elif instrument == 'epd-eis': pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_epd-eis_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf' elif instrument == 'feeps': if drate != 'brst': time_format = '%Y%m%d000000' pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_feeps_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf' elif instrument == 'hpca': time_format = '%Y%m%d??????' pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_hpca_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf' elif instrument == 'mec': pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_mec_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf' elif instrument == 'scm': pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_scm_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf' elif instrument == 'dsp': pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_dsp_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf' elif instrument == 'edp': pathformat = remote_path + dtype + '/%Y/%m/mms' + prb + '_edp_' + drate + '_' + lvl + '_' + dtype + '_' + time_format + '_v*.cdf' if drate == 'brst': if isinstance(trange[0], float): trange = [trange[0] - 300., trange[1]] else: trange = [time_double(trange[0]) - 300., trange[1]] # find the full remote path names using the trange remote_names = dailynames(file_format=pathformat, trange=trange, res=file_res) out_files = [] files = download(remote_file=remote_names, remote_path=CONFIG['remote_data_dir'], local_path=CONFIG['local_data_dir']) if files is not None: for file in files: out_files.append(file) out_files = sorted(out_files) filtered_out_files = mms_file_filter( out_files, latest_version=latest_version, major_version=major_version, min_version=min_version, version=cdf_version) tvars = cdf_to_tplot(filtered_out_files, varformat=varformat, varnames=varnames, get_support_data=get_support_data, suffix=suffix, center_measurement=center_measurement, notplot=notplot) if tvars is not None: tvars_created.extend(tvars) if time_clip: for new_var in tvars_created: tclip(new_var, trange[0], trange[1], suffix='') return tvars_created
def mms_get_local_state_files(probe='1', level='def', filetype='eph', trange=None): """ Search for local state MMS files in case a list cannot be retrieved from the remote server. Returns a sorted list of file paths. Parameters: trange : list of str time range of interest [starttime, endtime] with the format 'YYYY-MM-DD','YYYY-MM-DD'] or to specify more or less than a day ['YYYY-MM-DD/hh:mm:ss','YYYY-MM-DD/hh:mm:ss'] probe: str probe #, e.g., '4' for MMS4 level: str state data level; either 'def' (for definitive) or 'pred' (for predicted) filetype: str state file type, e.g. 'eph' (for ephemeris) or 'att' (for attitude) Returns: List of local files found matching the input parameters. """ if trange is None: logging.info('No trange specified in mms_get_local_state_files') return # directory and file name search patterns # For now # -all ancillary data is in one directory: # mms\ancillary # -assume file names are of the form: # SPACECRAFT_FILETYPE_startDate_endDate.version # where SPACECRAFT is [MMS1, MMS2, MMS3, MMS4] in uppercase # and FILETYPE is either DEFATT, PREDATT, DEFEPH, PREDEPH in uppercase # and start/endDate is YYYYDOY # and version is Vnn (.V00, .V01, etc..) dir_pattern = os.sep.join([ CONFIG['local_data_dir'], 'ancillary', 'mms' + probe, level + filetype ]) file_pattern = 'MMS' + probe + '_' + level.upper() + filetype.upper( ) + '_' + '???????_???????.V??' files_in_trange = [] out_files = [] files = glob.glob(os.sep.join([dir_pattern, file_pattern])) # find the files within the trange file_regex = re.compile( os.sep.join([ dir_pattern, 'MMS' + probe + '_' + level.upper() + filetype.upper() + '_([0-9]{7})_([0-9]{7}).V[0-9]{2}' ])) for file in files: time_match = file_regex.match(file) if time_match != None: start_time = pd.to_datetime(time_match.group(1), format='%Y%j').timestamp() end_time = pd.to_datetime(time_match.group(2), format='%Y%j').timestamp() if start_time < time_double(trange[1]) and end_time >= time_double( trange[0]): files_in_trange.append(file) # ensure only the latest version of each file is loaded for file in files_in_trange: this_file = file[0:-3] + 'V??' versions = fnmatch.filter(files_in_trange, this_file) if len(versions) > 1: out_files.append( sorted(versions)[-1]) # only grab the latest version else: out_files.append(versions[0]) return list(set(out_files))
def mms_get_state_data(probe='1', trange=['2015-10-16', '2015-10-17'], datatypes=['pos', 'vel'], level='def', no_download=False, pred_or_def=True, suffix='', always_prompt=False): """ Helper routine for loading state data (ASCII files from the SDC); not meant to be called directly; see pyspedas.mms.state instead """ if not isinstance(probe, list): probe = [probe] local_data_dir = CONFIG['local_data_dir'] download_only = CONFIG['download_only'] start_time = time_double(trange[0]) - 60 * 60 * 24. end_time = time_double(trange[1]) # check if end date is anything other than 00:00:00, if so # add a day to the end time to ensure that all data is downloaded if type(trange[1]) == str: endtime_day = time_double( time_string(time_double(trange[1]), fmt='%Y-%m-%d')) else: endtime_day = time_double(time_string(trange[1], fmt='%Y-%m-%d')) if end_time > endtime_day: add_day = 60 * 60 * 24. else: add_day = 0 start_time_str = time_string(start_time, fmt='%Y-%m-%d') end_time_str = time_string(end_time + add_day, fmt='%Y-%m-%d') filetypes = [] if 'pos' in datatypes or 'vel' in datatypes: filetypes.append('eph') if 'spinras' in datatypes or 'spindec' in datatypes: filetypes.append('att') user = None if not no_download: sdc_session, user = mms_login_lasp(always_prompt=always_prompt) for probe_id in probe: # probe will need to be a string from now on probe_id = str(probe_id) for filetype in filetypes: file_dir = local_data_dir + 'ancillary/' + 'mms' + probe_id + '/' + level + filetype + '/' product = level + filetype # predicted doesn't support start_date/end_date if level == 'def': dates_for_query = '&start_date=' + start_time_str + '&end_date=' + end_time_str else: dates_for_query = '' if user == None: url = 'https://lasp.colorado.edu/mms/sdc/public/files/api/v1/file_info/ancillary?sc_id=mms' + probe_id + '&product=' + product + dates_for_query else: url = 'https://lasp.colorado.edu/mms/sdc/sitl/files/api/v1/file_info/ancillary?sc_id=mms' + probe_id + '&product=' + product + dates_for_query with warnings.catch_warnings(): warnings.simplefilter("ignore", category=ResourceWarning) http_request = sdc_session.get(url, verify=True) http_json = http_request.json() out_dir = os.sep.join([ local_data_dir, 'ancillary', 'mms' + probe_id, level + filetype ]) files_in_interval = [] out_files = [] # since predicted doesn't support start_date/end_date, we'll need to parse the correct dates if level != 'def': for file in http_json['files']: # first, remove the dates that start after the end of the trange if time_double(file['start_date']) > endtime_day: continue # now remove files that end before the start of the trange if start_time > time_double(file['end_date']): continue files_in_interval.append(file) break else: files_in_interval = http_json['files'] for file in files_in_interval: out_file = os.sep.join([out_dir, file['file_name']]) if os.path.exists(out_file) and str( os.stat(out_file).st_size) == str(file['file_size']): if not download_only: logging.info('Loading ' + out_file) out_files.append(out_file) http_request.close() continue if user == None: download_url = 'https://lasp.colorado.edu/mms/sdc/public/files/api/v1/download/ancillary?file=' + file[ 'file_name'] else: download_url = 'https://lasp.colorado.edu/mms/sdc/sitl/files/api/v1/download/ancillary?file=' + file[ 'file_name'] logging.info('Downloading ' + file['file_name'] + ' to ' + out_dir) with warnings.catch_warnings(): warnings.simplefilter("ignore", category=ResourceWarning) fsrc = sdc_session.get(download_url, stream=True, verify=True) ftmp = NamedTemporaryFile(delete=False) with open(ftmp.name, 'wb') as f: copyfileobj(fsrc.raw, f) if not os.path.exists(out_dir): os.makedirs(out_dir) # if the download was successful, copy to data directory copy(ftmp.name, out_file) out_files.append(out_file) fsrc.close() ftmp.close() if download_only: continue if filetype == 'eph': mms_load_eph_tplot(sorted(out_files), level=level, probe=probe_id, datatypes=datatypes, suffix=suffix, trange=trange) elif filetype == 'att': mms_load_att_tplot(sorted(out_files), level=level, probe=probe_id, datatypes=datatypes, suffix=suffix, trange=trange) if not no_download: sdc_session.close()
def mms_feeps_getgyrophase(trange=['2017-07-11/22:30', '2017-07-11/22:35'], probe='2', data_rate='brst', level='l2', datatype='electron'): """ Calculates FEEPS gyrophase angles for electron burst data Notes: This is intended for use with burst mode data, but the output gyrophase product, mms*_feeps_gyrophase, can be interpolated onto FEEPS burst or survey cadence to produce spectra Based on the IDL code writen by Drew Turner (10 Oct 2017): mms_feeps_getgyrophase.pro """ mec_vars = mms.mec(trange=trange, probe=probe, data_rate=data_rate) if mec_vars is None: print( 'Problem loading MEC data for calculating FEEPS gyrophase angles') qeci2sm = get_data('mms' + probe + '_mec_quat_eci_to_sm') qeci2bcs = get_data('mms' + probe + '_mec_quat_eci_to_bcs') rsun = get_data('mms' + probe + '_mec_r_sun_de421_eci') rsunbcs = np.zeros((len(rsun.times), 3)) rduskbcs = np.zeros((len(rsun.times), 3)) rdusksm = [0, 1, 0] for i in range(len(rsun.times)): q = qeci2bcs.y[i, :] # Quaternion rotation matrix: s = 1 # these quaternions are unit-qs R = np.array([ [ 1 - 2 * s * (q[2]**2 + q[3]**2), 2 * s * (q[1] * q[2] - q[3] * q[0]), 2 * s * (q[1] * q[3] + q[2] * q[0]) ], # ECI to BCS [ 2 * s * (q[1] * q[2] + q[3] * q[0]), 1 - 2 * s * (q[1]**2 + q[3]**2), 2 * s * (q[2] * q[3] - q[1] * q[0]) ], [ 2 * s * (q[1] * q[3] - q[2] * q[0]), 2 * s * (q[2] * q[3] + q[1] * q[0]), 1 - 2 * s * (q[1]**2 + q[2]**2) ] ]) R = R.T rsunbcs[i, :] = np.array([ R[0, 0] * rsun.y[i, 0] + R[1, 0] * rsun.y[i, 1] + R[2, 0] * rsun.y[i, 2], R[0, 1] * rsun.y[i, 0] + R[1, 1] * rsun.y[i, 1] + R[2, 1] * rsun.y[i, 2], R[0, 2] * rsun.y[i, 0] + R[1, 2] * rsun.y[i, 1] + R[2, 2] * rsun.y[i, 2] ]) # now make second vector for gyroplane reference, dusk direction (+Y in SM) q = qeci2sm.y[i, :] # Quaternion rotation matrix: s = 1 # these quaternions are unit-qs R2 = np.array([ [ 1 - 2 * s * (q[2]**2 + q[3]**2), 2 * s * (q[1] * q[2] - q[3] * q[0]), 2 * s * (q[1] * q[3] + q[2] * q[0]) ], # ECI to SM [ 2 * s * (q[1] * q[2] + q[3] * q[0]), 1 - 2 * s * (q[1]**2 + q[3]**2), 2 * s * (q[2] * q[3] - q[1] * q[0]) ], [ 2 * s * (q[1] * q[3] - q[2] * q[0]), 2 * s * (q[2] * q[3] + q[1] * q[0]), 1 - 2 * s * (q[1]**2 + q[2]**2) ] ]) # going from SM to ECI, so invert R: R2 = np.linalg.inv(R2) # SM to ECI R2 = R2.T rduskeci = [ R2[0, 0] * rdusksm[0] + R2[1, 0] * rdusksm[1] + R2[2, 0] * rdusksm[2], R2[0, 1] * rdusksm[0] + R2[1, 1] * rdusksm[1] + R2[2, 1] * rdusksm[2], R2[0, 2] * rdusksm[0] + R2[1, 2] * rdusksm[1] + R2[2, 2] * rdusksm[2] ] # Now convert to BCS: rduskbcs[i, :] = np.array([ R[0, 0] * rduskeci[0] + R[1, 0] * rduskeci[1] + R[2, 0] * rduskeci[2], R[0, 1] * rduskeci[0] + R[1, 1] * rduskeci[1] + R[2, 1] * rduskeci[2], R[0, 2] * rduskeci[0] + R[1, 2] * rduskeci[1] + R[2, 2] * rduskeci[2] ]) saved = store_data('mms' + probe + '_mec_r_sun_bcs', data={ 'x': rsun.times, 'y': rsunbcs }) if not saved: print('Problem saving r_sun_bcs') saved = store_data('mms' + probe + '_mec_r_dusk_bcs', data={ 'x': rsun.times, 'y': rduskbcs }) if not saved: print('Problem saving r_dusk_bcs') # Rotation matrices for FEEPS coord system (FCS) into body coordinate system (BCS): Ttop = np.array([[1. / np.sqrt(2.), -1. / np.sqrt(2.), 0], [1. / np.sqrt(2.), 1. / np.sqrt(2.), 0], [0, 0, 1]]).T Tbot = np.array([[-1. / np.sqrt(2.), -1. / np.sqrt(2.), 0], [-1. / np.sqrt(2.), 1. / np.sqrt(2.), 0], [0, 0, -1]]).T # Telescope vectors in FCS: # Electrons V1fcs = [0.347, -0.837, 0.423] V2fcs = [0.347, -0.837, -0.423] V3fcs = [0.837, -0.347, 0.423] V4fcs = [0.837, -0.347, -0.423] V5fcs = [-0.087, 0.000, 0.996] V9fcs = [0.837, 0.347, 0.423] V10fcs = [0.837, 0.347, -0.423] V11fcs = [0.347, 0.837, 0.423] V12fcs = [0.347, 0.837, -0.423] # Ions V6fcs = [0.104, 0.180, 0.978] V7fcs = [0.654, -0.377, 0.656] V8fcs = [0.654, -0.377, -0.656] # Now telescope vectors in Body Coordinate System: # Factors of -1 account for 180 deg shift between particle velocity and telescope normal direction: # Top: Vt1bcs = [ -1. * (Ttop[0, 0] * V1fcs[0] + Ttop[1, 0] * V1fcs[1] + Ttop[2, 0] * V1fcs[2]), -1. * (Ttop[0, 1] * V1fcs[0] + Ttop[1, 1] * V1fcs[1] + Ttop[2, 1] * V1fcs[2]), -1. * (Ttop[0, 2] * V1fcs[0] + Ttop[1, 2] * V1fcs[1] + Ttop[2, 2] * V1fcs[2]) ] Vt2bcs = [ -1. * (Ttop[0, 0] * V2fcs[0] + Ttop[1, 0] * V2fcs[1] + Ttop[2, 0] * V2fcs[2]), -1. * (Ttop[0, 1] * V2fcs[0] + Ttop[1, 1] * V2fcs[1] + Ttop[2, 1] * V2fcs[2]), -1. * (Ttop[0, 2] * V2fcs[0] + Ttop[1, 2] * V2fcs[1] + Ttop[2, 2] * V2fcs[2]) ] Vt3bcs = [ -1. * (Ttop[0, 0] * V3fcs[0] + Ttop[1, 0] * V3fcs[1] + Ttop[2, 0] * V3fcs[2]), -1. * (Ttop[0, 1] * V3fcs[0] + Ttop[1, 1] * V3fcs[1] + Ttop[2, 1] * V3fcs[2]), -1. * (Ttop[0, 2] * V3fcs[0] + Ttop[1, 2] * V3fcs[1] + Ttop[2, 2] * V3fcs[2]) ] Vt4bcs = [ -1. * (Ttop[0, 0] * V4fcs[0] + Ttop[1, 0] * V4fcs[1] + Ttop[2, 0] * V4fcs[2]), -1. * (Ttop[0, 1] * V4fcs[0] + Ttop[1, 1] * V4fcs[1] + Ttop[2, 1] * V4fcs[2]), -1. * (Ttop[0, 2] * V4fcs[0] + Ttop[1, 2] * V4fcs[1] + Ttop[2, 2] * V4fcs[2]) ] Vt5bcs = [ -1. * (Ttop[0, 0] * V5fcs[0] + Ttop[1, 0] * V5fcs[1] + Ttop[2, 0] * V5fcs[2]), -1. * (Ttop[0, 1] * V5fcs[0] + Ttop[1, 1] * V5fcs[1] + Ttop[2, 1] * V5fcs[2]), -1. * (Ttop[0, 2] * V5fcs[0] + Ttop[1, 2] * V5fcs[1] + Ttop[2, 2] * V5fcs[2]) ] Vt6bcs = [ -1. * (Ttop[0, 0] * V6fcs[0] + Ttop[1, 0] * V6fcs[1] + Ttop[2, 0] * V6fcs[2]), -1. * (Ttop[0, 1] * V6fcs[0] + Ttop[1, 1] * V6fcs[1] + Ttop[2, 1] * V6fcs[2]), -1. * (Ttop[0, 2] * V6fcs[0] + Ttop[1, 2] * V6fcs[1] + Ttop[2, 2] * V6fcs[2]) ] Vt7bcs = [ -1. * (Ttop[0, 0] * V7fcs[0] + Ttop[1, 0] * V7fcs[1] + Ttop[2, 0] * V7fcs[2]), -1. * (Ttop[0, 1] * V7fcs[0] + Ttop[1, 1] * V7fcs[1] + Ttop[2, 1] * V7fcs[2]), -1. * (Ttop[0, 2] * V7fcs[0] + Ttop[1, 2] * V7fcs[1] + Ttop[2, 2] * V7fcs[2]) ] Vt8bcs = [ -1. * (Ttop[0, 0] * V8fcs[0] + Ttop[1, 0] * V8fcs[1] + Ttop[2, 0] * V8fcs[2]), -1. * (Ttop[0, 1] * V8fcs[0] + Ttop[1, 1] * V8fcs[1] + Ttop[2, 1] * V8fcs[2]), -1. * (Ttop[0, 2] * V8fcs[0] + Ttop[1, 2] * V8fcs[1] + Ttop[2, 2] * V8fcs[2]) ] Vt9bcs = [ -1. * (Ttop[0, 0] * V9fcs[0] + Ttop[1, 0] * V9fcs[1] + Ttop[2, 0] * V9fcs[2]), -1. * (Ttop[0, 1] * V9fcs[0] + Ttop[1, 1] * V9fcs[1] + Ttop[2, 1] * V9fcs[2]), -1. * (Ttop[0, 2] * V9fcs[0] + Ttop[1, 2] * V9fcs[1] + Ttop[2, 2] * V9fcs[2]) ] Vt10bcs = [ -1. * (Ttop[0, 0] * V10fcs[0] + Ttop[1, 0] * V10fcs[1] + Ttop[2, 0] * V10fcs[2]), -1. * (Ttop[0, 1] * V10fcs[0] + Ttop[1, 1] * V10fcs[1] + Ttop[2, 1] * V10fcs[2]), -1. * (Ttop[0, 2] * V10fcs[0] + Ttop[1, 2] * V10fcs[1] + Ttop[2, 2] * V10fcs[2]) ] Vt11bcs = [ -1. * (Ttop[0, 0] * V11fcs[0] + Ttop[1, 0] * V11fcs[1] + Ttop[2, 0] * V11fcs[2]), -1. * (Ttop[0, 1] * V11fcs[0] + Ttop[1, 1] * V11fcs[1] + Ttop[2, 1] * V11fcs[2]), -1. * (Ttop[0, 2] * V11fcs[0] + Ttop[1, 2] * V11fcs[1] + Ttop[2, 2] * V11fcs[2]) ] Vt12bcs = [ -1. * (Ttop[0, 0] * V12fcs[0] + Ttop[1, 0] * V12fcs[1] + Ttop[2, 0] * V12fcs[2]), -1. * (Ttop[0, 1] * V12fcs[0] + Ttop[1, 1] * V12fcs[1] + Ttop[2, 1] * V12fcs[2]), -1. * (Ttop[0, 2] * V12fcs[0] + Ttop[1, 2] * V12fcs[1] + Ttop[2, 2] * V12fcs[2]) ] # Bottom: Vb1bcs = [ -1. * (Tbot[0, 0] * V1fcs[0] + Tbot[1, 0] * V1fcs[1] + Tbot[2, 0] * V1fcs[2]), -1. * (Tbot[0, 1] * V1fcs[0] + Tbot[1, 1] * V1fcs[1] + Tbot[2, 1] * V1fcs[2]), -1. * (Tbot[0, 2] * V1fcs[0] + Tbot[1, 2] * V1fcs[1] + Tbot[2, 2] * V1fcs[2]) ] Vb2bcs = [ -1. * (Tbot[0, 0] * V2fcs[0] + Tbot[1, 0] * V2fcs[1] + Tbot[2, 0] * V2fcs[2]), -1. * (Tbot[0, 1] * V2fcs[0] + Tbot[1, 1] * V2fcs[1] + Tbot[2, 1] * V2fcs[2]), -1. * (Tbot[0, 2] * V2fcs[0] + Tbot[1, 2] * V2fcs[1] + Tbot[2, 2] * V2fcs[2]) ] Vb3bcs = [ -1. * (Tbot[0, 0] * V3fcs[0] + Tbot[1, 0] * V3fcs[1] + Tbot[2, 0] * V3fcs[2]), -1. * (Tbot[0, 1] * V3fcs[0] + Tbot[1, 1] * V3fcs[1] + Tbot[2, 1] * V3fcs[2]), -1. * (Tbot[0, 2] * V3fcs[0] + Tbot[1, 2] * V3fcs[1] + Tbot[2, 2] * V3fcs[2]) ] Vb4bcs = [ -1. * (Tbot[0, 0] * V4fcs[0] + Tbot[1, 0] * V4fcs[1] + Tbot[2, 0] * V4fcs[2]), -1. * (Tbot[0, 1] * V4fcs[0] + Tbot[1, 1] * V4fcs[1] + Tbot[2, 1] * V4fcs[2]), -1. * (Tbot[0, 2] * V4fcs[0] + Tbot[1, 2] * V4fcs[1] + Tbot[2, 2] * V4fcs[2]) ] Vb5bcs = [ -1. * (Tbot[0, 0] * V5fcs[0] + Tbot[1, 0] * V5fcs[1] + Tbot[2, 0] * V5fcs[2]), -1. * (Tbot[0, 1] * V5fcs[0] + Tbot[1, 1] * V5fcs[1] + Tbot[2, 1] * V5fcs[2]), -1. * (Tbot[0, 2] * V5fcs[0] + Tbot[1, 2] * V5fcs[1] + Tbot[2, 2] * V5fcs[2]) ] Vb6bcs = [ -1. * (Tbot[0, 0] * V6fcs[0] + Tbot[1, 0] * V6fcs[1] + Tbot[2, 0] * V6fcs[2]), -1. * (Tbot[0, 1] * V6fcs[0] + Tbot[1, 1] * V6fcs[1] + Tbot[2, 1] * V6fcs[2]), -1. * (Tbot[0, 2] * V6fcs[0] + Tbot[1, 2] * V6fcs[1] + Tbot[2, 2] * V6fcs[2]) ] Vb7bcs = [ -1. * (Tbot[0, 0] * V7fcs[0] + Tbot[1, 0] * V7fcs[1] + Tbot[2, 0] * V7fcs[2]), -1. * (Tbot[0, 1] * V7fcs[0] + Tbot[1, 1] * V7fcs[1] + Tbot[2, 1] * V7fcs[2]), -1. * (Tbot[0, 2] * V7fcs[0] + Tbot[1, 2] * V7fcs[1] + Tbot[2, 2] * V7fcs[2]) ] Vb8bcs = [ -1. * (Tbot[0, 0] * V8fcs[0] + Tbot[1, 0] * V8fcs[1] + Tbot[2, 0] * V8fcs[2]), -1. * (Tbot[0, 1] * V8fcs[0] + Tbot[1, 1] * V8fcs[1] + Tbot[2, 1] * V8fcs[2]), -1. * (Tbot[0, 2] * V8fcs[0] + Tbot[1, 2] * V8fcs[1] + Tbot[2, 2] * V8fcs[2]) ] Vb9bcs = [ -1. * (Tbot[0, 0] * V9fcs[0] + Tbot[1, 0] * V9fcs[1] + Tbot[2, 0] * V9fcs[2]), -1. * (Tbot[0, 1] * V9fcs[0] + Tbot[1, 1] * V9fcs[1] + Tbot[2, 1] * V9fcs[2]), -1. * (Tbot[0, 2] * V9fcs[0] + Tbot[1, 2] * V9fcs[1] + Tbot[2, 2] * V9fcs[2]) ] Vb10bcs = [ -1. * (Tbot[0, 0] * V10fcs[0] + Tbot[1, 0] * V10fcs[1] + Tbot[2, 0] * V10fcs[2]), -1. * (Tbot[0, 1] * V10fcs[0] + Tbot[1, 1] * V10fcs[1] + Tbot[2, 1] * V10fcs[2]), -1. * (Tbot[0, 2] * V10fcs[0] + Tbot[1, 2] * V10fcs[1] + Tbot[2, 2] * V10fcs[2]) ] Vb11bcs = [ -1. * (Tbot[0, 0] * V11fcs[0] + Tbot[1, 0] * V11fcs[1] + Tbot[2, 0] * V11fcs[2]), -1. * (Tbot[0, 1] * V11fcs[0] + Tbot[1, 1] * V11fcs[1] + Tbot[2, 1] * V11fcs[2]), -1. * (Tbot[0, 2] * V11fcs[0] + Tbot[1, 2] * V11fcs[1] + Tbot[2, 2] * V11fcs[2]) ] Vb12bcs = [ -1. * (Tbot[0, 0] * V12fcs[0] + Tbot[1, 0] * V12fcs[1] + Tbot[2, 0] * V12fcs[2]), -1. * (Tbot[0, 1] * V12fcs[0] + Tbot[1, 1] * V12fcs[1] + Tbot[2, 1] * V12fcs[2]), -1. * (Tbot[0, 2] * V12fcs[0] + Tbot[1, 2] * V12fcs[1] + Tbot[2, 2] * V12fcs[2]) ] fgm_vars = mms.fgm( trange=[time_double(trange[0]) - 600, time_double(trange[1]) + 600], probe=probe, data_rate='srvy') if fgm_vars is None: print( 'Problem loading FGM vars for calculating FEEPS gyrophase angles') # interpolate the FGM var to the MEC var timestamps tinterpol('mms' + probe + '_fgm_b_bcs_srvy_l2_bvec', 'mms' + probe + '_mec_r_sun_bcs', newname='mms' + probe + '_fgm_b_bcs_srvy_l2_bvec_int') B = get_data('mms' + probe + '_fgm_b_bcs_srvy_l2_bvec_int') # Now calculate gyrophase # Telescope vectors perp to B: Tperp = np.zeros((len(rsunbcs[:, 0]), 3, 24)) # Gyrophase: phi = np.zeros((len(rsunbcs[:, 0]), 24)) for i in range(len(rsunbcs[:, 0])): uB = B.y[i, :] / np.sqrt(B.y[i, 0]**2 + B.y[i, 1]**2 + B.y[i, 2]**2) # Sun vector perp to B: Sperp = np.cross( np.cross(uB, rsunbcs[i, :] / np.sqrt(np.nansum(rsunbcs[i, :]**2))), uB) # Dusk vector perp to B: Dperp = np.cross( np.cross(uB, rduskbcs[i, :] / np.sqrt(np.nansum(rduskbcs[i, :]**2))), uB) Tperp[i, :, 0] = np.cross(np.cross(uB, Vt1bcs), uB) Tperp[i, :, 1] = np.cross(np.cross(uB, Vt2bcs), uB) Tperp[i, :, 2] = np.cross(np.cross(uB, Vt3bcs), uB) Tperp[i, :, 3] = np.cross(np.cross(uB, Vt4bcs), uB) Tperp[i, :, 4] = np.cross(np.cross(uB, Vt5bcs), uB) Tperp[i, :, 5] = np.cross(np.cross(uB, Vt6bcs), uB) Tperp[i, :, 6] = np.cross(np.cross(uB, Vt7bcs), uB) Tperp[i, :, 7] = np.cross(np.cross(uB, Vt8bcs), uB) Tperp[i, :, 8] = np.cross(np.cross(uB, Vt9bcs), uB) Tperp[i, :, 9] = np.cross(np.cross(uB, Vt10bcs), uB) Tperp[i, :, 10] = np.cross(np.cross(uB, Vt11bcs), uB) Tperp[i, :, 11] = np.cross(np.cross(uB, Vt12bcs), uB) Tperp[i, :, 12] = np.cross(np.cross(uB, Vb1bcs), uB) Tperp[i, :, 13] = np.cross(np.cross(uB, Vb2bcs), uB) Tperp[i, :, 14] = np.cross(np.cross(uB, Vb3bcs), uB) Tperp[i, :, 15] = np.cross(np.cross(uB, Vb4bcs), uB) Tperp[i, :, 16] = np.cross(np.cross(uB, Vb5bcs), uB) Tperp[i, :, 17] = np.cross(np.cross(uB, Vb6bcs), uB) Tperp[i, :, 18] = np.cross(np.cross(uB, Vb7bcs), uB) Tperp[i, :, 19] = np.cross(np.cross(uB, Vb8bcs), uB) Tperp[i, :, 20] = np.cross(np.cross(uB, Vb9bcs), uB) Tperp[i, :, 21] = np.cross(np.cross(uB, Vb10bcs), uB) Tperp[i, :, 22] = np.cross(np.cross(uB, Vb11bcs), uB) Tperp[i, :, 23] = np.cross(np.cross(uB, Vb12bcs), uB) for j in range(24): th1 = np.arccos( np.nansum(Tperp[i, :, j] * Sperp) / (np.sqrt(np.nansum(Tperp[i, :, j]**2)) * np.sqrt(np.nansum(Sperp**2)))) th2 = np.arccos( np.nansum(Tperp[i, :, j] * Dperp) / (np.sqrt(np.nansum(Tperp[i, :, j]**2)) * np.sqrt(np.nansum(Dperp**2)))) if th1 <= np.pi / 2.0 and th2 < np.pi / 2: phi[i, j] = 2 * np.pi - th1 if th1 < np.pi / 2.0 and th2 >= np.pi / 2.0: phi[i, j] = th1 if th1 > np.pi / 2.0 and th2 <= np.pi / 2.0: phi[i, j] = 270.0 * np.pi / 180.0 - th2 if th1 >= np.pi / 2.0 and th2 > np.pi / 2.0: phi[i, j] = th1 saved = store_data('mms' + probe + '_epd_feeps_' + data_rate + '_gyrophase', data={ 'x': rsun.times, 'y': phi * 180. / np.pi }) if not saved: print('Problem saving gyrophase angles') return options('mms' + probe + '_epd_feeps_' + data_rate + '_gyrophase', 'yrange', [0, 360.0]) # Gyrophase always returns on time stamps from MEC data, get those closest to FEEPS time stamps: eyes = mms_feeps_active_eyes(trange, probe, data_rate, datatype, level) sensor_types = ['top', 'bottom'] feepst = get_data('mms' + probe + '_epd_feeps_' + data_rate + '_' + level + '_' + datatype + '_spinsectnum') indt = np.zeros(len(feepst.times), dtype='int32') gpd = get_data('mms' + probe + '_epd_feeps_' + data_rate + '_gyrophase') for i in range(len(feepst.times)): indt[i] = np.argwhere( np.abs(gpd.times - feepst.times[i]) == np.min( np.abs(gpd.times - feepst.times[i]))).flatten()[0] # Gyrophase always returns all 24 FEEPS telescopes, downselect based on species: iT = np.array([ np.array(eyes[sensor_types[0]]) - 1, np.array(eyes[sensor_types[0]]) + 11 ]).flatten().tolist() gp_data = np.zeros((len(gpd.times[indt]), len(iT))) #return (iT, gp_data, gpd) for i in range(len(iT)): gp_data[:, i] = gpd.y[indt, iT[i]] saved = store_data('mms' + probe + '_epd_feeps_' + data_rate + '_' + level + '_' + datatype + '_gyrophase', data={ 'x': gpd.times[indt], 'y': gp_data }) if saved: options( 'mms' + probe + '_epd_feeps_' + data_rate + '_' + level + '_' + datatype + '_gyrophase', 'yrange', [0.0, 360.0]) return 'mms' + probe + '_epd_feeps_' + data_rate + '_' + level + '_' + datatype + '_gyrophase'
def hapi(trange=None, server=None, dataset=None, parameters='', suffix='', catalog=False): """ Loads data from a HAPI server into pytplot variables Parameters ----------- trange: list of str or list of float Time range to load the data for server: str HAPI server to load the data from dataset: str HAPI dataset to load parameters: str or list of str Parameters in the dataset to load; default is to load them all suffix: str Suffix to append to the tplot variables catalog: bool If True, returns the server's catalog of datasets Returns ------- List of tplot variables created. """ if server is None: print('Error, no server specified; example servers include:') print('- https://cdaweb.gsfc.nasa.gov/hapi') print('- https://pds-ppi.igpp.ucla.edu/hapi') print('- http://planet.physics.uiowa.edu/das/das2Server/hapi') print('- https://iswa.gsfc.nasa.gov/IswaSystemWebApp/hapi') print('- http://lasp.colorado.edu/lisird/hapi') return if catalog: catalog = load_hapi(server) items = [] if 'catalog' in catalog.keys(): items = catalog['catalog'] print('Available datasets: ') for item in items: if 'title' in item.keys(): print(item['id'] + ': ' + item['title']) else: print(item['id']) return if dataset is None: print('Error, no dataset specified; please see the catalog for a list of available data sets.') return if trange is None: print('Error, no trange specified') return if isinstance(parameters, list): parameters = ','.join(parameters) opts = {'logging': False} data, hapi_metadata = load_hapi(server, dataset, parameters, trange[0], trange[1], **opts) out_vars = [] # loop through the parameters in this dataset params = hapi_metadata['parameters'] for param in params[1:]: spec = False param_name = param.get('name') print('Loading ' + param_name) # load the data only for this parameter try: with warnings.catch_warnings(): warnings.simplefilter("ignore", category=ResourceWarning) data, hapi_metadata = load_hapi(server, dataset, param_name, trange[0], trange[1], **opts) except: breakpoint() print('Error! 95') continue timestamps = [datapoint[0] for datapoint in data] unixtimes = [time_double(timestamp.decode('utf-8')) for timestamp in timestamps] param_type = hapi_metadata['parameters'][1].get('type') if param_type is None: param_type = 'double' data_size = hapi_metadata['parameters'][1].get('size') if data_size is None: single_line = True try: if param_type == 'double': single_line = isinstance(data[0][1], np.float64) elif param_type == 'integer': single_line = isinstance(data[0][1], np.int32) except IndexError: breakpoint() print('Error! 103') continue if single_line: data_out = np.zeros((len(data))) else: try: data_out = np.zeros((len(data), len(data[0][1]))) except TypeError: print('Error! 112') breakpoint() continue for idx, datapoint in enumerate(data): if single_line: data_out[idx] = datapoint[1] else: data_out[idx, :] = datapoint[1] data_out = data_out.squeeze() # check for fill values fill_value = hapi_metadata['parameters'][1].get('fill') if fill_value is not None: if param_type == 'double': fill_value = float(fill_value) data_out[data_out == fill_value] = np.nan elif param_type == 'integer': # NaN is only floating point, so we replace integer fill # values with 0 instead of NaN fill_value = int(fill_value) data_out[data_out == fill_value] = 0 bins = param.get('bins') if bins is not None: centers = bins[0].get('centers') if centers is not None: spec = True data_table = {'x': unixtimes, 'y': data_out} if spec: data_table['v'] = centers saved = store_data(param_name + suffix, data=data_table) metadata = get_data(param_name + suffix, metadata=True) metadata['HAPI'] = hapi_metadata if spec: options(param_name + suffix, 'spec', True) if saved: out_vars.append(param_name + suffix) # wait for a second before going to the next variable # to avoid hitting the server too quickly sleep(1) return out_vars