def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): remote_url = remote_base_url + str(directory) # Now load remotely util.load(fname + extension, local_base_dir / directory, remote_url)
def download_func(remote_base_url, local_base_dir, directory, fname, extension): remote_url = '{}{}'.format(remote_base_url, directory) util.load(fname + extension, local_base_dir / directory, remote_url, guessversion=True)
def _merged_fromascii(probe, year, doy, try_download): """ Read in a single day of merged data. Data is loaded from orignal ascii files, and saved to a hdf file for faster access after first read in. Parameters ---------- probe : int, string Helios probe to import data from. Must be 1 or 2. year : int Year doy : int Day of year Returns ------- data : DataFrame Merged data set """ probe = _check_probe(probe) local_dir = _merged_localdir(probe) remote_url = ('ftp://cdaweb.gsfc.nasa.gov/pub/data/helios/' 'helios{}/merged/he{}_40sec'.format(probe, probe)) filename = _merged_fname(probe, year, doy) + '.dat' asciiloc = os.path.join(local_dir, filename) # Make sure file is downloaded util.load(filename, local_dir, remote_url, try_download=try_download) # Load data data = pd.read_table(asciiloc, delim_whitespace=True) # Process data data['year'] = data['year'].astype(int) # Convert date info to datetime data['Time'] = pd.to_datetime(data['year'], format='%Y') + \ pd.to_timedelta(data['day'] - 1, unit='d') + \ pd.to_timedelta(data['hour'], unit='h') + \ pd.to_timedelta(data['min'], unit='m') + \ pd.to_timedelta(data['sec'], unit='s') data = data.drop(['year', 'day', 'hour', 'min', 'sec', 'dechr'], axis=1) # Set zero values to nans data.replace(0.0, np.nan, inplace=True) # Save data to a hdf store if use_hdf: _save_hdf(data, _merged_localdir(probe), _merged_fname(probe, year, doy)) return (data)
def _mag_hires_helper(year, doy, local_dir, url, coords): fname = str(year)[2:] + doy + '_FGM_' + coords hdf_fname = '{}_{}.hdf'.format(year, doy) hdfloc = local_dir / hdf_fname if hdfloc.exists(): return pd.read_hdf(hdfloc) f = util.load(fname + '.TAB', local_dir, url) if f is None: raise RuntimeError( 'No file named {} exits on remote server'.format(fname)) elif 'error_message' in f.readline(): location = f.name f.close() os.remove(f.name) raise RuntimeError( 'No file named {} exits on remote server'.format(fname)) df = pd.read_table(f, names=['Time', 'Bx', 'By', 'Bz'], delim_whitespace=True, parse_dates=[0], index_col=0) f.close() if use_hdf: df.to_hdf(hdfloc, key='data', mode='w') return df
def mitplasma_h0(probe, starttime, endtime): """ Import mit h0 plasma data. Parameters ---------- probe : string Probe number. starttime : datetime Start of interval. endtime : datetime End of interval. Returns ------- data : DataFrame Requested data. """ data = [] dtimes = util._daysplitinterval(starttime, endtime) # Loop through years for dtime in dtimes: date = dtime[0] intervalstring = str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) filename = 'i' + probe + '_h0_mitplasma_' + intervalstring + '_v01.cdf' # Location of file relative to local directory or remote url relative_loc = 'imp' + probe + '/plasma_mit/mitplasma_h0/' +\ str(date.year) local_dir = os.path.join(imp_dir, relative_loc) remote_url = imp_url + relative_loc cdf = util.load(filename, local_dir, remote_url) keys = { 'EW_flowangle_best': 'EW_flowangle_best', 'EW_flowangle_mom': 'EW_flowangle_best', 'Epoch': 'Time', 'Flow_elevation_thresh': 'Flow_elevation_thresh', 'Flow_elevation_threshsp': 'Flow_elevation_threshsp', 'Region': 'Region', 'V_fit': 'V_fit', 'V_mom': 'V_fit', 'mode': 'mode', 'protonV_thermal_fit': 'protonV_thermal_fit', 'protonV_thermal_mom': 'protonV_thermal_fit', 'proton_density_fit': 'proton_density_fit', 'proton_density_mom': 'proton_density_mom', 'xyzgse': ['x_gse', 'y_gse', 'z_gse'], 'ygsm': 'ygsm', 'zgsm': 'zgsm' } thisdata = util.cdf2df(cdf, 'Epoch', keys) data.append(thisdata) data = pd.concat(data) data = data[(data['Time'] > starttime) & (data['Time'] < endtime)] data.index.name = 'Time' return data
def download_func(remote_base_url, local_base_dir, directory, fname, extension): def check_exists(): # Because the version might be different to the one we guess, work # out the downloaded filename for f in os.listdir(os.path.join(local_base_dir, directory)): if (f[:-6] == (fname + extension)[:-6]): # Return filename with '.cdf' stripped off the end return f[:-4] if check_exists() is not None: return check_exists() # Now load remotely util.load(fname + extension, os.path.join(local_base_dir, directory), remote_base_url + directory, guessversion=True) if check_exists() is not None: return check_exists()
def mag320ms(probe, startTime, endTime): """ Import 320ms cadence magnetic field data. Parameters ---------- probe : string Probe number. starttime : datetime Start of interval. endtime : datetime End of interval. Returns ------- data : DataFrame Requested data. """ data = [] dtimes = util._daysplitinterval(startTime, endTime) # Loop through years for dtime in dtimes: date = dtime[0] intervalstring = str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) filename = 'i8_320msec_mag_' + intervalstring + '_v01.cdf' hdffname = filename[:-3] + '.hdf' # Location of file relative to local directory or remote url relative_loc = 'imp' + probe + '/mag/mag_320msec_cdaweb/' +\ str(date.year) local_dir = os.path.join(imp_dir, relative_loc) hdffile = os.path.join(local_dir, hdffname) if os.path.exists(hdffile): thisdata = pd.read_hdf(hdffile) data.append(thisdata) continue remote_url = imp_url + relative_loc cdf = util.load(filename, local_dir, remote_url) keys = { 'B': '|B|', 'BX': 'Bx', 'BY': 'By', 'BZ': 'Bz', 'Epoch': 'Time' } thisdata = util.cdf2df(cdf, 'Epoch', keys) data.append(thisdata) if use_hdf: thisdata.to_hdf(hdffile, key='merged', mode='w') data = pd.concat(data) data = data[(data['Time'] > startTime) & (data['Time'] < endTime)] data = data.drop(columns='Time') data.index.name = 'Time' return data
def fgm_hires(starttime, endtime): """ Import high resolution fluxgate magnetometer data. Parameters ---------- starttime : datetime Start of interval endtime : datetime End of interval Returns ------- data : DataFrame Requested data """ fgm_options = url_options readargs = { 'names': ['year', 'doy', 'hour', 'minute', 'second', 'Bx', 'By', 'Bz', '|B|'], 'delim_whitespace': True } data = [] dtimes = util._daysplitinterval(starttime, endtime) # Loop through years for dtime in dtimes: date = dtime[0] yearstr = date.strftime('%Y') fgm_options['FILE_NAME'] = ('U' + yearstr[-2:] + date.strftime('%j') + 'SH.ASC') # Local locaiton to download to local_dir = ulysses_dir / 'fgm' / 'hires' / yearstr local_file = local_dir / fgm_options['FILE_NAME'] local_hdf = local_file.with_suffix('.hdf') # If we have already saved a hdf file if local_hdf.exists(): thisdata = pd.read_hdf(local_hdf) else: # Put together remote url fgm_options['FILE_PATH'] = '/ufa/HiRes/VHM-FGM/' + yearstr remote_url = ulysses_url for key in fgm_options: remote_url += key + '=' + fgm_options[key] + '&' f = util.load(fgm_options['FILE_NAME'], local_dir, remote_url) # Read in data thisdata = pd.read_table(f, **readargs) # Process data/time thisdata = _convert_ulysses_time(thisdata) if use_hdf: thisdata.to_hdf(local_hdf, 'fgm_hires') data.append(thisdata) return util.timefilter(data, starttime, endtime)
def download_func(remote_base_url, local_base_dir, directory, fname, extension): def check_exists(): # Because the version might be different to the one we guess, work # out the downloaded filename for f in (local_base_dir / directory).iterdir(): fstr = str(f.name) if (fstr[:-6] == (fname + extension)[:-6]): # Return filename with '.cdf' stripped off the end return fstr[:-4] if check_exists() is not None: return check_exists() # Now load remotely util.load(fname + extension, local_base_dir / directory, remote_base_url + str(directory), guessversion=True) return check_exists()
def _mag_ness_fromascii(probe, year, doy, try_download=True): """ Read in a single day of 6 second magnetic field data. Data is read from orignal ascii files, and saved to a hdf file for faster access after the first read. Parameters ---------- probe : int, string Helios probe to import data from. Must be 1 or 2. year : int Year doy : int Day of year Returns ------- data : DataFrame 6 second magnetic field data set """ probe = _check_probe(probe) local_dir = _ness_localdir(probe, year) remote_url = ('ftp://spdf.sci.gsfc.nasa.gov/pub/data/helios/helios' + probe + '/mag/6sec_ness/' + str(year) + '/') fname = _ness_fname(probe, year, doy) + '.asc' f = util.load(fname, local_dir, remote_url, try_download=try_download) # Read in data headings = [ 'probe', 'year', 'doy', 'hour', 'minute', 'second', 'naverage', 'Bx', 'By', 'Bz', '|B|', 'sigma_Bx', 'sigma_By', 'sigma_Bz' ] colspecs = [(1, 2), (2, 4), (4, 7), (7, 9), (9, 11), (11, 13), (13, 15), (15, 22), (22, 29), (29, 36), (36, 42), (42, 48), (48, 54), (54, 60)] data = pd.read_fwf(f, names=headings, header=None, colspecs=colspecs) # Process data data['year'] += 1900 # Convert date info to datetime data['Time'] = pd.to_datetime(data['year'], format='%Y') + \ pd.to_timedelta(data['doy'] - 1, unit='d') + \ pd.to_timedelta(data['hour'], unit='h') + \ pd.to_timedelta(data['minute'], unit='m') + \ pd.to_timedelta(data['second'], unit='s') data = data.drop(['year', 'doy', 'hour', 'minute', 'second'], axis=1) data = data.set_index('Time', drop=False) # Save data to a hdf store if use_hdf: _save_hdf(data, local_dir, _ness_fname(probe, year, doy)) return (data)
def processing_func(local_dir, fname): distkeys = [] for i in range(0, 13): distkeys.append('f_pitch_E' + str(i).zfill(2)) anglelabels = [] for i in range(0, 30): anglelabels.append((i + 0.5) * np.pi / 30) timekey = 'Epoch' energykey = 'Ve' cdf = util.load(fname, local_dir, '') df = util.pitchdist_cdf2df(cdf, distkeys, energykey, timekey, anglelabels) return df
def processing_func(directory, fname): cdf = util.load(fname, directory, '') epoch_dict = {'h0': 'Epoch3', 'h2': 'Epoch'} mag_dict = {'h0': 'B3GSE', 'h2': 'BGSE'} epoch_key = epoch_dict[version] mag_key = mag_dict[version] keys = {mag_key: ['Bx_gse', 'By_gse', 'Bz_gse'], epoch_key: 'Time'} badvalues = {'Bx_gse': -1e+31, 'By_gse': -1e+31, 'Bz_gse': -1e+31} df = util.cdf2df(cdf, index_key=epoch_key, keys=keys, badvalues=badvalues) return df
def fgm_survey(probe, starttime, endtime): """ Import fgm survey mode magnetic field data. Parameters ---------- probe : string Probe number, must be 1, 2, 3, or 4 starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame Imported data. """ # Directory relative to main MMS data directory relative_dir = os.path.join('mms' + probe, 'fgm', 'srvy', 'l2') daylist = util._daysplitinterval(starttime, endtime) data = [] for day in daylist: date = day[0] this_relative_dir = os.path.join(relative_dir, str(date.year), str(date.month).zfill(2)) filename = 'mms{}_fgm_srvy_l2_{}{:02}{:02}_v4.18.0.cdf'.format( probe, date.year, date.month, date.day) # Absolute path to local directory for this data file local_dir = os.path.join(mms_dir, this_relative_dir) util._checkdir(local_dir) remote_url = remote_mms_dir + this_relative_dir # Load cdf file cdf = util.load(filename, local_dir, remote_url) # Convert cdf to dataframe keys = { 'mms' + probe + '_fgm_b_gsm_srvy_l2': ['Bx', 'By', 'Bz', 'Br'], 'Epoch': 'Time' } df = util.cdf2df(cdf, 'Epoch', keys) data.append(df) return util.timefilter(data, starttime, endtime)
def _swics(starttime, endtime, names, product): swics_options = url_options readargs = {'names': names, 'delim_whitespace': True} data = [] dtimes = util._daysplitinterval(starttime, endtime) # Loop through years for year in range(starttime.year, endtime.year + 1): swics_options['FILE_NAME'] = '{}{}.dat'.format(product, str(year)[-2:]) # Local locaiton to download to local_dir = os.path.join(ulysses_dir, 'swics') local_file = os.path.join(local_dir, swics_options['FILE_NAME']) local_hdf = local_file[:-4] + '.hdf' # If we have already saved a hdf file if os.path.exists(local_hdf): thisdata = pd.read_hdf(local_hdf) else: # Put together remote url swics_options['FILE_PATH'] = '/ufa/HiRes/data/swics' remote_url = ulysses_url for key in swics_options: remote_url += key + '=' + swics_options[key] + '&' f = util.load(swics_options['FILE_NAME'], local_dir, remote_url) if f is None: print('File {}/{} not available\n'.format( remote_url, filename)) continue # Read in data thisdata = pd.read_table(f, **readargs) # Process data/time thisdata = _convert_ulysses_time(thisdata) if use_hdf: thisdata.to_hdf(local_hdf, 'swics') data.append(thisdata) return util.timefilter(data, starttime, endtime)
def threedp_sfpd(starttime, endtime): """ Import 'sfpd' wind data. 12 second energetic electron pitch-angle energy spectra from the foil SST Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame """ # Directory relative to main WIND data directory relative_dir = path.Path('3dp') / '3dp_sfpd' daylist = util._daysplitinterval(starttime, endtime) data = [] mag = [] for (date, _, _) in daylist: this_relative_dir = relative_dir / str(date.year) # Absolute path to local directory for this data file local_dir = wind_dir / this_relative_dir filename = 'wi_sfpd_3dp_{:{dfmt}}_v02'.format(date, dfmt='%Y%m%d') hdfname = filename + '.hdf' hdfloc = local_dir / hdfname if hdfloc.exists(): df = pd.read_hdf(hdfloc) data.append(df) continue util._checkdir(local_dir) remote_url = remote_wind_dir + str(this_relative_dir) cdf = util.load(filename + '.cdf', local_dir, remote_url, guessversion=True) if cdf is None: print('File {}/{} not available\n'.format(remote_url, filename)) continue data_today = [] # Loop through each timestamp to build up fluxes for i, time in enumerate(cdf['Epoch'][...]): energies = cdf['ENERGY'][i, :] angles = cdf['PANGLE'][i, :] fluxes = cdf['FLUX'][i, :, :] magfield = cdf['MAGF'][i, :] index = pd.MultiIndex.from_product( ([time], energies, angles), names=['Time', 'Energy', 'Pitch angle']) df = pd.DataFrame(fluxes.ravel(), index=index, columns=['Flux']) df['Bx'] = magfield[0] df['By'] = magfield[1] df['Bz'] = magfield[2] data_today.append(df) data_today = pd.concat(data_today) data_today = data_today.sort_index() if use_hdf: data_today.to_hdf(hdfloc, 'sfpd', mode='w') data.append(data_today) data = util.timefilter(data, starttime, endtime) data = data.reset_index(level=['Energy', 'Pitch angle']) return data
def threedp_pm(starttime, endtime): """ Import 'pm' wind data. 3 second time resolution solar wind proton and alpha particle moments from the PESA LOW sensor, computed on-board the spacecraft Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame """ # Directory relative to main WIND data directory relative_dir = path.Path('3dp') / '3dp_pm' daylist = util._daysplitinterval(starttime, endtime) data = [] for day in daylist: date = day[0] this_relative_dir = relative_dir / str(day[0].year) # Absolute path to local directory for this data file local_dir = wind_dir / this_relative_dir filename = 'wi_pm_3dp_' +\ str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) +\ '_v05.cdf' hdfname = filename[:-4] + 'hdf' hdfloc = local_dir / hdfname if hdfloc.exists(): df = pd.read_hdf(hdfloc) data.append(df) continue util._checkdir(local_dir) remote_url = remote_wind_dir + str(this_relative_dir) cdf = util.load(filename, local_dir, remote_url, guessversion=True) if cdf is None: print('File {}/{} not available\n'.format(remote_url, filename)) continue keys = { 'A_DENS': 'n_a', 'A_TEMP': 'T_a', 'A_VELS': ['va_x', 'va_y', 'va_z'], 'P_DENS': 'n_p', 'P_TEMP': 'T_p', 'P_VELS': ['vp_x', 'vp_y', 'vp_z'], 'Epoch': 'Time' } df = util.cdf2df(cdf, index_key='Epoch', keys=keys) if use_hdf: df.to_hdf(hdfloc, 'pm', mode='w') data.append(df) return util.timefilter(data, starttime, endtime)
def fpi_dis_moms(probe, mode, starttime, endtime): """ Import fpi ion distribution moment data. Parameters ---------- probe : string Probe number, must be 1, 2, 3, or 4 mode : string Data mode, must be 'fast' or 'brst' starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame Imported data. """ valid_modes = ['fast', 'brst'] if mode not in valid_modes: raise RuntimeError('Mode must be either fast or brst') # Directory relative to main MMS data directory relative_dir = os.path.join('mms' + probe, 'fpi', mode, 'l2', 'dis-moms') daylist = util._daysplitinterval(starttime, endtime) data = [] for day in daylist: date = day[0] starthour = day[1].hour endhour = day[2].hour + 1 # fips fast data product has files every two hours, so get nearest two # hour stamps starthour -= np.mod(starthour, 2) endhour += np.mod(endhour, 2) for h in range(starthour, endhour, 2): this_relative_dir = os.path.join(relative_dir, str(date.year), str(date.month).zfill(2)) filename = ('mms{}_fpi_{}_l2_dis-moms_' '{}{:02}{:02}{:02}0000_v3.3.0.cdf').format( probe, mode, date.year, date.month, date.day, h) # Absolute path to local directory for this data file local_dir = os.path.join(mms_dir, this_relative_dir) util._checkdir(local_dir) remote_url = remote_mms_dir + this_relative_dir # Load cdf file try: cdf = util.load(filename, local_dir, remote_url) except urllib.error.HTTPError as e: if str(e) == 'HTTP Error 404: Not Found': print('No data available for hours', str(h) + '-' + str(h + 2), 'on', date.strftime('%d/%m/%Y')) continue else: raise probestr = 'mms' + probe + '_' # Convert cdf to dataframe keys = {'Epoch': 'Time', probestr + 'dis_bulkv_gse_fast': ['bulkv_x', 'bulkv_y', 'bulkv_z'], probestr + 'dis_heatq_gse_fast': ['heatq_x', 'heatq_y', 'heatq_z'], probestr + 'dis_numberdensity_fast': 'n', probestr + 'dis_temppara_fast': 'T_par', probestr + 'dis_tempperp_fast': 'T_perp'} df = util.cdf2df(cdf, 'Epoch', keys) data.append(df) return util.timefilter(data, starttime, endtime)
def processing_func(directory, fname): cdf = util.load(fname, directory, '') return util.cdf2df(cdf, 'Epoch', badvalues=badvalues)
def mag_1min(starttime, endtime, coords): """ Import 1 minute magnetic field from Cassini. See http://pds-ppi.igpp.ucla.edu/search/view/?f=yes&id=pds://PPI/CO-E_SW_J_S-MAG-4-SUMM-1MINAVG-V1.0 for more information. Cassini Orbiter Magnetometer Calibrated MAG data in 1 minute averages available covering the period 1999-08-16 (DOY 228) to 2016-12-31 (DOY 366). The data are provided in RTN coordinates throughout the mission, with Earth, Jupiter, and Saturn centered coordinates for the respective flybys of those planets. Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. coords : strings Requested coordinate system. Must be one of ``['KRTP', 'KSM', 'KSO', 'RTN']`` Returns ------- data : DataFrame Requested data """ valid_coords = ['KRTP', 'KSM', 'KSO', 'RTN'] if coords not in valid_coords: raise ValueError('coords must be one of {}'.format(valid_coords)) base_url = ('http://pds-ppi.igpp.ucla.edu/ditdos/download?' 'id=pds://PPI/CO-E_SW_J_S-MAG-4-SUMM-1MINAVG-V1.0/DATA') data = [] for year in starttime.year, endtime.year: url = '{}/{}'.format(base_url, year) local_dir = os.path.join(cassini_dir, 'mag', '1min') fname = '{}_FGM_{}_1M'.format(year, coords) hdfloc = os.path.join(local_dir, fname + '.hdf') if os.path.isfile(hdfloc): df = pd.read_hdf(hdfloc) data.append(df) continue f = util.load(fname + '.TAB', local_dir, url) if 'error_message' in f.readline(): f.close() os.remove(os.path.join(local_dir, fname + '.TAB')) continue df = pd.read_table(f, names=[ 'Time', 'Bx', 'By', 'Bz', '|B|', 'X', 'Y', 'Z', 'Local hour', 'n points' ], delim_whitespace=True, parse_dates=[0], index_col=0) data.append(df) if use_hdf: df.to_hdf(hdfloc, key='data', mode='w') return util.timefilter(data, starttime, endtime)
def mag_rtn(starttime, endtime): """ Import magnetic field in RTN coordinates from Messenger. Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame """ # Directory relative to main WIND data directory relative_dir = 'rtn' daylist = util._daysplitinterval(starttime, endtime) data = [] for day in daylist: date = day[0] this_relative_dir = os.path.join(relative_dir, str(date.year)) hdffile = 'messenger_mag_rtn_' +\ str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) +\ '_v01.hdf' hdfloc = os.path.join(mess_dir, this_relative_dir, hdffile) # Try to load hdf file if os.path.isfile(hdfloc): df = pd.read_hdf(hdfloc) data.append(df) continue filename = hdffile[:-4] + '.cdf' # Absolute path to local directory for this data file local_dir = os.path.join(mess_dir, this_relative_dir) util._checkdir(local_dir) remote_url = os.path.join(remote_mess_dir, this_relative_dir) cdf = util.load(filename, local_dir, remote_url, guessversion=True) if cdf is None: print('File {}/{} not available\n'.format(remote_url, filename)) continue keys = { 'B_normal': 'Bn', 'B_radial': 'Br', 'B_tangential': 'Bt', 'Epoch': 'Time', 'azimuth_ecliptic': 'sc_Az', 'latitude_ecliptic': 'sc_Lat', 'radialDistance': 'sc_r', 'MissionElapsedTime': 'mission_time' } df = util.cdf2df(cdf, index_key='Epoch', keys=keys) if use_hdf: hdffile = filename[:-4] + '.hdf' df.to_hdf(hdfloc, key='data', mode='w') data.append(df) return util.timefilter(data, starttime, endtime)
def download_func(remote_base_url, local_base_dir, directory, fname, extension): remote_url = '{}{}'.format(remote_base_url, directory) util.load(fname + extension, os.path.join(local_base_dir, directory), remote_url)
def processing_func(local_dir, local_fname): cdf = util.load(local_fname, local_dir, '') return util.cdf2df(cdf, index_key='Epoch', keys=keys, badvalues=badvalues)
def _fourHz_fromascii(probe, year, doy, try_download=True): """ Read in a single day of 4Hz magnetic field data. Parameters ---------- probe : int, string Helios probe to import data from. Must be 1 or 2. year : int Year doy : int Day of year Returns ------- data : DataFrame 4Hz magnetic field data set """ probe = _check_probe(probe) local_dir = _4hz_localdir(probe) fname_prefix = _4hz_filename(probe, year, doy) # For some reason the last number in the filename is the hour at which # data starts from on that day... this means a loop to check each hour asciiloc = None fname = None for i in range(0, 24): testloc = os.path.join(local_dir, fname_prefix + str(i).zfill(2) + '.asc') if os.path.isfile(testloc): asciiloc = testloc break if asciiloc is not None: if os.name == 'nt': splitchar = '\\' else: splitchar = '/' fname = asciiloc.split(splitchar)[-1] remote_url = None elif try_download is not False: ftpsite = 'apollo.ssl.berkeley.edu' remote_dir = ('pub/helios-data/E2_experiment/' 'Data_Cologne_Nov2016_bestdata/' 'HR/helios{}'.format(probe)) remote_url = 'ftp://' + ftpsite + '/' + remote_dir fname = None with FTP(ftpsite) as ftp: ftp.login() filenames = ftp.nlst(remote_dir) for filename in filenames: if fname_prefix in filename: fname = filename break if fname is None: raise ValueError('No mag data available locally or remotely') asciiloc = util.load(fname, local_dir, remote_url) # Read in data headings = ['Time', 'Bx', 'By', 'Bz', '|B|'] cols = [0, 4, 5, 6, 7] data = pd.read_table(asciiloc, names=headings, header=None, usecols=cols, delim_whitespace=True) # Convert date info to datetime data['Time'] = pd.to_datetime(data['Time'], format='%Y-%m-%dT%H:%M:%S') data = data.set_index('Time', drop=False) # Save data to a hdf store if use_hdf: fname = _4hz_filename(probe, year, doy) _save_hdf(data, local_dir, fname) return (data)
def merged(probe, starttime, endtime, verbose=False): """ Import merged plasma data. See ftp://cdaweb.gsfc.nasa.gov/pub/data/imp/imp8/merged/00readme.txt for information on variables. Parameters ---------- probe : string Probe number. starttime : datetime Start of interval. endtime : datetime End of interval. verbose : bool, optional If ``True``, print information whilst loading. Default is ``False``. Returns ------- data : DataFrame Requested data. """ _check_probe(probe, ['8']) data = [] startyear = starttime.year endyear = endtime.year # Loop through years for year in range(startyear, endyear + 1): if year == startyear: startmonth = starttime.month else: startmonth = 1 if year == endyear: endmonth = endtime.month else: endmonth = 12 # Loop through months for month in range(startmonth, endmonth + 1): if verbose: print('Loading IMP merged probe {}, {:02d}/{}'.format( probe, month, year)) intervalstring = str(year) + str(month).zfill(2) filename = 'imp_min_merge' + intervalstring + '.asc' # Location of file relative to local directory or remote url relative_loc = os.path.join('imp' + probe, 'merged') local_dir = os.path.join(imp_dir, relative_loc) hdffile = os.path.join(local_dir, filename[:-4] + '.hdf') if os.path.isfile(hdffile): data.append(pd.read_hdf(hdffile)) continue remote_url = imp_url + relative_loc f = util.load(filename, local_dir, remote_url) if f is None: print('File {}/{} not available\n'.format( remote_url, filename)) continue readargs = { 'names': [ 'Year', 'doy', 'Hour', 'Minute', 'sw_flag', 'x_gse', 'y_gse', 'z_gse', 'y_gsm', 'z_gsm', 'Nm', 'FCm', 'DWm', '<|B|>', '|<B>|', '<B_lat>', '<B_long>', 'Bx_gse', 'By_gse', 'Bz_gse', 'By_gsm', 'Bz_gsm', 'sigma|B|', 'sigma B', 'sigma B_x', 'sigma B_y', 'sigma B_z', 'plas_reg', 'Npp', 'FCp', 'DWp', 'v_fit', 'vx_fit_gse', 'vy_fit_gse', 'vz_fit_gse', 'vlong_fit', 'vlat_fit', 'np_fit', 'Tp_fit', 'v_mom', 'vx_mom_gse', 'vy_mom_gse', 'vz_mom_gse', 'vlong_mom', 'vlat_mom', 'np_mom', 'Tp_mom' ], 'na_values': [ '9999', '999', '99', '99', '9', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9', '99', '9.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9', '9', '99', '9.99', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999999.', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999999.' ], 'delim_whitespace': True } # Read in data thisdata = pd.read_table(f, **readargs) thisdata['Time'] = ( pd.to_datetime(thisdata['Year'], format='%Y') + pd.to_timedelta(thisdata['doy'] - 1, unit='d') + pd.to_timedelta(thisdata['Hour'], unit='h') + pd.to_timedelta(thisdata['Minute'], unit='m')) if use_hdf: thisdata.to_hdf(hdffile, key='distparams', mode='w') data.append(thisdata) return util.timefilter(data, starttime, endtime)
def mag15s(probe, starttime, endtime, verbose=False): """ Import 15s cadence magnetic field data. Parameters ---------- probe : string Probe number. starttime : datetime Start of interval. endtime : datetime End of interval. verbose : bool, optional If ``True``, print information whilst loading. Default is ``False``. Returns ------- data : DataFrame Requested data. """ data = [] dtimes = util._daysplitinterval(starttime, endtime) # Loop through years for dtime in dtimes: startdt = dtime[0] year = startdt.year doy = util.dtime2doy(startdt) if verbose: print('Loading IMP 15s mag probe {}, {:03d}/{}'.format( probe, doy, year)) filename = '{}{:03d}_imp{}_mag_15s_v3.asc'.format(year, doy, probe) hdffname = filename[:-3] + 'hdf' # Location of file relative to local directory or remote url relative_loc = os.path.join('imp{}'.format(probe), 'mag', '15s_ascii_v3', str(year)) local_dir = os.path.join(imp_dir, relative_loc) hdffile = os.path.join(local_dir, hdffname) if os.path.exists(hdffile): thisdata = pd.read_hdf(hdffile) data.append(thisdata) continue remote_url = imp_url + relative_loc f = util.load(filename, local_dir, remote_url) readargs = { 'names': [ 'Year', 'doy', 'Second', 'Source flag', 'n points', 'x gse', 'y gse', 'z gse', 'y gsm', 'z gsm', '|B|', 'Bx gse', 'By gse', 'Bz gse', 'By gsm', 'Bz gsm', 'Bxx gse', 'Byy gse', 'Bzz gse', 'Byx gse', 'Bzx gse', 'Bzy gse', 'Time shift', 'sw flag' ], 'na_values': [ '9999', '999', '99', '9', '999', '99.99', '99.99', '99.99', '99.99', '99.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '999.9', '9' ], 'delim_whitespace': True } # Read in data thisdata = pd.read_table(f, **readargs) thisdata['Time'] = (pd.to_datetime(thisdata['Year'], format='%Y') + pd.to_timedelta(thisdata['doy'] - 1, unit='d') + pd.to_timedelta(thisdata['Second'], unit='s')) thisdata = thisdata.set_index('Time', drop=False) thisdata = thisdata.drop(['Year', 'doy', 'Second'], 1) if use_hdf: thisdata.to_hdf(hdffile, key='distparams', mode='w') data.append(thisdata) return util.timefilter(data, starttime, endtime)
def swoops_ions(starttime, endtime): """ Import SWOOPS ion data. Parameters ---------- starttime : datetime Start of interval endtime : datetime End of interval Returns ------- data : DataFrame Requested data """ swoops_options = url_options readargs = { 'names': [ 'year', 'doy', 'hour', 'minute', 'second', 'r', 'hlat', 'hlon', 'n_p', 'n_a', 'T_p_large', 'T_p_small', 'v_r', 'v_t', 'v_n', 'iqual' ], 'delim_whitespace': True } data = [] months_loaded = [] dtimes = util._daysplitinterval(starttime, endtime) # Loop through individual days for dtime in dtimes: thisdate = dtime[0] # Get first day of the month first_day = date(thisdate.year, thisdate.month, 1) # Check if this month's data already loaded if first_day in months_loaded: continue doy = first_day.strftime('%j') swoops_options['FILE_NAME'] = ('u' + first_day.strftime('%y') + doy + 'bam.dat') swoops_options['FILE_PATH'] =\ ('/ufa/stageIngestArea/swoops/ions/bamion' + first_day.strftime('%y') + '.zip_files') # Put together url for this days data remote_url = ulysses_url for key in swoops_options: remote_url += key + '=' + swoops_options[key] + '&' # Local locaiton to download to local_dir = os.path.join(ulysses_dir, 'swoops', 'ions', first_day.strftime('%Y')) # Load data try: f = util.load(swoops_options['FILE_NAME'], local_dir, remote_url) except HTTPError: print('No SWOOPS ion data available for date %s' % first_day) continue # Read in data thisdata = pd.read_table(f, **readargs) # Process data/time thisdata = _convert_ulysses_time(thisdata) data.append(thisdata) months_loaded.append(first_day) return util.timefilter(data, starttime, endtime)
def fgm(probe, rate, coords, starttime, endtime): """ Import fgm magnetic field data from THEMIS. Parameters ---------- probe : string Alowed values are [a, b, c, d, e]. rate : string Date rate to return. Allowed values are [e, h, l, s]. coords : string Magnetic field co-ordinate system. Allowed values are [dsl, gse, gsm, ssl]. NOTE: Add link to co-ordinate system descriptions. starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame """ valid_rates = ['e', 'h', 'l', 's'] valid_coords = ['dsl', 'gse', 'gsm', 'ssl'] _validate_probe(probe) if rate not in valid_rates: raise ValueError(('rate argument %s is not in list of allowed' 'rates: %s') % (rate, valid_rates)) if coords not in valid_coords: raise ValueError(('coords argument %s is not in list of allowed' 'co-ordinate systems: %s') % (rate, valid_rates)) # Directory relative to main THEMIS data directory relative_dir = os.path.join('th' + probe, 'l2', 'fgm') daylist = util._daysplitinterval(starttime, endtime) data = [] for day in daylist: date = day[0] this_relative_dir = os.path.join(relative_dir, str(date.year)) filename = 'th' + probe + '_l2_fgm_' +\ str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) +\ '_v01.cdf' # Absolute path to local directory for this data file local_dir = os.path.join(themis_dir, this_relative_dir) util._checkdir(local_dir) remote_url = remote_themis_dir + this_relative_dir cdf = util.load(filename, local_dir, remote_url) if cdf is None: print('File {}/{} not available\n'.format(remote_url, filename)) continue probestr = 'th' + probe ratestr = '_fg' + rate + '_' keys = { probestr + ratestr + 'btotal': '|B|', probestr + ratestr + coords: ['Bx_' + coords, 'By_' + coords, 'Bz_' + coords], probestr + ratestr + 'time': 'Time' } df = util.cdf2df(cdf, probestr + ratestr + 'time', keys, dtimeindex=False) df = df.set_index(pd.to_datetime(df.index.values, unit='s')) df['Time'] = df.index.values data.append(df) data = pd.concat(data) data = data[(data['Time'] > starttime) & (data['Time'] < endtime)] return data