def mag_hires(starttime, endtime): """ Import high resolution magnetic field from Cassini. See http://pds-ppi.igpp.ucla.edu/search/view/?f=yes&id=pds://PPI/CO-E_SW_J_S-MAG-3-RDR-FULL-RES-V1.0 for more information. Cassini Orbiter Magnetometer Calibrated MAG data at the highest time resolution available covering the period 1999-08-16 (DOY 228) to 2016-12-31 (DOY 366). The data are in RTN coordinates prior Cassini's arrival at Saturn, and Kronographic (KRTP) coordinates at Saturn (beginning 2004-05-14, DOY 135). Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame Requested data """ base_url = ('http://pds-ppi.igpp.ucla.edu/ditdos/download?id=' 'pds://PPI/CO-E_SW_J_S-MAG-3-RDR-FULL-RES-V1.0/DATA') daylist = util._daysplitinterval(starttime, endtime) data = [] for [day, stime, etime] in daylist: year = day.year url = '{}/{}'.format(base_url, year) if calendar.isleap(year): monthstr = leapmonth2str[day.month] else: monthstr = month2str[day.month] url = '{}/{}'.format(url, monthstr) doy = day.strftime('%j') local_dir = cassini_dir / 'mag' / 'hires' / str(year) # No way to work out co-ordinates, so guess Kronian and then RTN try: coords = 'KRTP' df = _mag_hires_helper(year, doy, local_dir, url, coords) except RuntimeError: try: coords = 'RTN' df = _mag_hires_helper(year, doy, local_dir, url, coords) except RuntimeError: continue df['coords'] = coords data.append(df) return util.timefilter(data, starttime, endtime)
def _load(probe, starttime, endtime, instrument, product_id, cdfkeys): daylist = util._daysplitinterval(starttime, endtime) data = [] for day in daylist: date = day[0] year = str(date.year) month = str(date.month).zfill(2) day = str(date.day).zfill(2) local_dir = cluster_dir / ('c' + probe) / instrument / year local_fname = 'C' + probe + '_' + product_id + '__' +\ year + month + day + '.cdf' local_file = local_dir / local_fname # If we don't have local file download it if not local_file.exists(): thisstart = datetime.combine(date, time.min) thisend = datetime.combine(date, time.max) try: _download(probe, thisstart, thisend, instrument, product_id) except Exception as err: print(str(err), '\n') continue from pycdf import pycdf cdf = pycdf.CDF(os.path.join(local_dir, local_fname)) for key, value in cdfkeys.items(): if value == 'Time': index_key = key break data.append(util.cdf2df(cdf, index_key, cdfkeys)) if len(data) == 0: raise RuntimeError('No data available to download during requested ' 'times') return util.timefilter(data, starttime, endtime)
def fgm_hires(starttime, endtime): """ Import high resolution fluxgate magnetometer data. Parameters ---------- starttime : datetime Start of interval endtime : datetime End of interval Returns ------- data : DataFrame Requested data """ fgm_options = url_options readargs = { 'names': ['year', 'doy', 'hour', 'minute', 'second', 'Bx', 'By', 'Bz', '|B|'], 'delim_whitespace': True } data = [] dtimes = util._daysplitinterval(starttime, endtime) # Loop through years for dtime in dtimes: date = dtime[0] yearstr = date.strftime('%Y') fgm_options['FILE_NAME'] = ('U' + yearstr[-2:] + date.strftime('%j') + 'SH.ASC') # Local locaiton to download to local_dir = ulysses_dir / 'fgm' / 'hires' / yearstr local_file = local_dir / fgm_options['FILE_NAME'] local_hdf = local_file.with_suffix('.hdf') # If we have already saved a hdf file if local_hdf.exists(): thisdata = pd.read_hdf(local_hdf) else: # Put together remote url fgm_options['FILE_PATH'] = '/ufa/HiRes/VHM-FGM/' + yearstr remote_url = ulysses_url for key in fgm_options: remote_url += key + '=' + fgm_options[key] + '&' f = util.load(fgm_options['FILE_NAME'], local_dir, remote_url) # Read in data thisdata = pd.read_table(f, **readargs) # Process data/time thisdata = _convert_ulysses_time(thisdata) if use_hdf: thisdata.to_hdf(local_hdf, 'fgm_hires') data.append(thisdata) return util.timefilter(data, starttime, endtime)
def fgm_survey(probe, starttime, endtime): """ Import fgm survey mode magnetic field data. Parameters ---------- probe : string Probe number, must be 1, 2, 3, or 4 starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame Imported data. """ # Directory relative to main MMS data directory relative_dir = os.path.join('mms' + probe, 'fgm', 'srvy', 'l2') daylist = util._daysplitinterval(starttime, endtime) data = [] for day in daylist: date = day[0] this_relative_dir = os.path.join(relative_dir, str(date.year), str(date.month).zfill(2)) filename = 'mms{}_fgm_srvy_l2_{}{:02}{:02}_v4.18.0.cdf'.format( probe, date.year, date.month, date.day) # Absolute path to local directory for this data file local_dir = os.path.join(mms_dir, this_relative_dir) util._checkdir(local_dir) remote_url = remote_mms_dir + this_relative_dir # Load cdf file cdf = util.load(filename, local_dir, remote_url) # Convert cdf to dataframe keys = { 'mms' + probe + '_fgm_b_gsm_srvy_l2': ['Bx', 'By', 'Bz', 'Br'], 'Epoch': 'Time' } df = util.cdf2df(cdf, 'Epoch', keys) data.append(df) return util.timefilter(data, starttime, endtime)
def _swics(starttime, endtime, names, product): swics_options = url_options readargs = {'names': names, 'delim_whitespace': True} data = [] dtimes = util._daysplitinterval(starttime, endtime) # Loop through years for year in range(starttime.year, endtime.year + 1): swics_options['FILE_NAME'] = '{}{}.dat'.format(product, str(year)[-2:]) # Local locaiton to download to local_dir = os.path.join(ulysses_dir, 'swics') local_file = os.path.join(local_dir, swics_options['FILE_NAME']) local_hdf = local_file[:-4] + '.hdf' # If we have already saved a hdf file if os.path.exists(local_hdf): thisdata = pd.read_hdf(local_hdf) else: # Put together remote url swics_options['FILE_PATH'] = '/ufa/HiRes/data/swics' remote_url = ulysses_url for key in swics_options: remote_url += key + '=' + swics_options[key] + '&' f = util.load(swics_options['FILE_NAME'], local_dir, remote_url) if f is None: print('File {}/{} not available\n'.format( remote_url, filename)) continue # Read in data thisdata = pd.read_table(f, **readargs) # Process data/time thisdata = _convert_ulysses_time(thisdata) if use_hdf: thisdata.to_hdf(local_hdf, 'swics') data.append(thisdata) return util.timefilter(data, starttime, endtime)
def integrated_dists(probe, starttime, endtime, verbose=False): """ Returns the integrated distributions from experiments i1a and i1b in Helios distribution function files. The distributions are integrated over all angles and given as a function of proton velocity. Parameters ---------- probe : int Helios probe to import data from. Must be 1 or 2. starttime : datetime Start of interval endtime : datetime End of interval verbose : bool, optional If ``True``, print information whilst loading. Default is ``False``. Returns ------- distinfo : Series Infromation stored in the top of distribution function files. """ extensions = ['hdm.0', 'hdm.1', 'ndm.0', 'ndm.1'] distlist = {'a': [], 'b': []} starttime_orig = starttime # Loop through each day while starttime < endtime: year = starttime.year doy = starttime.strftime('%j') # Directory for today's distribution files dist_dir = _dist_file_dir(probe, year, doy) # Locaiton of hdf file to save to/load from hdffile = 'h' + probe + str(year) + str(doy).zfill(3) +\ 'integrated_dists.hdf' hdffile = os.path.join(dist_dir, hdffile) todays_dists = {'a': [], 'b': []} # Check if data is already saved if os.path.isfile(hdffile): for key in todays_dists: todays_dists[key] = pd.read_hdf(hdffile, key=key) distlist[key].append(todays_dists[key]) starttime += timedelta(days=1) continue # If not saved, generate a derived file else: # Get every distribution function file present for this day for f in os.listdir(dist_dir): path = os.path.join(dist_dir, f) # Check for distribution function if path[-5:] in extensions: hour, minute, second = _dist_filename_to_hms(path) try: a, b = integrated_dists_single(probe, year, doy, hour, minute, second) except RuntimeError as err: strerr = 'No ion distribution function data in file' if str(err) == strerr: continue raise err t = datetime.combine(starttime.date(), time(hour, minute, second)) if verbose: print(t) dists = {'a': a, 'b': b} for key in dists: dist = dists[key] dist['Time'] = t dist = dist.set_index(['Time', 'v'], drop=True) todays_dists[key].append(dist) # Go through a and b and concat all the data for key in todays_dists: todays_dists[key] = pd.concat(todays_dists[key]) if use_hdf: todays_dists[key].to_hdf(hdffile, key=key, mode='a') distlist[key].append(todays_dists[key]) starttime += timedelta(days=1) for key in distlist: distlist[key] = util.timefilter(distlist[key], starttime_orig, endtime) return distlist
def mag_rtn(starttime, endtime): """ Import magnetic field in RTN coordinates from Messenger. Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame """ # Directory relative to main WIND data directory relative_dir = 'rtn' daylist = util._daysplitinterval(starttime, endtime) data = [] for day in daylist: date = day[0] this_relative_dir = os.path.join(relative_dir, str(date.year)) hdffile = 'messenger_mag_rtn_' +\ str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) +\ '_v01.hdf' hdfloc = os.path.join(mess_dir, this_relative_dir, hdffile) # Try to load hdf file if os.path.isfile(hdfloc): df = pd.read_hdf(hdfloc) data.append(df) continue filename = hdffile[:-4] + '.cdf' # Absolute path to local directory for this data file local_dir = os.path.join(mess_dir, this_relative_dir) util._checkdir(local_dir) remote_url = os.path.join(remote_mess_dir, this_relative_dir) cdf = util.load(filename, local_dir, remote_url, guessversion=True) if cdf is None: print('File {}/{} not available\n'.format(remote_url, filename)) continue keys = { 'B_normal': 'Bn', 'B_radial': 'Br', 'B_tangential': 'Bt', 'Epoch': 'Time', 'azimuth_ecliptic': 'sc_Az', 'latitude_ecliptic': 'sc_Lat', 'radialDistance': 'sc_r', 'MissionElapsedTime': 'mission_time' } df = util.cdf2df(cdf, index_key='Epoch', keys=keys) if use_hdf: hdffile = filename[:-4] + '.hdf' df.to_hdf(hdfloc, key='data', mode='w') data.append(df) return util.timefilter(data, starttime, endtime)
def mag_1min(starttime, endtime, coords): """ Import 1 minute magnetic field from Cassini. See http://pds-ppi.igpp.ucla.edu/search/view/?f=yes&id=pds://PPI/CO-E_SW_J_S-MAG-4-SUMM-1MINAVG-V1.0 for more information. Cassini Orbiter Magnetometer Calibrated MAG data in 1 minute averages available covering the period 1999-08-16 (DOY 228) to 2016-12-31 (DOY 366). The data are provided in RTN coordinates throughout the mission, with Earth, Jupiter, and Saturn centered coordinates for the respective flybys of those planets. Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. coords : strings Requested coordinate system. Must be one of ``['KRTP', 'KSM', 'KSO', 'RTN']`` Returns ------- data : DataFrame Requested data """ valid_coords = ['KRTP', 'KSM', 'KSO', 'RTN'] if coords not in valid_coords: raise ValueError('coords must be one of {}'.format(valid_coords)) base_url = ('http://pds-ppi.igpp.ucla.edu/ditdos/download?' 'id=pds://PPI/CO-E_SW_J_S-MAG-4-SUMM-1MINAVG-V1.0/DATA') data = [] for year in starttime.year, endtime.year: url = '{}/{}'.format(base_url, year) local_dir = os.path.join(cassini_dir, 'mag', '1min') fname = '{}_FGM_{}_1M'.format(year, coords) hdfloc = os.path.join(local_dir, fname + '.hdf') if os.path.isfile(hdfloc): df = pd.read_hdf(hdfloc) data.append(df) continue f = util.load(fname + '.TAB', local_dir, url) if 'error_message' in f.readline(): f.close() os.remove(os.path.join(local_dir, fname + '.TAB')) continue df = pd.read_table(f, names=[ 'Time', 'Bx', 'By', 'Bz', '|B|', 'X', 'Y', 'Z', 'Local hour', 'n points' ], delim_whitespace=True, parse_dates=[0], index_col=0) data.append(df) if use_hdf: df.to_hdf(hdfloc, key='data', mode='w') return util.timefilter(data, starttime, endtime)
def merged(probe, starttime, endtime, verbose=False): """ Import merged plasma data. See ftp://cdaweb.gsfc.nasa.gov/pub/data/imp/imp8/merged/00readme.txt for information on variables. Parameters ---------- probe : string Probe number. starttime : datetime Start of interval. endtime : datetime End of interval. verbose : bool, optional If ``True``, print information whilst loading. Default is ``False``. Returns ------- data : DataFrame Requested data. """ _check_probe(probe, ['8']) data = [] startyear = starttime.year endyear = endtime.year # Loop through years for year in range(startyear, endyear + 1): if year == startyear: startmonth = starttime.month else: startmonth = 1 if year == endyear: endmonth = endtime.month else: endmonth = 12 # Loop through months for month in range(startmonth, endmonth + 1): if verbose: print('Loading IMP merged probe {}, {:02d}/{}'.format( probe, month, year)) intervalstring = str(year) + str(month).zfill(2) filename = 'imp_min_merge' + intervalstring + '.asc' # Location of file relative to local directory or remote url relative_loc = os.path.join('imp' + probe, 'merged') local_dir = os.path.join(imp_dir, relative_loc) hdffile = os.path.join(local_dir, filename[:-4] + '.hdf') if os.path.isfile(hdffile): data.append(pd.read_hdf(hdffile)) continue remote_url = imp_url + relative_loc f = util.load(filename, local_dir, remote_url) if f is None: print('File {}/{} not available\n'.format( remote_url, filename)) continue readargs = { 'names': [ 'Year', 'doy', 'Hour', 'Minute', 'sw_flag', 'x_gse', 'y_gse', 'z_gse', 'y_gsm', 'z_gsm', 'Nm', 'FCm', 'DWm', '<|B|>', '|<B>|', '<B_lat>', '<B_long>', 'Bx_gse', 'By_gse', 'Bz_gse', 'By_gsm', 'Bz_gsm', 'sigma|B|', 'sigma B', 'sigma B_x', 'sigma B_y', 'sigma B_z', 'plas_reg', 'Npp', 'FCp', 'DWp', 'v_fit', 'vx_fit_gse', 'vy_fit_gse', 'vz_fit_gse', 'vlong_fit', 'vlat_fit', 'np_fit', 'Tp_fit', 'v_mom', 'vx_mom_gse', 'vy_mom_gse', 'vz_mom_gse', 'vlong_mom', 'vlat_mom', 'np_mom', 'Tp_mom' ], 'na_values': [ '9999', '999', '99', '99', '9', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9', '99', '9.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9', '9', '99', '9.99', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999999.', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999999.' ], 'delim_whitespace': True } # Read in data thisdata = pd.read_table(f, **readargs) thisdata['Time'] = ( pd.to_datetime(thisdata['Year'], format='%Y') + pd.to_timedelta(thisdata['doy'] - 1, unit='d') + pd.to_timedelta(thisdata['Hour'], unit='h') + pd.to_timedelta(thisdata['Minute'], unit='m')) if use_hdf: thisdata.to_hdf(hdffile, key='distparams', mode='w') data.append(thisdata) return util.timefilter(data, starttime, endtime)
def merged(probe, starttime, endtime, verbose=True, try_download=True): """ Read in merged data set. Parameters ---------- probe : int, string Helios probe to import data from. Must be 1 or 2. starttime : datetime Interval start time endtime : datetime Interval end time verbose : bool, optional If ``True``, print information as data is loading. Default is ``True``. Returns ------- data : DataFrame Merged data set Notes ----- This is an old dataset, and it is recommended to use `corefit` instead. """ probe = _check_probe(probe) daylist = util._daysplitinterval(starttime, endtime) data = [] floc = _merged_localdir(probe) for day in daylist: this_date = day[0] # Check that data for this day exists if probe == '1': if this_date < date(1974, 12, 12) or this_date > date(1985, 9, 4): continue if probe == '2': if this_date < date(1976, 1, 17) or this_date > date(1980, 3, 8): continue doy = int(this_date.strftime('%j')) year = this_date.year hdfloc = os.path.join(floc, _merged_fname(probe, year, doy) + '.hdf') # Data not processed yet, try to process and load it if not os.path.isfile(hdfloc): try: data.append( _merged_fromascii(probe, year, doy, try_download=try_download)) if verbose: print(year, doy, 'Processed ascii file') except (FileNotFoundError, URLError) as err: if verbose: print(str(err)) print(year, doy, 'No raw merged data') else: # Load data from already processed file data.append(pd.read_hdf(hdfloc, 'table')) if verbose: print(year, doy) if data == []: fmt = '%d-%m-%Y' raise ValueError('No data to import for probe ' + probe + ' between ' + starttime.strftime(fmt) + ' and ' + endtime.strftime(fmt)) return util.timefilter(data, starttime, endtime)
def ion_dists(probe, starttime, endtime, remove_advect=False, verbose=False): """ Return 3D ion distributions between *starttime* and *endtime* Parameters ---------- probe : int Helios probe to import data from. Must be 1 or 2. starttime : datetime Start of interval endtime : datetime End of interval remove_advect : bool, optional If *False*, the distribution is returned in the spacecraft frame. If *True*, the distribution is returned in the solar wind frame, by subtracting the spacecraft velocity from the velcoity of each bin. Note this significantly slows down reading in the distribution. verbose : bool, optional If ``True``, print dates when loading files. Default is ``False``. Returns ------- distinfo : Series Infromation stored in the top of distribution function files. """ extensions = ['hdm.0', 'hdm.1', 'ndm.0', 'ndm.1'] distlist = [] # Loop through each day starttime_orig = starttime while starttime < endtime: year = starttime.year doy = int(starttime.strftime('%j')) if verbose: print('Loading ion dists from year', year, 'doy', doy) # Directory for today's distribution files dist_dir = _dist_file_dir(probe, year, doy) # If directory doesn't exist, print error and continue if not os.path.exists(dist_dir): print('No ion distributions available for year', year, 'doy', doy) starttime += timedelta(days=1) continue # Locaiton of hdf file to save to/load from hdffile = 'h' + probe + str(year) + str(doy).zfill(3) +\ 'ion_dists.hdf' hdffile = os.path.join(dist_dir, hdffile) if os.path.isfile(hdffile): todays_dist = pd.read_hdf(hdffile) distlist.append(todays_dist) starttime += timedelta(days=1) continue todays_dist = [] # Get every distribution function file present for this day for f in os.listdir(dist_dir): path = os.path.join(dist_dir, f) # Check for distribution function if path[-5:] in extensions: hour, minute, second = _dist_filename_to_hms(path) try: d = ion_dist_single(probe, year, doy, hour, minute, second) except RuntimeError as err: strerr = 'No ion distribution function data in file' if str(err) == strerr: continue raise err t = datetime.combine(starttime.date(), time(hour, minute, second)) d['Time'] = t if verbose: print(t) todays_dist.append(d) if todays_dist == []: starttime += timedelta(days=1) continue todays_dist = pd.concat(todays_dist) todays_dist = todays_dist.set_index('Time', append=True) if use_hdf: todays_dist.to_hdf(hdffile, key='ion_dist', mode='w') distlist.append(todays_dist) starttime += timedelta(days=1) if distlist == []: raise RuntimeError('No data available for times ' + str(starttime_orig) + ' to ' + str(endtime)) return util.timefilter(distlist, starttime_orig, endtime)
def distparams(probe, starttime, endtime, verbose=False): """ Read in distribution parameters found in the header of distribution files. Parameters ---------- probe : int Helios probe to import data from. Must be 1 or 2. starttime : datetime Start of interval endtime : datetime End of interval verbose : bool, optional If ``True``, print information whilst loading. Default is ``False``. Returns ------- distinfo : Series Infromation stored in the top of distribution function files """ extensions = ['hdm.0', 'hdm.1', 'ndm.0', 'ndm.1'] paramlist = [] starttime_orig = starttime # Loop through each day while starttime < endtime: year = starttime.year doy = starttime.strftime('%j') # Directory for today's distribution files dist_dir = _dist_file_dir(probe, year, doy) # Locaiton of hdf file to save to/load from hdffile = 'h' + probe + str(year) + str(doy).zfill(3) +\ 'distparams.hdf' hdffile = os.path.join(dist_dir, hdffile) if os.path.isfile(hdffile): todays_params = pd.read_hdf(hdffile) elif not os.path.isdir(dist_dir): starttime += timedelta(days=1) continue else: todays_params = [] # Get every distribution function file present for this day for f in os.listdir(dist_dir): path = os.path.join(dist_dir, f) # Check for distribution function if path[-5:] in extensions: hour, minute, second = _dist_filename_to_hms(path) if verbose: print(starttime.date(), hour, minute, second) p = distparams_single(probe, year, doy, hour, minute, second) todays_params.append(p) todays_params = pd.concat(todays_params, ignore_index=True, axis=1).T todays_params = todays_params.set_index('Time', drop=False) # Convert columns to numeric types todays_params = todays_params.apply(pd.to_numeric, errors='ignore') if use_hdf: todays_params.to_hdf(hdffile, key='distparams', mode='w') paramlist.append(todays_params) starttime += timedelta(days=1) return util.timefilter(paramlist, starttime_orig, endtime)
def fpi_dis_moms(probe, mode, starttime, endtime): """ Import fpi ion distribution moment data. Parameters ---------- probe : string Probe number, must be 1, 2, 3, or 4 mode : string Data mode, must be 'fast' or 'brst' starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame Imported data. """ valid_modes = ['fast', 'brst'] if mode not in valid_modes: raise RuntimeError('Mode must be either fast or brst') # Directory relative to main MMS data directory relative_dir = os.path.join('mms' + probe, 'fpi', mode, 'l2', 'dis-moms') daylist = util._daysplitinterval(starttime, endtime) data = [] for day in daylist: date = day[0] starthour = day[1].hour endhour = day[2].hour + 1 # fips fast data product has files every two hours, so get nearest two # hour stamps starthour -= np.mod(starthour, 2) endhour += np.mod(endhour, 2) for h in range(starthour, endhour, 2): this_relative_dir = os.path.join(relative_dir, str(date.year), str(date.month).zfill(2)) filename = ('mms{}_fpi_{}_l2_dis-moms_' '{}{:02}{:02}{:02}0000_v3.3.0.cdf').format( probe, mode, date.year, date.month, date.day, h) # Absolute path to local directory for this data file local_dir = os.path.join(mms_dir, this_relative_dir) util._checkdir(local_dir) remote_url = remote_mms_dir + this_relative_dir # Load cdf file try: cdf = util.load(filename, local_dir, remote_url) except urllib.error.HTTPError as e: if str(e) == 'HTTP Error 404: Not Found': print('No data available for hours', str(h) + '-' + str(h + 2), 'on', date.strftime('%d/%m/%Y')) continue else: raise probestr = 'mms' + probe + '_' # Convert cdf to dataframe keys = {'Epoch': 'Time', probestr + 'dis_bulkv_gse_fast': ['bulkv_x', 'bulkv_y', 'bulkv_z'], probestr + 'dis_heatq_gse_fast': ['heatq_x', 'heatq_y', 'heatq_z'], probestr + 'dis_numberdensity_fast': 'n', probestr + 'dis_temppara_fast': 'T_par', probestr + 'dis_tempperp_fast': 'T_perp'} df = util.cdf2df(cdf, 'Epoch', keys) data.append(df) return util.timefilter(data, starttime, endtime)
def threedp_pm(starttime, endtime): """ Import 'pm' wind data. 3 second time resolution solar wind proton and alpha particle moments from the PESA LOW sensor, computed on-board the spacecraft Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame """ # Directory relative to main WIND data directory relative_dir = path.Path('3dp') / '3dp_pm' daylist = util._daysplitinterval(starttime, endtime) data = [] for day in daylist: date = day[0] this_relative_dir = relative_dir / str(day[0].year) # Absolute path to local directory for this data file local_dir = wind_dir / this_relative_dir filename = 'wi_pm_3dp_' +\ str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) +\ '_v05.cdf' hdfname = filename[:-4] + 'hdf' hdfloc = local_dir / hdfname if hdfloc.exists(): df = pd.read_hdf(hdfloc) data.append(df) continue util._checkdir(local_dir) remote_url = remote_wind_dir + str(this_relative_dir) cdf = util.load(filename, local_dir, remote_url, guessversion=True) if cdf is None: print('File {}/{} not available\n'.format(remote_url, filename)) continue keys = { 'A_DENS': 'n_a', 'A_TEMP': 'T_a', 'A_VELS': ['va_x', 'va_y', 'va_z'], 'P_DENS': 'n_p', 'P_TEMP': 'T_p', 'P_VELS': ['vp_x', 'vp_y', 'vp_z'], 'Epoch': 'Time' } df = util.cdf2df(cdf, index_key='Epoch', keys=keys) if use_hdf: df.to_hdf(hdfloc, 'pm', mode='w') data.append(df) return util.timefilter(data, starttime, endtime)
def mag_4hz(probe, starttime, endtime, verbose=True, try_download=True): """ Read in 4Hz magnetic field data. Parameters ---------- probe : int, string Helios probe to import data from. Must be 1 or 2. starttime : datetime Interval start time endtime : datetime Interval end time verbose : bool, optional If ``True``, print more information as data is loading. Default is ``True``. try_download : bool, optional If ``False`` don't try to download data if it is missing locally. Default is ``False``. Returns ------- data : DataFrame 4Hz magnetic field data set """ probe = _check_probe(probe) data = [] # Loop through years for year in range(starttime.year, endtime.year + 1): floc = _4hz_localdir(probe) # Calculate start day of year if year == starttime.year: startdoy = int(starttime.strftime('%j')) else: startdoy = 1 # Calculate end day of year if year == endtime.year: enddoy = int(endtime.strftime('%j')) else: enddoy = 366 # Loop through days of year for doy in range(startdoy, enddoy + 1): hdfloc = os.path.join(floc, _4hz_filename(probe, year, doy) + '.hdf') if not os.path.isfile(hdfloc): # Data not processed yet, try to process and load it try: data.append( _fourHz_fromascii(probe, year, doy, try_download=try_download)) except ValueError as err: if 'No mag data available' in str(err): if verbose: print('{}/{:03} ' '4Hz mag data not available'.format( year, doy)) else: raise else: # Load data from already processed file data.append(pd.read_hdf(hdfloc, 'table')) if verbose: print('{}/{:03} 4Hz mag data loaded'.format(year, doy)) if data == []: raise ValueError('No raw 4Hz mag data available') data = util.timefilter(data, starttime, endtime) if data.empty: raise ValueError('No 4Hz raw mag data available for entire interval') return (data)
def mag15s(probe, starttime, endtime, verbose=False): """ Import 15s cadence magnetic field data. Parameters ---------- probe : string Probe number. starttime : datetime Start of interval. endtime : datetime End of interval. verbose : bool, optional If ``True``, print information whilst loading. Default is ``False``. Returns ------- data : DataFrame Requested data. """ data = [] dtimes = util._daysplitinterval(starttime, endtime) # Loop through years for dtime in dtimes: startdt = dtime[0] year = startdt.year doy = util.dtime2doy(startdt) if verbose: print('Loading IMP 15s mag probe {}, {:03d}/{}'.format( probe, doy, year)) filename = '{}{:03d}_imp{}_mag_15s_v3.asc'.format(year, doy, probe) hdffname = filename[:-3] + 'hdf' # Location of file relative to local directory or remote url relative_loc = os.path.join('imp{}'.format(probe), 'mag', '15s_ascii_v3', str(year)) local_dir = os.path.join(imp_dir, relative_loc) hdffile = os.path.join(local_dir, hdffname) if os.path.exists(hdffile): thisdata = pd.read_hdf(hdffile) data.append(thisdata) continue remote_url = imp_url + relative_loc f = util.load(filename, local_dir, remote_url) readargs = { 'names': [ 'Year', 'doy', 'Second', 'Source flag', 'n points', 'x gse', 'y gse', 'z gse', 'y gsm', 'z gsm', '|B|', 'Bx gse', 'By gse', 'Bz gse', 'By gsm', 'Bz gsm', 'Bxx gse', 'Byy gse', 'Bzz gse', 'Byx gse', 'Bzx gse', 'Bzy gse', 'Time shift', 'sw flag' ], 'na_values': [ '9999', '999', '99', '9', '999', '99.99', '99.99', '99.99', '99.99', '99.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '999.9', '9' ], 'delim_whitespace': True } # Read in data thisdata = pd.read_table(f, **readargs) thisdata['Time'] = (pd.to_datetime(thisdata['Year'], format='%Y') + pd.to_timedelta(thisdata['doy'] - 1, unit='d') + pd.to_timedelta(thisdata['Second'], unit='s')) thisdata = thisdata.set_index('Time', drop=False) thisdata = thisdata.drop(['Year', 'doy', 'Second'], 1) if use_hdf: thisdata.to_hdf(hdffile, key='distparams', mode='w') data.append(thisdata) return util.timefilter(data, starttime, endtime)
def swoops_ions(starttime, endtime): """ Import SWOOPS ion data. Parameters ---------- starttime : datetime Start of interval endtime : datetime End of interval Returns ------- data : DataFrame Requested data """ swoops_options = url_options readargs = { 'names': [ 'year', 'doy', 'hour', 'minute', 'second', 'r', 'hlat', 'hlon', 'n_p', 'n_a', 'T_p_large', 'T_p_small', 'v_r', 'v_t', 'v_n', 'iqual' ], 'delim_whitespace': True } data = [] months_loaded = [] dtimes = util._daysplitinterval(starttime, endtime) # Loop through individual days for dtime in dtimes: thisdate = dtime[0] # Get first day of the month first_day = date(thisdate.year, thisdate.month, 1) # Check if this month's data already loaded if first_day in months_loaded: continue doy = first_day.strftime('%j') swoops_options['FILE_NAME'] = ('u' + first_day.strftime('%y') + doy + 'bam.dat') swoops_options['FILE_PATH'] =\ ('/ufa/stageIngestArea/swoops/ions/bamion' + first_day.strftime('%y') + '.zip_files') # Put together url for this days data remote_url = ulysses_url for key in swoops_options: remote_url += key + '=' + swoops_options[key] + '&' # Local locaiton to download to local_dir = os.path.join(ulysses_dir, 'swoops', 'ions', first_day.strftime('%Y')) # Load data try: f = util.load(swoops_options['FILE_NAME'], local_dir, remote_url) except HTTPError: print('No SWOOPS ion data available for date %s' % first_day) continue # Read in data thisdata = pd.read_table(f, **readargs) # Process data/time thisdata = _convert_ulysses_time(thisdata) data.append(thisdata) months_loaded.append(first_day) return util.timefilter(data, starttime, endtime)
def threedp_sfpd(starttime, endtime): """ Import 'sfpd' wind data. 12 second energetic electron pitch-angle energy spectra from the foil SST Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame """ # Directory relative to main WIND data directory relative_dir = path.Path('3dp') / '3dp_sfpd' daylist = util._daysplitinterval(starttime, endtime) data = [] mag = [] for (date, _, _) in daylist: this_relative_dir = relative_dir / str(date.year) # Absolute path to local directory for this data file local_dir = wind_dir / this_relative_dir filename = 'wi_sfpd_3dp_{:{dfmt}}_v02'.format(date, dfmt='%Y%m%d') hdfname = filename + '.hdf' hdfloc = local_dir / hdfname if hdfloc.exists(): df = pd.read_hdf(hdfloc) data.append(df) continue util._checkdir(local_dir) remote_url = remote_wind_dir + str(this_relative_dir) cdf = util.load(filename + '.cdf', local_dir, remote_url, guessversion=True) if cdf is None: print('File {}/{} not available\n'.format(remote_url, filename)) continue data_today = [] # Loop through each timestamp to build up fluxes for i, time in enumerate(cdf['Epoch'][...]): energies = cdf['ENERGY'][i, :] angles = cdf['PANGLE'][i, :] fluxes = cdf['FLUX'][i, :, :] magfield = cdf['MAGF'][i, :] index = pd.MultiIndex.from_product( ([time], energies, angles), names=['Time', 'Energy', 'Pitch angle']) df = pd.DataFrame(fluxes.ravel(), index=index, columns=['Flux']) df['Bx'] = magfield[0] df['By'] = magfield[1] df['Bz'] = magfield[2] data_today.append(df) data_today = pd.concat(data_today) data_today = data_today.sort_index() if use_hdf: data_today.to_hdf(hdfloc, 'sfpd', mode='w') data.append(data_today) data = util.timefilter(data, starttime, endtime) data = data.reset_index(level=['Energy', 'Pitch angle']) return data
def mag_ness(probe, starttime, endtime, verbose=True, try_download=True): """ Read in 6 second magnetic field data. Parameters ---------- probe : int, string Helios probe to import data from. Must be 1 or 2. starttime : datetime Interval start time endtime : datetime Interval end time verbose : bool, optional If ``True``, print more information as data is loading. Default is ``True``. Returns ------- data : DataFrame 6 second magnetic field data set """ probe = _check_probe(probe) startdate = starttime.date() enddate = endtime.date() def _check_doy(probe, year, doy): ''' Returns False if year and doy are out of bounds for given probe ''' if probe == '1': minyear = 1974 mindoy = 349 maxyear = 1981 maxdoy = 167 elif probe == '2': minyear = 1976 mindoy = 17 maxyear = 1980 maxdoy = 68 if (year == minyear and doy < mindoy) or (year < minyear): return False elif (year == maxyear and doy > maxdoy) or (year > maxyear): return False else: return True data = [] # Loop through years for year in range(startdate.year, enddate.year + 1): floc = _ness_localdir(probe, year) # Calculate start day startdoy = 1 if year == startdate.year: startdoy = int(startdate.strftime('%j')) # Calculate end day enddoy = 366 if year == enddate.year: enddoy = int(enddate.strftime('%j')) # Loop through days of year for doy in range(startdoy, enddoy + 1): nodatastr = '{}/{:03} 6s mag data not available'.format(year, doy) datastr = '{}/{:03} 6s mag data loaded'.format(year, doy) if not _check_doy(probe, year, doy): if verbose: print(nodatastr) continue hdfloc = os.path.join(floc, _ness_fname(probe, year, doy) + 'hdf') if os.path.isfile(hdfloc): # Load data from already processed file data.append(pd.read_hdf(hdfloc, 'table')) print(datastr) continue # Data not processed yet, try to process and load it try: data.append( _mag_ness_fromascii(probe, year, doy, try_download=try_download)) if verbose: print(datastr) except ValueError: if verbose: print(nodatastr) if data == []: raise ValueError('No 6s mag data avaialble between ' '{} and {}'.format(starttime, endtime)) return util.timefilter(data, starttime, endtime)