def _process_cdas(starttime, endtime, identifier, dataset, base_dir, units=None, badvalues=None, warn_missing_units=True): """ Generic method for downloading cdas data. """ relative_dir = pathlib.Path(identifier) # Directory relative to main WIND data directory daylist = util._daysplitinterval(starttime, endtime) dirs = [] fnames = [] dates = [] extension = '.cdf' for day in daylist: date = day[0] dates.append(date) filename = '{}_{}_{}{:02}{:02}'.format( dataset, identifier, date.year, date.month, date.day) fnames.append(filename) this_relative_dir = relative_dir / str(date.year) dirs.append(this_relative_dir) def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension, date): return get_data(identifier, date) def processing_func(cdf): return util.cdf2df(cdf, index_key='Epoch', badvalues=badvalues) return util.process(dirs, fnames, extension, base_dir, '', download_func, processing_func, starttime, endtime, units=units, download_info=dates, warn_missing_units=warn_missing_units)
def _load_wind_cdf(starttime, endtime, instrument, data_product, fname, badvalues={}): relative_dir = path.Path(instrument) / data_product # Get directories and filenames dirs = [] fnames = [] daylist = util._daysplitinterval(starttime, endtime) for day in daylist: date = day[0] filename = 'wi_{}_{}{:02}{:02}_v01'.format(fname, date.year, date.month, date.day) fnames.append(filename) local_dir = relative_dir / str(date.year) dirs.append(local_dir) extension = '.cdf' local_base_dir = wind_dir remote_base_url = remote_wind_dir def download_func(remote_base_url, local_base_dir, directory, fname, extension): remote_url = '{}{}'.format(remote_base_url, directory) util.load(fname + extension, local_base_dir / directory, remote_url) def processing_func(cdf): return util.cdf2df(cdf, 'Epoch', badvalues=badvalues) return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime)
def mag_hires(starttime, endtime): """ Import high resolution magnetic field from Cassini. See http://pds-ppi.igpp.ucla.edu/search/view/?f=yes&id=pds://PPI/CO-E_SW_J_S-MAG-3-RDR-FULL-RES-V1.0 for more information. Cassini Orbiter Magnetometer Calibrated MAG data at the highest time resolution available covering the period 1999-08-16 (DOY 228) to 2016-12-31 (DOY 366). The data are in RTN coordinates prior Cassini's arrival at Saturn, and Kronographic (KRTP) coordinates at Saturn (beginning 2004-05-14, DOY 135). Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame Requested data """ base_url = ('http://pds-ppi.igpp.ucla.edu/ditdos/download?id=' 'pds://PPI/CO-E_SW_J_S-MAG-3-RDR-FULL-RES-V1.0/DATA') daylist = util._daysplitinterval(starttime, endtime) data = [] for [day, stime, etime] in daylist: year = day.year url = '{}/{}'.format(base_url, year) if calendar.isleap(year): monthstr = leapmonth2str[day.month] else: monthstr = month2str[day.month] url = '{}/{}'.format(url, monthstr) doy = day.strftime('%j') local_dir = cassini_dir / 'mag' / 'hires' / str(year) # No way to work out co-ordinates, so guess Kronian and then RTN try: coords = 'KRTP' df = _mag_hires_helper(year, doy, local_dir, url, coords) except RuntimeError: try: coords = 'RTN' df = _mag_hires_helper(year, doy, local_dir, url, coords) except RuntimeError: continue df['coords'] = coords data.append(df) return util.timefilter(data, starttime, endtime)
def _load(probe, starttime, endtime, instrument, product_id, cdfkeys): daylist = util._daysplitinterval(starttime, endtime) data = [] for day in daylist: date = day[0] year = str(date.year) month = str(date.month).zfill(2) day = str(date.day).zfill(2) local_dir = cluster_dir / ('c' + probe) / instrument / year local_fname = 'C' + probe + '_' + product_id + '__' +\ year + month + day + '.cdf' local_file = local_dir / local_fname # If we don't have local file download it if not local_file.exists(): thisstart = datetime.combine(date, time.min) thisend = datetime.combine(date, time.max) try: _download(probe, thisstart, thisend, instrument, product_id) except Exception as err: print(str(err), '\n') continue from pycdf import pycdf cdf = pycdf.CDF(os.path.join(local_dir, local_fname)) for key, value in cdfkeys.items(): if value == 'Time': index_key = key break data.append(util.cdf2df(cdf, index_key, cdfkeys)) if len(data) == 0: raise RuntimeError('No data available to download during requested ' 'times') return util.timefilter(data, starttime, endtime)
def mitplasma_h0(probe, starttime, endtime): """ Import mit h0 plasma data. Parameters ---------- probe : string Probe number. starttime : datetime Start of interval. endtime : datetime End of interval. Returns ------- data : DataFrame Requested data. """ data = [] dtimes = util._daysplitinterval(starttime, endtime) # Loop through years for dtime in dtimes: date = dtime[0] intervalstring = str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) filename = 'i' + probe + '_h0_mitplasma_' + intervalstring + '_v01.cdf' # Location of file relative to local directory or remote url relative_loc = 'imp' + probe + '/plasma_mit/mitplasma_h0/' +\ str(date.year) local_dir = os.path.join(imp_dir, relative_loc) remote_url = imp_url + relative_loc cdf = util.load(filename, local_dir, remote_url) keys = { 'EW_flowangle_best': 'EW_flowangle_best', 'EW_flowangle_mom': 'EW_flowangle_best', 'Epoch': 'Time', 'Flow_elevation_thresh': 'Flow_elevation_thresh', 'Flow_elevation_threshsp': 'Flow_elevation_threshsp', 'Region': 'Region', 'V_fit': 'V_fit', 'V_mom': 'V_fit', 'mode': 'mode', 'protonV_thermal_fit': 'protonV_thermal_fit', 'protonV_thermal_mom': 'protonV_thermal_fit', 'proton_density_fit': 'proton_density_fit', 'proton_density_mom': 'proton_density_mom', 'xyzgse': ['x_gse', 'y_gse', 'z_gse'], 'ygsm': 'ygsm', 'zgsm': 'zgsm' } thisdata = util.cdf2df(cdf, 'Epoch', keys) data.append(thisdata) data = pd.concat(data) data = data[(data['Time'] > starttime) & (data['Time'] < endtime)] data.index.name = 'Time' return data
def mag320ms(probe, starttime, endtime, try_download=True): """ Import 320ms cadence magnetic field data. Parameters ---------- probe : string Probe number. starttime : datetime Start of interval. endtime : datetime End of interval. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` Requested data. """ fnames = [] dirs = [] extension = '.cdf' dtimes = util._daysplitinterval(starttime, endtime) # Loop through years for dtime in dtimes: date = dtime[0] intervalstring = str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) filename = 'i8_320msec_mag_' + intervalstring + '_v01' fnames.append(filename) # Location of file relative to local directory or remote url relative_loc = 'imp' + probe + '/mag/mag_320msec_cdaweb/' +\ str(date.year) dirs.append(relative_loc) local_base_dir = imp_dir remote_base_url = imp_url def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): remote_url = remote_base_url + str(directory) filename = fname + extension local_dir = local_base_dir / directory util._download_remote(remote_url, filename, local_dir) def processing_func(f): thisdata = util.cdf2df(f, 'Epoch') thisdata.index.name = 'Time' return thisdata return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, try_download=try_download)
def threedp_pm(starttime, endtime): """ Import 'pm' wind data. 3 second time resolution solar wind proton and alpha particle moments from the PESA LOW sensor, computed on-board the spacecraft Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` """ # Directory relative to main WIND data directory relative_dir = path.Path('3dp') / '3dp_pm' daylist = util._daysplitinterval(starttime, endtime) ignore = ['TIME'] dirs = [] fnames = [] units = OrderedDict([('P_DENS', u.cm**-3), ('P_VELS', u.km / u.s), ('P_TEMP', u.eV), ('A_DENS', u.cm**-3), ('A_VELS', u.km / u.s), ('A_TEMP', u.eV), ('GAP', u.dimensionless_unscaled), ('E_RANGE', u.eV), ('VALID', u.dimensionless_unscaled), ('VC', u.dimensionless_unscaled), ('SPIN', u.dimensionless_unscaled)]) extension = '.cdf' for day in daylist: date = day[0] this_relative_dir = relative_dir / str(day[0].year) filename = 'wi_pm_3dp_' +\ str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) +\ '_v[0-9][0-9]' fnames.append(filename) dirs.append(this_relative_dir) local_base_dir = wind_dir remote_base_url = remote_wind_dir def download_func(*args): util._download_remote_unknown_version(*args) def processing_func(cdf): return util.cdf2df(cdf, 'Epoch', ignore=ignore) return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units)
def swe_h3(starttime, endtime): """ Import 'h3' solar wind electron data product from WIND. Electron pitch angle files providing electron fluxes at 30 directional bins relative to the instantaneous magnetic field direction at 13 different energy levels Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame """ relative_dir = os.path.join('swe', 'swe_h3') # Get directories and filenames dirs = [] fnames = [] daylist = util._daysplitinterval(starttime, endtime) for day in daylist: date = day[0] filename = 'wi_h3_swe_{}{:02}{:02}_v01'.format(date.year, date.month, date.day) fnames.append(filename) local_dir = os.path.join(relative_dir, str(date.year)) dirs.append(local_dir) extension = '.cdf' local_base_dir = wind_dir remote_base_url = remote_wind_dir def download_func(remote_base_url, local_base_dir, directory, fname, extension): remote_url = '{}{}'.format(remote_base_url, directory) util.load(fname + extension, os.path.join(local_base_dir, directory), remote_url) def processing_func(local_dir, fname): distkeys = [] for i in range(0, 13): distkeys.append('f_pitch_E' + str(i).zfill(2)) anglelabels = [] for i in range(0, 30): anglelabels.append((i + 0.5) * np.pi / 30) timekey = 'Epoch' energykey = 'Ve' cdf = util.load(fname, local_dir, '') df = util.pitchdist_cdf2df(cdf, distkeys, energykey, timekey, anglelabels) return df return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime)
def fgm_hires(starttime, endtime, try_download=True): """ Import high resolution fluxgate magnetometer data. Parameters ---------- starttime : datetime Start of interval endtime : datetime End of interval Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` Requested data """ dtimes = util._daysplitinterval(starttime, endtime) dirs = [] fnames = [] extension = '.ASC' local_base_dir = ulysses_dir / 'fgm' / 'hires' remote_base_url = ulysses_url units = OrderedDict([('Bx', u.nT), ('By', u.nT), ('Bz', u.nT), ('|B|', u.nT)]) def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): local_dir = local_base_dir / directory fgm_options = url_options fgm_options['FILE_NAME'] = fname + extension fgm_options['FILE_PATH'] = '/ufa/HiRes/VHM-FGM/' + yearstr remote_url = ulysses_url for key in fgm_options: remote_url += key + '=' + fgm_options[key] + '&' try: util._load_remote(remote_url, fname + extension, local_dir, 'ascii') except Exception as err: return def processing_func(f): readargs = {'names': ['year', 'doy', 'hour', 'minute', 'second', 'Bx', 'By', 'Bz', '|B|'], 'delim_whitespace': True} thisdata = pd.read_table(f, **readargs) thisdata = _convert_ulysses_time(thisdata) return thisdata for dtime in dtimes: date = dtime[0] yearstr = date.strftime('%Y') filename = ('U' + yearstr[-2:] + date.strftime('%j') + 'SH') fnames.append(filename) this_relative_dir = local_base_dir / yearstr dirs.append(this_relative_dir) return util.process( dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units, try_download=True)
def fgm_survey(probe, starttime, endtime): """ Import fgm survey mode magnetic field data. Parameters ---------- probe : string Probe number, must be 1, 2, 3, or 4 starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` Imported data. """ # Directory relative to main MMS data directory relative_dir = path.Path('mms' + probe) / 'fgm' / 'srvy' / 'l2' daylist = util._daysplitinterval(starttime, endtime) dirs = [] fnames = [] extension = '.cdf' units = OrderedDict([('mms{}_fgm_mode_srvy_l2'.format(probe), u.dimensionless_unscaled)]) data = [] for day in daylist: date = day[0] this_relative_dir = (relative_dir / str(date.year) / str(date.month).zfill(2)) filename = 'mms{}_fgm_srvy_l2_{}{:02}{:02}_v4.18.0'.format( probe, date.year, date.month, date.day) fnames.append(filename) dirs.append(this_relative_dir) remote_base_url = remote_mms_dir local_base_dir = mms_dir def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): remote_url = remote_base_url + str(directory) util.load(fname + extension, local_base_dir / directory, remote_url) def processing_func(cdf): df = util.cdf2df(cdf, 'Epoch') return df return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units)
def mag320ms(probe, startTime, endTime): """ Import 320ms cadence magnetic field data. Parameters ---------- probe : string Probe number. starttime : datetime Start of interval. endtime : datetime End of interval. Returns ------- data : DataFrame Requested data. """ data = [] dtimes = util._daysplitinterval(startTime, endTime) # Loop through years for dtime in dtimes: date = dtime[0] intervalstring = str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) filename = 'i8_320msec_mag_' + intervalstring + '_v01.cdf' hdffname = filename[:-3] + '.hdf' # Location of file relative to local directory or remote url relative_loc = 'imp' + probe + '/mag/mag_320msec_cdaweb/' +\ str(date.year) local_dir = os.path.join(imp_dir, relative_loc) hdffile = os.path.join(local_dir, hdffname) if os.path.exists(hdffile): thisdata = pd.read_hdf(hdffile) data.append(thisdata) continue remote_url = imp_url + relative_loc cdf = util.load(filename, local_dir, remote_url) keys = { 'B': '|B|', 'BX': 'Bx', 'BY': 'By', 'BZ': 'Bz', 'Epoch': 'Time' } thisdata = util.cdf2df(cdf, 'Epoch', keys) data.append(thisdata) if use_hdf: thisdata.to_hdf(hdffile, key='merged', mode='w') data = pd.concat(data) data = data[(data['Time'] > startTime) & (data['Time'] < endTime)] data = data.drop(columns='Time') data.index.name = 'Time' return data
def mag_rtn(starttime, endtime, try_download=True): """ Import magnetic field in RTN coordinates from Messenger. Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` """ # Directory relative to main WIND data directory relative_dir = path.Path('rtn') fnames = [] dirs = [] extension = '.cdf' data = [] ignore = ['Quality_Flag'] daylist = util._daysplitinterval(starttime, endtime) for day in daylist: date = day[0] this_relative_dir = relative_dir / str(date.year) filename = 'messenger_mag_rtn_' +\ str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) +\ '_v01' fnames.append(filename) dirs.append(this_relative_dir) local_base_dir = mess_dir remote_base_url = remote_mess_dir def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): remote_url = remote_base_url + str(directory) filename = fname + extension local_dir = local_base_dir / directory util._download_remote(remote_url, filename, local_dir) def processing_func(cdf): thisdata = util.cdf2df(cdf, 'Epoch', ignore=ignore) return thisdata return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, try_download=try_download)
def swe_h3(starttime, endtime): """ Import 'h3' solar wind electron data product from WIND. Electron pitch angle files providing electron fluxes at 30 directional bins relative to the instantaneous magnetic field direction at 13 different energy levels Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` """ warnings.warn(_deprecation_msg, DeprecationWarning) relative_dir = path.Path('swe') / 'swe_h3' # Get directories and filenames dirs = [] fnames = [] units = OrderedDict([('Angle', u .deg), ('Energy', u.eV), ('df', u.cm/u.s)]) daylist = util._daysplitinterval(starttime, endtime) for day in daylist: date = day[0] filename = 'wi_h3_swe_{}{:02}{:02}_v[0-9][0-9]'.format( date.year, date.month, date.day) fnames.append(filename) local_dir = relative_dir / str(date.year) dirs.append(local_dir) extension = '.cdf' local_base_dir = wind_dir remote_base_url = remote_wind_dir distkeys = [] for i in range(0, 13): distkeys.append('f_pitch_E' + str(i).zfill(2)) anglelabels = [] for i in range(0, 30): anglelabels.append((i + 0.5) * np.pi / 30) timekey = 'Epoch' energykey = 'Ve' def download_func(*args): util._download_remote_unknown_version(*args) def processing_func(cdf): df = util.pitchdist_cdf2df(cdf, distkeys, energykey, timekey, anglelabels) df = df.reset_index(level=['Energy', 'Angle']) return df return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units)
def _ace(starttime, endtime, instrument, product, fname, keys=None, version='01', badvalues={}): """ Generic method for downloading ACE data from cdaweb. """ # Directory relative to main WIND data directory relative_dir = os.path.join(instrument, 'level_2_cdaweb', product) daylist = util._daysplitinterval(starttime, endtime) dirs = [] fnames = [] extension = '.cdf' for day in daylist: date = day[0] filename = 'ac_{}_{}{:02}{:02}_v{}'.format(fname, date.year, date.month, date.day, version) fnames.append(filename) this_relative_dir = os.path.join(relative_dir, str(date.year)) dirs.append(this_relative_dir) def download_func(remote_base_url, local_base_dir, directory, fname, extension): def check_exists(): # Because the version might be different to the one we guess, work # out the downloaded filename for f in os.listdir(os.path.join(local_base_dir, directory)): if (f[:-6] == (fname + extension)[:-6]): # Return filename with '.cdf' stripped off the end return f[:-4] if check_exists() is not None: return check_exists() # Now load remotely util.load(fname + extension, os.path.join(local_base_dir, directory), remote_base_url + directory, guessversion=True) if check_exists() is not None: return check_exists() def processing_func(local_dir, local_fname): cdf = util.load(local_fname, local_dir, '') return util.cdf2df(cdf, index_key='Epoch', keys=keys, badvalues=badvalues) return util.process(dirs, fnames, extension, ace_dir, remote_ace_dir, download_func, processing_func, starttime, endtime)
def fgm_hires(starttime, endtime): """ Import high resolution fluxgate magnetometer data. Parameters ---------- starttime : datetime Start of interval endtime : datetime End of interval Returns ------- data : DataFrame Requested data """ fgm_options = url_options readargs = { 'names': ['year', 'doy', 'hour', 'minute', 'second', 'Bx', 'By', 'Bz', '|B|'], 'delim_whitespace': True } data = [] dtimes = util._daysplitinterval(starttime, endtime) # Loop through years for dtime in dtimes: date = dtime[0] yearstr = date.strftime('%Y') fgm_options['FILE_NAME'] = ('U' + yearstr[-2:] + date.strftime('%j') + 'SH.ASC') # Local locaiton to download to local_dir = ulysses_dir / 'fgm' / 'hires' / yearstr local_file = local_dir / fgm_options['FILE_NAME'] local_hdf = local_file.with_suffix('.hdf') # If we have already saved a hdf file if local_hdf.exists(): thisdata = pd.read_hdf(local_hdf) else: # Put together remote url fgm_options['FILE_PATH'] = '/ufa/HiRes/VHM-FGM/' + yearstr remote_url = ulysses_url for key in fgm_options: remote_url += key + '=' + fgm_options[key] + '&' f = util.load(fgm_options['FILE_NAME'], local_dir, remote_url) # Read in data thisdata = pd.read_table(f, **readargs) # Process data/time thisdata = _convert_ulysses_time(thisdata) if use_hdf: thisdata.to_hdf(local_hdf, 'fgm_hires') data.append(thisdata) return util.timefilter(data, starttime, endtime)
def _load(probe, starttime, endtime, instrument, product_id, product_list, try_download): dirs = [] fnames = [] download_info = [] for day in util._daysplitinterval(starttime, endtime): date = day[0] year = str(date.year) month = str(date.month).zfill(2) day = str(date.day).zfill(2) dirs.append(year) local_fname = 'C' + probe + '_' + product_id + '__' +\ year + month + day fnames.append(local_fname) thisstart = datetime.combine(date, time.min) thisend = datetime.combine(date, time.max) download_info.append((thisstart, thisend)) extension = '.cdf' local_base_dir = cluster_dir / ('c' + probe) / instrument remote_base_url = csa_url def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension, download_info): starttime, endtime = download_info _download(probe, starttime, endtime, instrument, product_id) def processing_func(file): for non_empty_var in list(file.cdf_info().keys()): if 'variable' in non_empty_var.lower(): if len(file.cdf_info()[non_empty_var]) > 0: var_list = non_empty_var break for key in file.cdf_info()[var_list]: if 'CDF_EPOCH' in file.varget(key, expand=True).values(): index_key = key break return util.cdf2xr(file, starttime, endtime, index_key, product_list) return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, try_download=try_download, units=None, download_info=download_info)
def _swics(starttime, endtime, names, product, units=None, try_download=True): data = [] dtimes = util._daysplitinterval(starttime, endtime) dirs = [] fnames = [] extension = '.dat' local_base_dir = ulysses_dir / 'swics' remote_base_url = ulysses_url def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): local_dir = local_base_dir / directory swics_options = url_options swics_options['FILE_NAME'] = fname + extension swics_options['FILE_PATH'] = '/ufa/HiRes/data/swics' for key in swics_options: remote_base_url += key + '=' + swics_options[key] + '&' try: util._load_remote(remote_base_url, fname + extension, local_dir, 'ascii') # f = util.load('', local_dir, remote_base_url) except Exception as err: return def processing_func(f): readargs = { 'names': names, 'delim_whitespace': True, 'na_values': ['******'] } thisdata = pd.read_csv(f, **readargs) thisdata = _convert_ulysses_time(thisdata) return thisdata # Loop through years for year in range(starttime.year, endtime.year + 1): fname = '{}{}'.format(product, str(year)[-2:]) # Local locaiton to download to dirs.append('') fnames.append(fname) return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units, try_download=True)
def fgm_survey(probe, starttime, endtime): """ Import fgm survey mode magnetic field data. Parameters ---------- probe : string Probe number, must be 1, 2, 3, or 4 starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame Imported data. """ # Directory relative to main MMS data directory relative_dir = os.path.join('mms' + probe, 'fgm', 'srvy', 'l2') daylist = util._daysplitinterval(starttime, endtime) data = [] for day in daylist: date = day[0] this_relative_dir = os.path.join(relative_dir, str(date.year), str(date.month).zfill(2)) filename = 'mms{}_fgm_srvy_l2_{}{:02}{:02}_v4.18.0.cdf'.format( probe, date.year, date.month, date.day) # Absolute path to local directory for this data file local_dir = os.path.join(mms_dir, this_relative_dir) util._checkdir(local_dir) remote_url = remote_mms_dir + this_relative_dir # Load cdf file cdf = util.load(filename, local_dir, remote_url) # Convert cdf to dataframe keys = { 'mms' + probe + '_fgm_b_gsm_srvy_l2': ['Bx', 'By', 'Bz', 'Br'], 'Epoch': 'Time' } df = util.cdf2df(cdf, 'Epoch', keys) data.append(df) return util.timefilter(data, starttime, endtime)
def mag_h0(starttime, endtime): """ Imports magnetic field data from DSCOVR Spacecraft. Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` """ dirs = [] fnames = [] extension = '.cdf' ignore = ['Time1_PB5'] daylist = util._daysplitinterval(starttime, endtime) for day in daylist: date = day[0] filename = "dscovr_h0_mag_{}{:02}{:02}_v01".format( date.year, date.month, date.day) fnames.append(filename) this_relative_dir = 'h0/mag/' + str(date.year) dirs.append(this_relative_dir) local_base_dir = dscovr_dir remote_base_url = dscovr_url def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): remote_url = remote_base_url + str(directory) filename = fname + extension local_dir = local_base_dir / directory util.load(fname + extension, local_base_dir / directory, remote_url) def processing_func(cdf): df = util.cdf2df(cdf, 'Epoch1', ignore=ignore) df.index.name = 'Time' return df return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime)
def _ace(starttime, endtime, instrument, product, fname, units=None, version='01', badvalues={}): """ Generic method for downloading ACE data from cdaweb. """ # Directory relative to main WIND data directory relative_dir = path.Path(instrument) / 'level_2_cdaweb' / product daylist = util._daysplitinterval(starttime, endtime) dirs = [] fnames = [] extension = '.cdf' for day in daylist: date = day[0] filename = 'ac_{}_{}{:02}{:02}_v[0-9][0-9]'.format( fname, date.year, date.month, date.day) fnames.append(filename) this_relative_dir = relative_dir / str(date.year) dirs.append(this_relative_dir) def download_func(*args): util._download_remote_unknown_version(*args) def processing_func(cdf): return util.cdf2df(cdf, index_key='Epoch', badvalues=badvalues) return util.process(dirs, fnames, extension, ace_dir, remote_ace_dir, download_func, processing_func, starttime, endtime, units=units)
def _swics(starttime, endtime, names, product): swics_options = url_options readargs = {'names': names, 'delim_whitespace': True} data = [] dtimes = util._daysplitinterval(starttime, endtime) # Loop through years for year in range(starttime.year, endtime.year + 1): swics_options['FILE_NAME'] = '{}{}.dat'.format(product, str(year)[-2:]) # Local locaiton to download to local_dir = os.path.join(ulysses_dir, 'swics') local_file = os.path.join(local_dir, swics_options['FILE_NAME']) local_hdf = local_file[:-4] + '.hdf' # If we have already saved a hdf file if os.path.exists(local_hdf): thisdata = pd.read_hdf(local_hdf) else: # Put together remote url swics_options['FILE_PATH'] = '/ufa/HiRes/data/swics' remote_url = ulysses_url for key in swics_options: remote_url += key + '=' + swics_options[key] + '&' f = util.load(swics_options['FILE_NAME'], local_dir, remote_url) if f is None: print('File {}/{} not available\n'.format( remote_url, filename)) continue # Read in data thisdata = pd.read_table(f, **readargs) # Process data/time thisdata = _convert_ulysses_time(thisdata) if use_hdf: thisdata.to_hdf(local_hdf, 'swics') data.append(thisdata) return util.timefilter(data, starttime, endtime)
def mag_hires(starttime, endtime, try_download=True): """ Import high resolution magnetic field from Cassini. See http://pds-ppi.igpp.ucla.edu/search/view/?f=yes&id=pds://PPI/CO-E_SW_J_S-MAG-3-RDR-FULL-RES-V1.0 for more information. Cassini Orbiter Magnetometer Calibrated MAG data at the highest time resolution available covering the period 1999-08-16 (DOY 228) to 2016-12-31 (DOY 366). The data are in RTN coordinates prior Cassini's arrival at Saturn, and Kronographic (KRTP) coordinates at Saturn (beginning 2004-05-14, DOY 135). Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` Requested data """ remote_base_url = ('https://pds-ppi.igpp.ucla.edu/ditdos/download?id=' 'pds://PPI/CO-E_SW_J_S-MAG-3-RDR-FULL-RES-V2.0/DATA') dirs = [] fnames = [] extension = '.TAB' units = OrderedDict([('Bx', u.nT), ('By', u.nT), ('Bz', u.nT), ('coords', u.dimensionless_unscaled)]) local_base_dir = cassini_dir / 'mag' / 'hires' for [day, _, _] in util._daysplitinterval(starttime, endtime): year = day.year if calendar.isleap(year): monthstr = leapmonth2str[day.month] else: monthstr = month2str[day.month] if day < datetime.date(2004, 5, 14): coords = 'RTN' else: coords = 'KRTP' doy = day.strftime('%j') dirs.append(pathlib.Path(str(year)) / monthstr) fnames.append(str(year)[2:] + doy + '_FGM_{}'.format(coords)) def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): url = remote_base_url + '/' + str(directory) util._download_remote(url, fname + extension, local_base_dir / directory) def processing_func(f): if 'error_message' in f.readline(): f.close() os.remove(f.name) raise util.NoDataError() df = pd.read_csv(f, names=['Time', 'Bx', 'By', 'Bz'], delim_whitespace=True, parse_dates=[0], index_col=0) return df return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units, try_download=try_download)
def threedp_sfpd(starttime, endtime): """ Import 'sfpd' wind data. 12 second energetic electron pitch-angle energy spectra from the foil SST Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame """ # Directory relative to main WIND data directory relative_dir = path.Path('3dp') / '3dp_sfpd' daylist = util._daysplitinterval(starttime, endtime) data = [] mag = [] for (date, _, _) in daylist: this_relative_dir = relative_dir / str(date.year) # Absolute path to local directory for this data file local_dir = wind_dir / this_relative_dir filename = 'wi_sfpd_3dp_{:{dfmt}}_v02'.format(date, dfmt='%Y%m%d') hdfname = filename + '.hdf' hdfloc = local_dir / hdfname if hdfloc.exists(): df = pd.read_hdf(hdfloc) data.append(df) continue util._checkdir(local_dir) remote_url = remote_wind_dir + str(this_relative_dir) cdf = util.load(filename + '.cdf', local_dir, remote_url, guessversion=True) if cdf is None: print('File {}/{} not available\n'.format(remote_url, filename)) continue data_today = [] # Loop through each timestamp to build up fluxes for i, time in enumerate(cdf['Epoch'][...]): energies = cdf['ENERGY'][i, :] angles = cdf['PANGLE'][i, :] fluxes = cdf['FLUX'][i, :, :] magfield = cdf['MAGF'][i, :] index = pd.MultiIndex.from_product( ([time], energies, angles), names=['Time', 'Energy', 'Pitch angle']) df = pd.DataFrame(fluxes.ravel(), index=index, columns=['Flux']) df['Bx'] = magfield[0] df['By'] = magfield[1] df['Bz'] = magfield[2] data_today.append(df) data_today = pd.concat(data_today) data_today = data_today.sort_index() if use_hdf: data_today.to_hdf(hdfloc, 'sfpd', mode='w') data.append(data_today) data = util.timefilter(data, starttime, endtime) data = data.reset_index(level=['Energy', 'Pitch angle']) return data
def threedp_pm(starttime, endtime): """ Import 'pm' wind data. 3 second time resolution solar wind proton and alpha particle moments from the PESA LOW sensor, computed on-board the spacecraft Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame """ # Directory relative to main WIND data directory relative_dir = path.Path('3dp') / '3dp_pm' daylist = util._daysplitinterval(starttime, endtime) data = [] for day in daylist: date = day[0] this_relative_dir = relative_dir / str(day[0].year) # Absolute path to local directory for this data file local_dir = wind_dir / this_relative_dir filename = 'wi_pm_3dp_' +\ str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) +\ '_v05.cdf' hdfname = filename[:-4] + 'hdf' hdfloc = local_dir / hdfname if hdfloc.exists(): df = pd.read_hdf(hdfloc) data.append(df) continue util._checkdir(local_dir) remote_url = remote_wind_dir + str(this_relative_dir) cdf = util.load(filename, local_dir, remote_url, guessversion=True) if cdf is None: print('File {}/{} not available\n'.format(remote_url, filename)) continue keys = { 'A_DENS': 'n_a', 'A_TEMP': 'T_a', 'A_VELS': ['va_x', 'va_y', 'va_z'], 'P_DENS': 'n_p', 'P_TEMP': 'T_p', 'P_VELS': ['vp_x', 'vp_y', 'vp_z'], 'Epoch': 'Time' } df = util.cdf2df(cdf, index_key='Epoch', keys=keys) if use_hdf: df.to_hdf(hdfloc, 'pm', mode='w') data.append(df) return util.timefilter(data, starttime, endtime)
def _mfi(starttime, endtime, version, units=None): """ Import mfi magnetic field data products from WIND. Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame """ # Directory relative to main WIND data directory relative_dir = path.Path('mfi') / ('mfi_' + version) # Get directories and filenames dirs = [] fnames = [] daylist = util._daysplitinterval(starttime, endtime) for day in daylist: date = day[0] # Absolute path to local directory for this data file local_dir = relative_dir / str(day[0].year) dirs.append(local_dir) filename = 'wi_' + version + '_mfi_' +\ str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) +\ '_v05' fnames.append(filename) extension = '.cdf' local_base_dir = wind_dir remote_base_url = remote_wind_dir def download_func(remote_base_url, local_base_dir, directory, fname, extension): remote_url = '{}{}'.format(remote_base_url, directory) util.load(fname + extension, local_base_dir / directory, remote_url, guessversion=True) def processing_func(cdf): epoch_dict = {'h0': 'Epoch3', 'h2': 'Epoch'} mag_dict = {'h0': 'B3GSE', 'h2': 'BGSE'} epoch_key = epoch_dict[version] mag_key = mag_dict[version] keys = {mag_key: ['Bx_gse', 'By_gse', 'Bz_gse'], epoch_key: 'Time'} badvalues = {'Bx_gse': -1e+31, 'By_gse': -1e+31, 'Bz_gse': -1e+31} df = util.cdf2df(cdf, index_key=epoch_key, keys=keys, badvalues=badvalues) return df return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units)
def threedp_sfpd(starttime, endtime): """ Import 'sfpd' wind data. 12 second energetic electron pitch-angle energy spectra from the foil SST Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` """ # Directory relative to main WIND data directory relative_dir = path.Path('3dp') / '3dp_sfpd' daylist = util._daysplitinterval(starttime, endtime) data = [] fnames = [] dirs = [] units = OrderedDict([('Energy', u.eV), ('Bx', u.nT), ('By', u.nT), ('Bz', u.nT), ('Pitch angle', u.deg), ('Flux', (u.cm**2 * u.sr * u.eV * u.s)**-1)]) extension = '.cdf' for (date, _, _) in daylist: this_relative_dir = relative_dir / str(date.year) dirs.append(this_relative_dir) filename = 'wi_sfpd_3dp_{:{dfmt}}_v02'.format( date, dfmt='%Y%m%d') fnames.append(filename) local_base_dir = wind_dir remote_base_url = remote_wind_dir def download_func(*args): util._download_remote_unknown_version(*args) def processing_func(cdf): data_today = [] # Loop through each timestamp to build up fluxes for non_empty_var in list(cdf.cdf_info().keys()): if 'variable' in non_empty_var.lower(): if len(cdf.cdf_info()[non_empty_var]) > 0: var_list = non_empty_var break index_ = cdf.varget('Epoch')[...] index_ = cdflib.cdfepoch.breakdown(index_) index_ = np.asarray([dt.datetime(*x) for x in index_]) energies_ = cdf.varget('ENERGY')[...] angles_ = cdf.varget('PANGLE')[...] fluxes_ = cdf.varget('FLUX')[...] magfield_ = cdf.varget('MAGF')[...] for i, time in enumerate(index_): energies = energies_[i, :] angles = angles_[i, :] fluxes = fluxes_[i, :, :] magfield = magfield_[i, :] index = pd.MultiIndex.from_product( ([time], energies, angles), names=['Time', 'Energy', 'Pitch angle']) df = pd.DataFrame(fluxes.ravel(), index=index, columns=['Flux']) df = df.reset_index(level=['Energy', 'Pitch angle']) df['Bx'] = magfield[0] df['By'] = magfield[1] df['Bz'] = magfield[2] data_today.append(df) data_today = pd.concat(data_today) data_today = data_today.sort_index() return data_today return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units)
def _download(probe, starttime, endtime, instrument, product_id): if cda_cookie == 'none': raise RuntimeError('Cluster download cookie not set') daylist = util._daysplitinterval(starttime, endtime) for day in daylist: date = day[0] start = datetime.combine(date, time.min) end = datetime.combine(date, time.max) # Add start and end time to request dictionary request_dict = generic_dict request_dict['START_DATE'] = start.strftime(cda_time_fmt) request_dict['END_DATE'] = end.strftime(cda_time_fmt) # Create request string request_str = '' request_str += 'DATASET_ID' + '=' request_str += 'C' + probe + '_' + product_id for item in request_dict: request_str += '&' request_str += item request_str += '=' request_str += request_dict[item] # Create request url request_str += '&NON_BROWSER' request_url = csa_url + request_str # Work out local directory to download to year = str(starttime.year) month = str(starttime.month).zfill(2) day = str(starttime.day).zfill(2) local_dir = cluster_dir / ('c' + probe) / instrument / year local_fname = 'C' + probe + '_' + product_id + '__' +\ year + month + day + '.cdf' local_file = local_dir / local_fname print(request_url) # Download data util._checkdir(local_dir) urlreq.urlretrieve(request_url, filename=local_file, reporthook=util._reporthook) print('\n') # Extract tar.gz file tar = tarfile.open(local_file) tar.extractall(local_dir) # Delete tar.gz file os.remove(local_file) # The CSA timpstamps the downloaded file by when it is downloaded, # so manually list and retrieve the folder name dirlist = os.listdir(local_dir) for d in dirlist: if d[:13] == 'CSA_Download_': download_dir = local_dir / d / ('C' + probe + '_' + product_id) break # Remove request times from filename dirlist = os.listdir(download_dir) # Move to data folder cutoff = 3 + len(product_id) + 10 for f in dirlist: os.rename(download_dir / f, local_dir / (f[:cutoff] + '.cdf')) # Delte extra folders created by tar.gz file os.rmdir(download_dir) os.rmdir(os.path.join(local_dir, d))
def fgm(probe, rate, coords, starttime, endtime): """ Import fgm magnetic field data from THEMIS. Parameters ---------- probe : string Alowed values are [a, b, c, d, e]. rate : string Date rate to return. Allowed values are [e, h, l, s]. coords : string Magnetic field co-ordinate system. Allowed values are [dsl, gse, gsm, ssl]. NOTE: Add link to co-ordinate system descriptions. starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` """ valid_rates = ['e', 'h', 'l', 's'] valid_coords = ['dsl', 'gse', 'gsm', 'ssl'] _validate_probe(probe) if rate not in valid_rates: raise ValueError(('rate argument %s is not in list of allowed' 'rates: %s') % (rate, valid_rates)) if coords not in valid_coords: raise ValueError(('coords argument %s is not in list of allowed' 'co-ordinate systems: %s') % (rate, valid_rates)) # Directory relative to main THEMIS data directory fgm_dir = path.Path('th' + probe) / 'l2' / 'fgm' daylist = util._daysplitinterval(starttime, endtime) dirs = [] fnames = [] extension = '.cdf' units = OrderedDict([('|B|', u.nT), ('Bz_dsl', u.nT), ('By_dsl', u.nT), ('Bx_dsl', u.nT), ('th{}_fgm_fg{}_quality'.format(probe, rate), u.dimensionless_unscaled)]) for day in daylist: date = day[0] filename = 'th{}_l2_fgm_{}{:02}{:02}_v01'.format( probe, date.year, date.month, date.day) fnames.append(filename) this_relative_dir = fgm_dir / str(date.year) dirs.append(this_relative_dir) def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): remote_url = remote_base_url + str(directory) # Now load remotely util.load(fname + extension, local_base_dir / directory, remote_url) def processing_func(cdf, **kwargs): probe = kwargs.pop('probe') rate = kwargs.pop('rate') kwargs.pop('coords') probestr = 'th' + probe ratestr = '_fg' + rate + '_' df = util.cdf2df(cdf, probestr + ratestr + 'time', dtimeindex=False) df = df.set_index(pd.to_datetime(df.index.values, unit='s')) df.index.name = 'Time' return df processing_kwargs = {'probe': probe, 'rate': rate, 'coords': coords} return util.process(dirs, fnames, extension, artemis_dir, remote_themis_dir, download_func, processing_func, starttime, endtime, units=units, processing_kwargs=processing_kwargs)
def mag15s(probe, starttime, endtime, verbose=False, try_download=True): """ Import 15s cadence magnetic field data. Parameters ---------- probe : string Probe number. starttime : datetime Start of interval. endtime : datetime End of interval. verbose : bool, optional If ``True``, print information whilst loading. Default is ``False``. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` Requested data. """ fnames = [] dirs = [] extension = '.asc' units = OrderedDict([('n points', u.dimensionless_unscaled), ('Source flag', u.dimensionless_unscaled), ('x gse', u.earthRad), ('y gse', u.earthRad), ('z gse', u.earthRad), ('y gsm', u.earthRad), ('z gsm', u.earthRad), ('|B|', u.nT), ('Bx gse', u.nT), ('By gse', u.nT), ('Bz gse', u.nT), ('By gsm', u.nT), ('Bz gsm', u.nT), ('Bxx gse', u.nT**2), ('Byy gse', u.nT**2), ('Bzz gse', u.nT**2), ('Byx gse', u.nT**2), ('Bzx gse', u.nT**2), ('Bzy gse', u.nT**2), ('Time shift', u.s), ('sw flag', u.dimensionless_unscaled)]) dtimes = util._daysplitinterval(starttime, endtime) # Loop through years for dtime in dtimes: startdt = dtime[0] year = startdt.year doy = util.dtime2doy(startdt) if verbose: print('Loading IMP 15s mag probe {}, {:03d}/{}'.format( probe, doy, year)) filename = '{}{:03d}_imp{}_mag_15s_v3'.format(year, doy, probe) fnames.append(filename) # Location of file relative to local directory or remote url relative_loc = os.path.join('imp{}'.format(probe), 'mag', '15s_ascii_v3', str(year)) dirs.append(relative_loc) local_base_dir = imp_dir remote_base_url = imp_url def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): remote_url = remote_base_url + str(directory) filename = fname + extension local_dir = local_base_dir / directory util._download_remote(remote_url, filename, local_dir) # Read in data def processing_func(f): readargs = { 'names': [ 'Year', 'doy', 'Second', 'Source flag', 'n points', 'x gse', 'y gse', 'z gse', 'y gsm', 'z gsm', '|B|', 'Bx gse', 'By gse', 'Bz gse', 'By gsm', 'Bz gsm', 'Bxx gse', 'Byy gse', 'Bzz gse', 'Byx gse', 'Bzx gse', 'Bzy gse', 'Time shift', 'sw flag' ], 'na_values': [ '9999', '999', '99', '9', '999', '99.99', '99.99', '99.99', '99.99', '99.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '999.9', '9' ], 'delim_whitespace': True } thisdata = pd.read_table(f, **readargs) thisdata['Time'] = (pd.to_datetime(thisdata['Year'], format='%Y') + pd.to_timedelta(thisdata['doy'] - 1, unit='d') + pd.to_timedelta(thisdata['Second'], unit='s')) thisdata = thisdata.set_index('Time', drop=False) thisdata = thisdata.drop(['Year', 'doy', 'Second'], 1) return thisdata return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units, try_download=try_download)
def download_files(probe, instrument, data_rate, starttime, endtime, verbose=True, product_string='', warn_missing_units=True): """ Download MMS files. Parameters ---------- probe : int or str MMS probe number. Must be in 1-4 inclusive. instrument : str MMS instrument. Must be in ``['afg', 'aspoc', 'dfg', 'dsp', 'edi', 'edp', 'fields', 'scm', 'sdp']`` data_rate : str Data rate. Must be in ``['slow', 'fast', 'brst', 'srvy']`` starttime : ~datetime.datetime Start time. endtime : ~datetime.datetime End time. verbose : bool If ``True``, show a progress bar while downloading. product_string : str If not empty, this string must be in the filename for it to be downloaded. warn_missing_units : bool If ``True``, warnings will be shown for each variable that does not have associated units. Returns ------- df : :class:`~sunpy.timeseries.GenericTimeSeries` Requested data. """ _validate_instrument(instrument) probe = _validate_probe(probe) dirs = [] fnames = [] daylist = util._daysplitinterval(starttime, endtime) for date, stime, etime in daylist: files = available_files(probe, instrument, starttime, endtime, data_rate, product_string) for file in files: fname = pathlib.Path(file).stem if product_string in fname and len(fname): fnames.append(fname) dirs.append('') extension = '.cdf' local_base_dir = mms_dir / probe / instrument / data_rate remote_base_url = dl_url def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): url = remote_base_url + '?file=' + fname + extension local_fname = os.path.join(local_base_dir, fname + extension) with requests.get(url, stream=True) as request: with open(local_fname, 'wb') as fd: for chunk in tqdm( request.iter_content(chunk_size=128)): fd.write(chunk) def processing_func(cdf): return util.cdf2df(cdf, index_key='Epoch') return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, warn_missing_units=warn_missing_units)