def _process_cdas(starttime, endtime, identifier, dataset, base_dir, units=None, badvalues=None, warn_missing_units=True): """ Generic method for downloading cdas data. """ relative_dir = pathlib.Path(identifier) # Directory relative to main WIND data directory daylist = util._daysplitinterval(starttime, endtime) dirs = [] fnames = [] dates = [] extension = '.cdf' for day in daylist: date = day[0] dates.append(date) filename = '{}_{}_{}{:02}{:02}'.format( dataset, identifier, date.year, date.month, date.day) fnames.append(filename) this_relative_dir = relative_dir / str(date.year) dirs.append(this_relative_dir) def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension, date): return get_data(identifier, date) def processing_func(cdf): return util.cdf2df(cdf, index_key='Epoch', badvalues=badvalues) return util.process(dirs, fnames, extension, base_dir, '', download_func, processing_func, starttime, endtime, units=units, download_info=dates, warn_missing_units=warn_missing_units)
def mag320ms(probe, starttime, endtime, try_download=True): """ Import 320ms cadence magnetic field data. Parameters ---------- probe : string Probe number. starttime : datetime Start of interval. endtime : datetime End of interval. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` Requested data. """ fnames = [] dirs = [] extension = '.cdf' dtimes = util._daysplitinterval(starttime, endtime) # Loop through years for dtime in dtimes: date = dtime[0] intervalstring = str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) filename = 'i8_320msec_mag_' + intervalstring + '_v01' fnames.append(filename) # Location of file relative to local directory or remote url relative_loc = 'imp' + probe + '/mag/mag_320msec_cdaweb/' +\ str(date.year) dirs.append(relative_loc) local_base_dir = imp_dir remote_base_url = imp_url def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): remote_url = remote_base_url + str(directory) filename = fname + extension local_dir = local_base_dir / directory util._download_remote(remote_url, filename, local_dir) def processing_func(f): thisdata = util.cdf2df(f, 'Epoch') thisdata.index.name = 'Time' return thisdata return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, try_download=try_download)
def _load_wind_cdf(starttime, endtime, instrument, data_product, fname, badvalues={}): relative_dir = path.Path(instrument) / data_product # Get directories and filenames dirs = [] fnames = [] daylist = util._daysplitinterval(starttime, endtime) for day in daylist: date = day[0] filename = 'wi_{}_{}{:02}{:02}_v01'.format(fname, date.year, date.month, date.day) fnames.append(filename) local_dir = relative_dir / str(date.year) dirs.append(local_dir) extension = '.cdf' local_base_dir = wind_dir remote_base_url = remote_wind_dir def download_func(remote_base_url, local_base_dir, directory, fname, extension): remote_url = '{}{}'.format(remote_base_url, directory) util.load(fname + extension, local_base_dir / directory, remote_url) def processing_func(cdf): return util.cdf2df(cdf, 'Epoch', badvalues=badvalues) return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime)
def fgm_hires(starttime, endtime, try_download=True): """ Import high resolution fluxgate magnetometer data. Parameters ---------- starttime : datetime Start of interval endtime : datetime End of interval Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` Requested data """ dtimes = util._daysplitinterval(starttime, endtime) dirs = [] fnames = [] extension = '.ASC' local_base_dir = ulysses_dir / 'fgm' / 'hires' remote_base_url = ulysses_url units = OrderedDict([('Bx', u.nT), ('By', u.nT), ('Bz', u.nT), ('|B|', u.nT)]) def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): local_dir = local_base_dir / directory fgm_options = url_options fgm_options['FILE_NAME'] = fname + extension fgm_options['FILE_PATH'] = '/ufa/HiRes/VHM-FGM/' + yearstr remote_url = ulysses_url for key in fgm_options: remote_url += key + '=' + fgm_options[key] + '&' try: util._load_remote(remote_url, fname + extension, local_dir, 'ascii') except Exception as err: return def processing_func(f): readargs = {'names': ['year', 'doy', 'hour', 'minute', 'second', 'Bx', 'By', 'Bz', '|B|'], 'delim_whitespace': True} thisdata = pd.read_table(f, **readargs) thisdata = _convert_ulysses_time(thisdata) return thisdata for dtime in dtimes: date = dtime[0] yearstr = date.strftime('%Y') filename = ('U' + yearstr[-2:] + date.strftime('%j') + 'SH') fnames.append(filename) this_relative_dir = local_base_dir / yearstr dirs.append(this_relative_dir) return util.process( dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units, try_download=True)
def threedp_pm(starttime, endtime): """ Import 'pm' wind data. 3 second time resolution solar wind proton and alpha particle moments from the PESA LOW sensor, computed on-board the spacecraft Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` """ # Directory relative to main WIND data directory relative_dir = path.Path('3dp') / '3dp_pm' daylist = util._daysplitinterval(starttime, endtime) ignore = ['TIME'] dirs = [] fnames = [] units = OrderedDict([('P_DENS', u.cm**-3), ('P_VELS', u.km / u.s), ('P_TEMP', u.eV), ('A_DENS', u.cm**-3), ('A_VELS', u.km / u.s), ('A_TEMP', u.eV), ('GAP', u.dimensionless_unscaled), ('E_RANGE', u.eV), ('VALID', u.dimensionless_unscaled), ('VC', u.dimensionless_unscaled), ('SPIN', u.dimensionless_unscaled)]) extension = '.cdf' for day in daylist: date = day[0] this_relative_dir = relative_dir / str(day[0].year) filename = 'wi_pm_3dp_' +\ str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) +\ '_v[0-9][0-9]' fnames.append(filename) dirs.append(this_relative_dir) local_base_dir = wind_dir remote_base_url = remote_wind_dir def download_func(*args): util._download_remote_unknown_version(*args) def processing_func(cdf): return util.cdf2df(cdf, 'Epoch', ignore=ignore) return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units)
def swe_h3(starttime, endtime): """ Import 'h3' solar wind electron data product from WIND. Electron pitch angle files providing electron fluxes at 30 directional bins relative to the instantaneous magnetic field direction at 13 different energy levels Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame """ relative_dir = os.path.join('swe', 'swe_h3') # Get directories and filenames dirs = [] fnames = [] daylist = util._daysplitinterval(starttime, endtime) for day in daylist: date = day[0] filename = 'wi_h3_swe_{}{:02}{:02}_v01'.format(date.year, date.month, date.day) fnames.append(filename) local_dir = os.path.join(relative_dir, str(date.year)) dirs.append(local_dir) extension = '.cdf' local_base_dir = wind_dir remote_base_url = remote_wind_dir def download_func(remote_base_url, local_base_dir, directory, fname, extension): remote_url = '{}{}'.format(remote_base_url, directory) util.load(fname + extension, os.path.join(local_base_dir, directory), remote_url) def processing_func(local_dir, fname): distkeys = [] for i in range(0, 13): distkeys.append('f_pitch_E' + str(i).zfill(2)) anglelabels = [] for i in range(0, 30): anglelabels.append((i + 0.5) * np.pi / 30) timekey = 'Epoch' energykey = 'Ve' cdf = util.load(fname, local_dir, '') df = util.pitchdist_cdf2df(cdf, distkeys, energykey, timekey, anglelabels) return df return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime)
def fgm_survey(probe, starttime, endtime): """ Import fgm survey mode magnetic field data. Parameters ---------- probe : string Probe number, must be 1, 2, 3, or 4 starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` Imported data. """ # Directory relative to main MMS data directory relative_dir = path.Path('mms' + probe) / 'fgm' / 'srvy' / 'l2' daylist = util._daysplitinterval(starttime, endtime) dirs = [] fnames = [] extension = '.cdf' units = OrderedDict([('mms{}_fgm_mode_srvy_l2'.format(probe), u.dimensionless_unscaled)]) data = [] for day in daylist: date = day[0] this_relative_dir = (relative_dir / str(date.year) / str(date.month).zfill(2)) filename = 'mms{}_fgm_srvy_l2_{}{:02}{:02}_v4.18.0'.format( probe, date.year, date.month, date.day) fnames.append(filename) dirs.append(this_relative_dir) remote_base_url = remote_mms_dir local_base_dir = mms_dir def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): remote_url = remote_base_url + str(directory) util.load(fname + extension, local_base_dir / directory, remote_url) def processing_func(cdf): df = util.cdf2df(cdf, 'Epoch') return df return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units)
def mag_rtn(starttime, endtime, try_download=True): """ Import magnetic field in RTN coordinates from Messenger. Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` """ # Directory relative to main WIND data directory relative_dir = path.Path('rtn') fnames = [] dirs = [] extension = '.cdf' data = [] ignore = ['Quality_Flag'] daylist = util._daysplitinterval(starttime, endtime) for day in daylist: date = day[0] this_relative_dir = relative_dir / str(date.year) filename = 'messenger_mag_rtn_' +\ str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) +\ '_v01' fnames.append(filename) dirs.append(this_relative_dir) local_base_dir = mess_dir remote_base_url = remote_mess_dir def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): remote_url = remote_base_url + str(directory) filename = fname + extension local_dir = local_base_dir / directory util._download_remote(remote_url, filename, local_dir) def processing_func(cdf): thisdata = util.cdf2df(cdf, 'Epoch', ignore=ignore) return thisdata return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, try_download=try_download)
def swe_h3(starttime, endtime): """ Import 'h3' solar wind electron data product from WIND. Electron pitch angle files providing electron fluxes at 30 directional bins relative to the instantaneous magnetic field direction at 13 different energy levels Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` """ warnings.warn(_deprecation_msg, DeprecationWarning) relative_dir = path.Path('swe') / 'swe_h3' # Get directories and filenames dirs = [] fnames = [] units = OrderedDict([('Angle', u .deg), ('Energy', u.eV), ('df', u.cm/u.s)]) daylist = util._daysplitinterval(starttime, endtime) for day in daylist: date = day[0] filename = 'wi_h3_swe_{}{:02}{:02}_v[0-9][0-9]'.format( date.year, date.month, date.day) fnames.append(filename) local_dir = relative_dir / str(date.year) dirs.append(local_dir) extension = '.cdf' local_base_dir = wind_dir remote_base_url = remote_wind_dir distkeys = [] for i in range(0, 13): distkeys.append('f_pitch_E' + str(i).zfill(2)) anglelabels = [] for i in range(0, 30): anglelabels.append((i + 0.5) * np.pi / 30) timekey = 'Epoch' energykey = 'Ve' def download_func(*args): util._download_remote_unknown_version(*args) def processing_func(cdf): df = util.pitchdist_cdf2df(cdf, distkeys, energykey, timekey, anglelabels) df = df.reset_index(level=['Energy', 'Angle']) return df return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units)
def _ace(starttime, endtime, instrument, product, fname, keys=None, version='01', badvalues={}): """ Generic method for downloading ACE data from cdaweb. """ # Directory relative to main WIND data directory relative_dir = os.path.join(instrument, 'level_2_cdaweb', product) daylist = util._daysplitinterval(starttime, endtime) dirs = [] fnames = [] extension = '.cdf' for day in daylist: date = day[0] filename = 'ac_{}_{}{:02}{:02}_v{}'.format(fname, date.year, date.month, date.day, version) fnames.append(filename) this_relative_dir = os.path.join(relative_dir, str(date.year)) dirs.append(this_relative_dir) def download_func(remote_base_url, local_base_dir, directory, fname, extension): def check_exists(): # Because the version might be different to the one we guess, work # out the downloaded filename for f in os.listdir(os.path.join(local_base_dir, directory)): if (f[:-6] == (fname + extension)[:-6]): # Return filename with '.cdf' stripped off the end return f[:-4] if check_exists() is not None: return check_exists() # Now load remotely util.load(fname + extension, os.path.join(local_base_dir, directory), remote_base_url + directory, guessversion=True) if check_exists() is not None: return check_exists() def processing_func(local_dir, local_fname): cdf = util.load(local_fname, local_dir, '') return util.cdf2df(cdf, index_key='Epoch', keys=keys, badvalues=badvalues) return util.process(dirs, fnames, extension, ace_dir, remote_ace_dir, download_func, processing_func, starttime, endtime)
def _load(probe, starttime, endtime, instrument, product_id, product_list, try_download): dirs = [] fnames = [] download_info = [] for day in util._daysplitinterval(starttime, endtime): date = day[0] year = str(date.year) month = str(date.month).zfill(2) day = str(date.day).zfill(2) dirs.append(year) local_fname = 'C' + probe + '_' + product_id + '__' +\ year + month + day fnames.append(local_fname) thisstart = datetime.combine(date, time.min) thisend = datetime.combine(date, time.max) download_info.append((thisstart, thisend)) extension = '.cdf' local_base_dir = cluster_dir / ('c' + probe) / instrument remote_base_url = csa_url def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension, download_info): starttime, endtime = download_info _download(probe, starttime, endtime, instrument, product_id) def processing_func(file): for non_empty_var in list(file.cdf_info().keys()): if 'variable' in non_empty_var.lower(): if len(file.cdf_info()[non_empty_var]) > 0: var_list = non_empty_var break for key in file.cdf_info()[var_list]: if 'CDF_EPOCH' in file.varget(key, expand=True).values(): index_key = key break return util.cdf2xr(file, starttime, endtime, index_key, product_list) return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, try_download=try_download, units=None, download_info=download_info)
def _swics(starttime, endtime, names, product, units=None, try_download=True): data = [] dtimes = util._daysplitinterval(starttime, endtime) dirs = [] fnames = [] extension = '.dat' local_base_dir = ulysses_dir / 'swics' remote_base_url = ulysses_url def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): local_dir = local_base_dir / directory swics_options = url_options swics_options['FILE_NAME'] = fname + extension swics_options['FILE_PATH'] = '/ufa/HiRes/data/swics' for key in swics_options: remote_base_url += key + '=' + swics_options[key] + '&' try: util._load_remote(remote_base_url, fname + extension, local_dir, 'ascii') # f = util.load('', local_dir, remote_base_url) except Exception as err: return def processing_func(f): readargs = { 'names': names, 'delim_whitespace': True, 'na_values': ['******'] } thisdata = pd.read_csv(f, **readargs) thisdata = _convert_ulysses_time(thisdata) return thisdata # Loop through years for year in range(starttime.year, endtime.year + 1): fname = '{}{}'.format(product, str(year)[-2:]) # Local locaiton to download to dirs.append('') fnames.append(fname) return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units, try_download=True)
def mag_h0(starttime, endtime): """ Imports magnetic field data from DSCOVR Spacecraft. Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` """ dirs = [] fnames = [] extension = '.cdf' ignore = ['Time1_PB5'] daylist = util._daysplitinterval(starttime, endtime) for day in daylist: date = day[0] filename = "dscovr_h0_mag_{}{:02}{:02}_v01".format( date.year, date.month, date.day) fnames.append(filename) this_relative_dir = 'h0/mag/' + str(date.year) dirs.append(this_relative_dir) local_base_dir = dscovr_dir remote_base_url = dscovr_url def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): remote_url = remote_base_url + str(directory) filename = fname + extension local_dir = local_base_dir / directory util.load(fname + extension, local_base_dir / directory, remote_url) def processing_func(cdf): df = util.cdf2df(cdf, 'Epoch1', ignore=ignore) df.index.name = 'Time' return df return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime)
def _ace(starttime, endtime, instrument, product, fname, units=None, version='01', badvalues={}): """ Generic method for downloading ACE data from cdaweb. """ # Directory relative to main WIND data directory relative_dir = path.Path(instrument) / 'level_2_cdaweb' / product daylist = util._daysplitinterval(starttime, endtime) dirs = [] fnames = [] extension = '.cdf' for day in daylist: date = day[0] filename = 'ac_{}_{}{:02}{:02}_v[0-9][0-9]'.format( fname, date.year, date.month, date.day) fnames.append(filename) this_relative_dir = relative_dir / str(date.year) dirs.append(this_relative_dir) def download_func(*args): util._download_remote_unknown_version(*args) def processing_func(cdf): return util.cdf2df(cdf, index_key='Epoch', badvalues=badvalues) return util.process(dirs, fnames, extension, ace_dir, remote_ace_dir, download_func, processing_func, starttime, endtime, units=units)
def download_files(probe, instrument, data_rate, starttime, endtime, verbose=True, product_string='', warn_missing_units=True): """ Download MMS files. Parameters ---------- probe : int or str MMS probe number. Must be in 1-4 inclusive. instrument : str MMS instrument. Must be in ``['afg', 'aspoc', 'dfg', 'dsp', 'edi', 'edp', 'fields', 'scm', 'sdp']`` data_rate : str Data rate. Must be in ``['slow', 'fast', 'brst', 'srvy']`` starttime : ~datetime.datetime Start time. endtime : ~datetime.datetime End time. verbose : bool If ``True``, show a progress bar while downloading. product_string : str If not empty, this string must be in the filename for it to be downloaded. warn_missing_units : bool If ``True``, warnings will be shown for each variable that does not have associated units. Returns ------- df : :class:`~sunpy.timeseries.GenericTimeSeries` Requested data. """ _validate_instrument(instrument) probe = _validate_probe(probe) dirs = [] fnames = [] daylist = util._daysplitinterval(starttime, endtime) for date, stime, etime in daylist: files = available_files(probe, instrument, starttime, endtime, data_rate, product_string) for file in files: fname = pathlib.Path(file).stem if product_string in fname and len(fname): fnames.append(fname) dirs.append('') extension = '.cdf' local_base_dir = mms_dir / probe / instrument / data_rate remote_base_url = dl_url def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): url = remote_base_url + '?file=' + fname + extension local_fname = os.path.join(local_base_dir, fname + extension) with requests.get(url, stream=True) as request: with open(local_fname, 'wb') as fd: for chunk in tqdm( request.iter_content(chunk_size=128)): fd.write(chunk) def processing_func(cdf): return util.cdf2df(cdf, index_key='Epoch') return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, warn_missing_units=warn_missing_units)
def mitplasma_h0(probe, starttime, endtime, try_download=True): """ Import mit h0 plasma data. Parameters ---------- probe : string Probe number. starttime : datetime Start of interval. endtime : datetime End of interval. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` Requested data. """ dirs = [] fnames = [] extension = '.cdf' units = OrderedDict([('mode', u.dimensionless_unscaled), ('Region', u.dimensionless_unscaled), ('Spacecraft', u.dimensionless_unscaled)]) for date, _, _ in util._daysplitinterval(starttime, endtime): intervalstring = str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) filename = 'i' + probe + '_h0_mitplasma_' + intervalstring + '_v01' fnames.append(filename) # Location of file relative to local directory or remote url relative_loc = 'imp{}/plasma_mit/mitplasma_h0/{}'.format( probe, date.year) dirs.append(relative_loc) local_base_dir = imp_dir remote_base_url = imp_url def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): remote_url = remote_base_url + str(directory) filename = fname + extension local_dir = local_base_dir / directory util._download_remote(remote_url, filename, local_dir) def processing_func(f): thisdata = util.cdf2df(f, 'Epoch') thisdata.index.name = 'Time' return thisdata return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units, try_download=try_download)
def swoops_ions(starttime, endtime, try_download=True): """ Import SWOOPS ion data. Parameters ---------- starttime : datetime Start of interval endtime : datetime End of interval Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` Requested data """ dirs = [] fnames = [] extension = '.dat' local_base_dir = ulysses_dir / 'swoops' / 'ions' remote_base_url = ulysses_url units = OrderedDict([('T_p_large', u.K), ('T_p_small', u.K), ('v_t', u.km / u.s), ('v_r', u.km / u.s), ('v_n', u.km / u.s), ('r', u.au), ('n_a', u.cm**-3), ('n_p', u.cm**-3), ('hlat', u.deg), ('hlon', u.deg), ('iqual', u.dimensionless_unscaled)]) def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): local_dir = local_base_dir / directory swoops_options = url_options year = fname[1:3] # doy = fname[5:8] swoops_options['FILE_NAME'] = fname + extension swoops_options['FILE_PATH'] =\ ('/ufa/stageIngestArea/swoops/ions/bamion{}.zip_files'.format(year)) for key in swoops_options: remote_base_url += key + '=' + swoops_options[key] + '&' try: util._load_remote(remote_base_url, fname + extension, local_dir, 'ascii') # f = util.load('', local_dir, remote_base_url) except Exception as err: return def processing_func(f): readargs = { 'names': [ 'year', 'doy', 'hour', 'minute', 'second', 'r', 'hlat', 'hlon', 'n_p', 'n_a', 'T_p_large', 'T_p_small', 'v_r', 'v_t', 'v_n', 'iqual' ], 'delim_whitespace': True } thisdata = pd.read_csv(f, **readargs) thisdata = _convert_ulysses_time(thisdata) return thisdata # Loop through years for year in range(starttime.year, endtime.year + 1): if year == starttime.year: start_month = starttime.month else: start_month = 1 if year == endtime.year: end_month = endtime.month else: end_month = 12 for month in range(start_month, end_month + 1): doy = datetime(year, month, 1).strftime('%j') fanme = ('u{}{}bam'.format(str(year)[2:], doy)) # Local locaiton to download to dirs.append('{}'.format(year)) fnames.append(fanme) return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units, try_download=try_download)
def mag15s(probe, starttime, endtime, verbose=False, try_download=True): """ Import 15s cadence magnetic field data. Parameters ---------- probe : string Probe number. starttime : datetime Start of interval. endtime : datetime End of interval. verbose : bool, optional If ``True``, print information whilst loading. Default is ``False``. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` Requested data. """ fnames = [] dirs = [] extension = '.asc' units = OrderedDict([('n points', u.dimensionless_unscaled), ('Source flag', u.dimensionless_unscaled), ('x gse', u.earthRad), ('y gse', u.earthRad), ('z gse', u.earthRad), ('y gsm', u.earthRad), ('z gsm', u.earthRad), ('|B|', u.nT), ('Bx gse', u.nT), ('By gse', u.nT), ('Bz gse', u.nT), ('By gsm', u.nT), ('Bz gsm', u.nT), ('Bxx gse', u.nT**2), ('Byy gse', u.nT**2), ('Bzz gse', u.nT**2), ('Byx gse', u.nT**2), ('Bzx gse', u.nT**2), ('Bzy gse', u.nT**2), ('Time shift', u.s), ('sw flag', u.dimensionless_unscaled)]) dtimes = util._daysplitinterval(starttime, endtime) # Loop through years for dtime in dtimes: startdt = dtime[0] year = startdt.year doy = util.dtime2doy(startdt) if verbose: print('Loading IMP 15s mag probe {}, {:03d}/{}'.format( probe, doy, year)) filename = '{}{:03d}_imp{}_mag_15s_v3'.format(year, doy, probe) fnames.append(filename) # Location of file relative to local directory or remote url relative_loc = os.path.join('imp{}'.format(probe), 'mag', '15s_ascii_v3', str(year)) dirs.append(relative_loc) local_base_dir = imp_dir remote_base_url = imp_url def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): remote_url = remote_base_url + str(directory) filename = fname + extension local_dir = local_base_dir / directory util._download_remote(remote_url, filename, local_dir) # Read in data def processing_func(f): readargs = { 'names': [ 'Year', 'doy', 'Second', 'Source flag', 'n points', 'x gse', 'y gse', 'z gse', 'y gsm', 'z gsm', '|B|', 'Bx gse', 'By gse', 'Bz gse', 'By gsm', 'Bz gsm', 'Bxx gse', 'Byy gse', 'Bzz gse', 'Byx gse', 'Bzx gse', 'Bzy gse', 'Time shift', 'sw flag' ], 'na_values': [ '9999', '999', '99', '9', '999', '99.99', '99.99', '99.99', '99.99', '99.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '999.9', '9' ], 'delim_whitespace': True } thisdata = pd.read_table(f, **readargs) thisdata['Time'] = (pd.to_datetime(thisdata['Year'], format='%Y') + pd.to_timedelta(thisdata['doy'] - 1, unit='d') + pd.to_timedelta(thisdata['Second'], unit='s')) thisdata = thisdata.set_index('Time', drop=False) thisdata = thisdata.drop(['Year', 'doy', 'Second'], 1) return thisdata return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units, try_download=try_download)
def mag_1min(starttime, endtime, coords): """ Import 1 minute magnetic field from Cassini. See http://pds-ppi.igpp.ucla.edu/search/view/?f=yes&id=pds://PPI/CO-E_SW_J_S-MAG-4-SUMM-1MINAVG-V1.0 for more information. Cassini Orbiter Magnetometer Calibrated MAG data in 1 minute averages available covering the period 1999-08-16 (DOY 228) to 2016-12-31 (DOY 366). The data are provided in RTN coordinates throughout the mission, with Earth, Jupiter, and Saturn centered coordinates for the respective flybys of those planets. Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. coords : strings Requested coordinate system. Must be one of ``['KRTP', 'KSM', 'KSO', 'RTN']`` Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` Requested data """ valid_coords = ['KRTP', 'KSM', 'KSO', 'RTN'] if coords not in valid_coords: raise ValueError('coords must be one of {}'.format(valid_coords)) base_url = ('http://pds-ppi.igpp.ucla.edu/ditdos/download?' 'id=pds://PPI/CO-E_SW_J_S-MAG-4-SUMM-1MINAVG-V1.0/DATA') Rs = u.def_unit('saturnRad', 60268 * u.km) if (coords == 'KRTP'): units = OrderedDict([('Bx', u.nT), ('By', u.nT), ('Bz', u.nT), ('X', Rs), ('|B|', u.nT), ('Y', u.deg), ('Z', u.deg), ('Local hour', u.dimensionless_unscaled), ('n points', u.dimensionless_unscaled)]) if (coords == 'RTN'): units = OrderedDict([('Bx', u.nT), ('By', u.nT), ('Bz', u.nT), ('X', u.AU), ('Y', u.AU), ('Z', u.AU), ('|B|', u.nT), ('Local hour', u.dimensionless_unscaled), ('n points', u.dimensionless_unscaled)]) if (coords == 'KSM' or coords == 'KSO'): units = OrderedDict([('Bx', u.nT), ('By', u.nT), ('Bz', u.nT), ('X', Rs), ('Y', Rs), ('Z', Rs), ('|B|', u.nT), ('Local hour', u.dimensionless_unscaled), ('n points', u.dimensionless_unscaled)]) local_base_dir = cassini_dir / 'mag' / '1min' dirs = [] fnames = [] extension = '.TAB' for year in starttime.year, endtime.year: dirs.append('{}'.format(year)) fnames.append('{}_FGM_{}_1M'.format(year, coords)) def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): url = '{}/{}'.format(base_url, year) util._download_remote(url, fname + extension, local_base_dir / directory) def processing_func(f): if 'error_message' in f.readline(): location = f.name f.close() os.remove(f.name) raise util.NoDataError() data = pd.read_table(f, names=[ 'Time', 'Bx', 'By', 'Bz', '|B|', 'X', 'Y', 'Z', 'Local hour', 'n points' ], delim_whitespace=True, parse_dates=[0], index_col=0) f.close() return data return util.process(dirs, fnames, extension, local_base_dir, base_url, download_func, processing_func, starttime, endtime, units=units)
def threedp_sfpd(starttime, endtime): """ Import 'sfpd' wind data. 12 second energetic electron pitch-angle energy spectra from the foil SST Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` """ # Directory relative to main WIND data directory relative_dir = path.Path('3dp') / '3dp_sfpd' daylist = util._daysplitinterval(starttime, endtime) data = [] fnames = [] dirs = [] units = OrderedDict([('Energy', u.eV), ('Bx', u.nT), ('By', u.nT), ('Bz', u.nT), ('Pitch angle', u.deg), ('Flux', (u.cm**2 * u.sr * u.eV * u.s)**-1)]) extension = '.cdf' for (date, _, _) in daylist: this_relative_dir = relative_dir / str(date.year) dirs.append(this_relative_dir) filename = 'wi_sfpd_3dp_{:{dfmt}}_v02'.format( date, dfmt='%Y%m%d') fnames.append(filename) local_base_dir = wind_dir remote_base_url = remote_wind_dir def download_func(*args): util._download_remote_unknown_version(*args) def processing_func(cdf): data_today = [] # Loop through each timestamp to build up fluxes for non_empty_var in list(cdf.cdf_info().keys()): if 'variable' in non_empty_var.lower(): if len(cdf.cdf_info()[non_empty_var]) > 0: var_list = non_empty_var break index_ = cdf.varget('Epoch')[...] index_ = cdflib.cdfepoch.breakdown(index_) index_ = np.asarray([dt.datetime(*x) for x in index_]) energies_ = cdf.varget('ENERGY')[...] angles_ = cdf.varget('PANGLE')[...] fluxes_ = cdf.varget('FLUX')[...] magfield_ = cdf.varget('MAGF')[...] for i, time in enumerate(index_): energies = energies_[i, :] angles = angles_[i, :] fluxes = fluxes_[i, :, :] magfield = magfield_[i, :] index = pd.MultiIndex.from_product( ([time], energies, angles), names=['Time', 'Energy', 'Pitch angle']) df = pd.DataFrame(fluxes.ravel(), index=index, columns=['Flux']) df = df.reset_index(level=['Energy', 'Pitch angle']) df['Bx'] = magfield[0] df['By'] = magfield[1] df['Bz'] = magfield[2] data_today.append(df) data_today = pd.concat(data_today) data_today = data_today.sort_index() return data_today return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units)
def merged(probe, starttime, endtime, try_download=True): """ Import merged plasma data. See ftp://cdaweb.gsfc.nasa.gov/pub/data/imp/imp8/merged/00readme.txt for information on variables. Parameters ---------- probe : string Probe number. starttime : datetime Start of interval. endtime : datetime End of interval. verbose : bool, optional If ``True``, print information whilst loading. Default is ``False``. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` Requested data. """ _check_probe(probe, ['8']) dirs = [] fnames = [] extension = '.asc' units = OrderedDict([ ('sw_flag', u.dimensionless_unscaled), ('x_gse', u.R_earth), ('y_gse', u.R_earth), ('z_gse', u.R_earth), ('y_gsm', u.R_earth), ('z_gsm', u.R_earth), ('Nm', u.dimensionless_unscaled), ('<|B|>', u.nT), ('|<B>|', u.nT), ('<B_lat>', u.nT), ('<B_long>', u.nT), ('Bx_gse', u.nT), ('By_gse', u.nT), ('Bz_gse', u.nT), ('By_gsm', u.nT), ('Bz_gsm', u.nT), ('sigma|B|', u.nT), ('sigma B', u.nT), ('sigma B_x', u.nT), ('sigma B_y', u.nT), ('sigma B_z', u.nT), ('plas_reg', u.dimensionless_unscaled), ('Npp', u.dimensionless_unscaled), ('v_fit', u.km / u.s), ('vx_fit_gse', u.km / u.s), ('vy_fit_gse', u.km / u.s), ('vz_fit_gse', u.km / u.s), ('vlong_fit', u.deg), ('vlat_fit', u.deg), ('np_fit', u.cm**-3), ('Tp_fit', u.K), ('v_mom', u.km / u.s), ('vx_mom_gse', u.km / u.s), ('vy_mom_gse', u.km / u.s), ('vz_mom_gse', u.km / u.s), ('vlong_mom', u.deg), ('vlat_mom', u.deg), ('np_mom', u.cm**-3), ('Tp_mom', u.K), ('FCp', u.dimensionless_unscaled), ('DWp', u.dimensionless_unscaled) ]) local_base_dir = imp_dir / 'imp{}'.format(probe) / 'merged' remote_base_url = imp_url + 'imp{}/merged'.format(probe) # Populate directories and filenames startyear = starttime.year endyear = endtime.year for year in range(startyear, endyear + 1): if year == startyear: startmonth = starttime.month else: startmonth = 1 if year == endyear: endmonth = endtime.month else: endmonth = 12 for month in range(startmonth, endmonth + 1): intervalstring = str(year) + str(month).zfill(2) fname = 'imp_min_merge' + intervalstring fnames.append(fname) dirs.append('') def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): filename = fname + extension local_dir = path.Path(local_base_dir) / directory util._download_remote(remote_base_url, filename, local_dir) def processing_func(f): readargs = { 'names': [ 'Year', 'doy', 'Hour', 'Minute', 'sw_flag', 'x_gse', 'y_gse', 'z_gse', 'y_gsm', 'z_gsm', 'Nm', 'FCm', 'DWm', '<|B|>', '|<B>|', '<B_lat>', '<B_long>', 'Bx_gse', 'By_gse', 'Bz_gse', 'By_gsm', 'Bz_gsm', 'sigma|B|', 'sigma B', 'sigma B_x', 'sigma B_y', 'sigma B_z', 'plas_reg', 'Npp', 'FCp', 'DWp', 'v_fit', 'vx_fit_gse', 'vy_fit_gse', 'vz_fit_gse', 'vlong_fit', 'vlat_fit', 'np_fit', 'Tp_fit', 'v_mom', 'vx_mom_gse', 'vy_mom_gse', 'vz_mom_gse', 'vlong_mom', 'vlat_mom', 'np_mom', 'Tp_mom' ], 'na_values': [ '9999', '999', '99', '99', '9', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9', '99', '9.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9999.99', '9', '9', '99', '9.99', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999999.', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999999.' ], 'delim_whitespace': True } # Read in data data = pd.read_table(f, **readargs) data['Time'] = (pd.to_datetime(data['Year'], format='%Y') + pd.to_timedelta(data['doy'] - 1, unit='d') + pd.to_timedelta(data['Hour'], unit='h') + pd.to_timedelta(data['Minute'], unit='m')) data = data.drop(['Year', 'doy', 'Hour', 'Minute', 'FCm', 'DWm'], axis=1) data = data.set_index('Time', drop=True) return data return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units, try_download=try_download)
def download_files(probe, instrument, data_rate, starttime, endtime, product_list=None, verbose=True, product_string='', warn_missing_units=True, want_xr=False): """ Download MMS files. Parameters ---------- probe : int or str MMS probe number. Must be in 1-4 inclusive. instrument : str MMS instrument. Must be in ``['afg', 'aspoc', 'dfg', 'dsp', 'edi', 'edp', 'fields', 'scm', 'sdp']`` data_rate : str Data rate. Must be in ``['slow', 'fast', 'brst', 'srvy']`` starttime : ~datetime.datetime Start time. endtime : ~datetime.datetime End time. verbose : bool, optional If ``True``, show a progress bar while downloading. product_string : str, optional If not empty, this string must be in the filename for it to be downloaded. warn_missing_units : bool, optional If ``True``, warnings will be shown for each variable that does not have associated units. Returns ------- df : :class:`~sunpy.timeseries.GenericTimeSeries` Requested data. """ _validate_instrument(instrument) probe = _validate_probe(probe) dirs = [] fnames = [] daylist = util._daysplitinterval(starttime, endtime) for date, stime, etime in daylist: files = available_files(probe, instrument, starttime, endtime, data_rate) for file in files: fname = pathlib.Path(file).stem # Make sure that only the needed files will be loaded (i.e., in the queried time interval) namestr = [ j for i, j in enumerate(fname.split('_')) if j.startswith(endtime.strftime('%Y%m%d')) ] # Select only one 'mec' (metadata) file (here 'epht89d') to avoid redundancy if instrument == 'mec': if 'epht89d' in fname: fnames.append(fname) dirs.append('') elif product_string in fname and len( fname ) and namestr and namestr[0] < endtime.strftime('%Y%m%d%H%M%S'): fnames.append(fname) dirs.append('') else: pass extension = '.cdf' local_base_dir = mms_dir / probe / instrument / data_rate remote_base_url = dl_url def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): url = remote_base_url + '?file=' + fname + extension local_fname = os.path.join(local_base_dir, fname + extension) with requests.get(url, stream=True) as request: with open(local_fname, 'wb') as fd: for chunk in tqdm(request.iter_content(chunk_size=128)): fd.write(chunk) def processing_func(cdf): if want_xr: return util.cdf2xr(cdf, starttime, endtime, 'Epoch', product_list) else: return util.cdf2df(cdf, starttime, endtime, 'Epoch', product_list) return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, want_xr, warn_missing_units=warn_missing_units)
def fgm(probe, rate, coords, starttime, endtime): """ Import fgm magnetic field data from THEMIS. Parameters ---------- probe : string Alowed values are [a, b, c, d, e]. rate : string Date rate to return. Allowed values are [e, h, l, s]. coords : string Magnetic field co-ordinate system. Allowed values are [dsl, gse, gsm, ssl]. NOTE: Add link to co-ordinate system descriptions. starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` """ valid_rates = ['e', 'h', 'l', 's'] valid_coords = ['dsl', 'gse', 'gsm', 'ssl'] _validate_probe(probe) if rate not in valid_rates: raise ValueError(('rate argument %s is not in list of allowed' 'rates: %s') % (rate, valid_rates)) if coords not in valid_coords: raise ValueError(('coords argument %s is not in list of allowed' 'co-ordinate systems: %s') % (rate, valid_rates)) # Directory relative to main THEMIS data directory fgm_dir = path.Path('th' + probe) / 'l2' / 'fgm' daylist = util._daysplitinterval(starttime, endtime) dirs = [] fnames = [] extension = '.cdf' units = OrderedDict([('|B|', u.nT), ('Bz_dsl', u.nT), ('By_dsl', u.nT), ('Bx_dsl', u.nT), ('th{}_fgm_fg{}_quality'.format(probe, rate), u.dimensionless_unscaled)]) for day in daylist: date = day[0] filename = 'th{}_l2_fgm_{}{:02}{:02}_v01'.format( probe, date.year, date.month, date.day) fnames.append(filename) this_relative_dir = fgm_dir / str(date.year) dirs.append(this_relative_dir) def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): remote_url = remote_base_url + str(directory) # Now load remotely util.load(fname + extension, local_base_dir / directory, remote_url) def processing_func(cdf, **kwargs): probe = kwargs.pop('probe') rate = kwargs.pop('rate') kwargs.pop('coords') probestr = 'th' + probe ratestr = '_fg' + rate + '_' df = util.cdf2df(cdf, probestr + ratestr + 'time', dtimeindex=False) df = df.set_index(pd.to_datetime(df.index.values, unit='s')) df.index.name = 'Time' return df processing_kwargs = {'probe': probe, 'rate': rate, 'coords': coords} return util.process(dirs, fnames, extension, artemis_dir, remote_themis_dir, download_func, processing_func, starttime, endtime, units=units, processing_kwargs=processing_kwargs)
def mag_ness(probe, starttime, endtime, try_download=True): """ Read in 6 second magnetic field data. Parameters ---------- probe : int, string Helios probe to import data from. Must be 1 or 2. starttime : datetime Interval start time endtime : datetime Interval end time try_download : bool, optional If ``False`` don't try to download data if it is missing locally. Returns ------- data : DataFrame 6 second magnetic field data set """ probe = _check_probe(probe) remote_base_url = ('http://helios-data.ssl.berkeley.edu/data/' 'E3_experiment/helios{}_6sec_ness/'.format(probe)) local_base_dir = (path.Path(helios_dir) / 'E3_experiment' / 'helios{}_6sec_ness'.format(probe)) dirs = [] fnames = [] extension = '.asc' units = OrderedDict([('probe', u.dimensionless_unscaled), ('naverage', u.dimensionless_unscaled), ('Bx', u.nT), ('By', u.nT), ('Bz', u.nT), ('|B|', u.nT), ('sigma_Bx', u.nT), ('sigma_By', u.nT), ('sigma_Bz', u.nT)]) daylist = util._daysplitinterval(starttime, endtime) for [day, _, _] in daylist: year = day.year doy = int(day.strftime('%j')) dirs.append('{}'.format(day.year)) fnames.append('h{}{}{:03}'.format(probe, year - 1900, doy)) def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): remote_url = remote_base_url + str(directory) local_dir = local_base_dir / directory filename = fname + extension try: util._download_remote(remote_url, filename, local_dir) except URLError: raise util.NoDataError def processing_func(f): # Read in data headings = [ 'probe', 'year', 'doy', 'hour', 'minute', 'second', 'naverage', 'Bx', 'By', 'Bz', '|B|', 'sigma_Bx', 'sigma_By', 'sigma_Bz' ] colspecs = [(1, 2), (2, 4), (4, 7), (7, 9), (9, 11), (11, 13), (13, 15), (15, 22), (22, 29), (29, 36), (36, 42), (42, 48), (48, 54), (54, 60)] data = pd.read_fwf(f, names=headings, header=None, colspecs=colspecs) # Process data data['year'] += 1900 # Convert date info to datetime data['Time'] = pd.to_datetime(data['year'], format='%Y') + \ pd.to_timedelta(data['doy'] - 1, unit='d') + \ pd.to_timedelta(data['hour'], unit='h') + \ pd.to_timedelta(data['minute'], unit='m') + \ pd.to_timedelta(data['second'], unit='s') data = data.drop(['year', 'doy', 'hour', 'minute', 'second'], axis=1) data = data.set_index('Time', drop=False) return data return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units, try_download=try_download)
def low(starttime, endtime, try_download=True): """ Import data from OMNI Web Interface. Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` """ # Directory relative to main OMNI data directory local_base_dir = omni_dir / 'low' remote_base_url = omni_url + '/low_res_omni' dirs = [] fnames = [] names = ['Year', 'Decimal Day', 'Hour', 'Bartels Rotation Number', 'ID IMF Spacecraft', 'ID SW Plasma Spacecraft', 'points(IMF Average)', 'points(Plasma Average)', '|B|', 'Magnitude of Avg Field Vector', 'Lat. Angle of Aver. Field Vector', 'Long. Angle of Aver. Field Vector', 'Bx GSE, GSM', 'By GSE', 'Bz GSE', 'By GSM', 'Bz GSM', 'sigma |B|', 'sigma B', 'sigma Bx', 'sigma By', 'sigma Bz', 'Proton Temperature', 'Proton Density', 'Plasma Flow Speed', 'Plasma Flow Long. Angle', 'Plasma Flow Lat. Angle', 'Na/Np', 'Flow Pressure', 'sigma T', 'sigma N', 'sigma V', 'sigma phi V', 'sigma theta V', 'sigma Na/Np', 'Electric Field', 'Plasma Beta', 'Alfven Mach Number', 'Kp', 'R', 'DST Index', 'AE Index', 'Proton Flux > 1MeV', 'Proton Flux > 2MeV', 'Proton Flux > 4MeV', 'Proton Flux > 10MeV', 'Proton Flux > 30MeV', 'Proton Flux > 60MeV', 'flag', 'ap index', 'f10.7 index', 'PC(N) index', 'AL index (Kyoto)', 'AU index (Kyoto)', 'Magnetosonic Mach No.'] badvalues = [np.nan, np.nan, np.nan, 9999, 99, 99, 999, 999, 999.9, 999.9, 999.9, 999.9, 999.9, 999.9, 999.9, 999.9, 999.9, 999.9, 999.9, 999.9, 999.9, 999.9, 9999999., 999.9, 9999., 999.9, 999.9, 9.999, 99.99, 9999999., 999.9, 9999., 999.9, 999.9, 9.999, 999.99, 999.99, 999.9, 99, 999, 99999, 9999, 999999.99, 99999.99, 99999.99, 99999.99, 99999.99, 99999.99, np.nan, 999, 999.9, 999.9, 99999, 99999, 99.9] sfu = u.def_unit('sfu', 10**-22 * u.m**-2 * u.Hz**-1) units = OrderedDict([('Bartels Rotation Number', u.dimensionless_unscaled), ('ID IMF Spacecraft', u.dimensionless_unscaled), ('ID SW Plasma Spacecraft', u.dimensionless_unscaled), ('points(IMF Average)', u.dimensionless_unscaled), ('points(Plasma Average)', u.dimensionless_unscaled), ('|B|', u.nT), ('Magnitude of Avg Field Vector', u.nT), ('Lat. Angle of Aver. Field Vector', u.deg), ('Long. Angle of Aver. Field Vector', u.deg), ('Bx GSE, GSM', u.nT), ('By GSE', u.nT), ('Bz GSE', u.nT), ('By GSM', u.nT), ('Bz GSM', u.nT), ('sigma |B|', u.nT), ('sigma B', u.nT), ('sigma Bx', u.nT), ('sigma By', u.nT), ('sigma Bz', u.nT), ('Proton Temperature', u.K), ('Proton Density', u.cm**-3), ('Plasma Flow Speed', u.km / u.s), ('Plasma Flow Long. Angle', u.deg), ('Plasma Flow Lat. Angle', u.deg), ('Na/Np', u.dimensionless_unscaled), ('Flow Pressure', u.nPa), ('sigma T', u.K), ('sigma N', u.cm**-3), ('sigma V', u.km / u.s), ('sigma phi V', u.deg), ('sigma theta V', u.deg), ('sigma Na/Np', u.dimensionless_unscaled), ('Electric Field', u.mV / u.m), ('Plasma Beta', u.dimensionless_unscaled), ('Alfven Mach Number', u.dimensionless_unscaled), ('Kp', u.dimensionless_unscaled), ('R', u.dimensionless_unscaled), ('AE Index', u.nT), ('DST Index', u.nT), ('Proton Flux > 1MeV', u.cm**-2), ('Proton Flux > 2MeV', u.cm**-2), ('Proton Flux > 4MeV', u.cm**-2), ('Proton Flux > 10MeV', u.cm**-2), ('Proton Flux > 30MeV', u.cm**-2), ('Proton Flux > 60MeV', u.cm**-2), ('flag', u.dimensionless_unscaled), ('ap index', u.nT), ('PC(N) index', u.dimensionless_unscaled), ('AL index (Kyoto)', u.nT), ('AU index (Kyoto)', u.nT), ('Magnetosonic Mach No.', u.dimensionless_unscaled), ('f10.7 index', sfu)]) extension = '.dat' for year in range(starttime.year, endtime.year + 1): fnames.append("omni2_{}".format(year)) dirs.append(local_base_dir) def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): url = '{}'.format(remote_base_url) util._download_remote(url, fname + extension, local_base_dir / directory) def processing_func(file): thisdata = pd.read_csv(file, names=names, delim_whitespace=True) for name, bad_value in zip(names, badvalues): if name in ['Year', 'Decimal Day', 'Hour']: continue thisdata[name] = thisdata[name].replace(bad_value, np.nan) year = thisdata['Year'][0] day_list = list(thisdata['Decimal Day']) hour_list = list(thisdata['Hour']) len_ = len(thisdata) time_index = convert_datetime(year, day_list, hour_list, len_) thisdata['Time'] = pd.to_datetime(time_index) thisdata = thisdata.set_index('Time') thisdata = thisdata.drop(['Year', 'Decimal Day', 'Hour'], axis=1) return thisdata def convert_datetime(year, day_list, hour_list, len_): datetime_index = [] base_date = dt.datetime(year, 1, 1, 0, 0, 0) for x in range(0, len_): time_delta = dt.timedelta(days=day_list[x] - 1, hours=hour_list[x]) datetime_index.append(base_date + time_delta) return datetime_index return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units)
def corefit(probe, starttime, endtime, try_download=True): """ Read in merged data set Parameters ---------- probe : int, string Helios probe to import data from. Must be 1 or 2. starttime : datetime Interval start time endtime : datetime Interval end time try_download : bool, optional If ``False`` don't try to download data if it is missing locally. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` Data set """ probe = _check_probe(probe) dirs = [] fnames = [] units = OrderedDict([('B instrument', u.dimensionless_unscaled), ('Bx', u.nT), ('By', u.nT), ('Bz', u.nT), ('sigma B', u.nT), ('Ion instrument', u.dimensionless_unscaled), ('Status', u.dimensionless_unscaled), ('Tp_par', u.K), ('Tp_perp', u.K), ('carrot', u.dimensionless_unscaled), ('r_sun', u.AU), ('clat', u.deg), ('clong', u.deg), ('earth_he_angle', u.deg), ('n_p', u.cm**-3), ('vp_x', u.km / u.s), ('vp_y', u.km / u.s), ('vp_z', u.km / u.s), ('vth_p_par', u.km / u.s), ('vth_p_perp', u.km / u.s)]) daylist = util._daysplitinterval(starttime, endtime) for day in daylist: this_date = day[0] # Check that data for this day exists if probe == '1': if this_date < date(1974, 12, 12) or this_date > date(1985, 9, 4): continue if probe == '2': if this_date < date(1976, 1, 17) or this_date > date(1980, 3, 8): continue doy = int(this_date.strftime('%j')) year = this_date.year floc = (path.Path('E1_experiment') / 'New_proton_corefit_data_2017' / 'ascii' / 'helios{}'.format(probe) / '{}'.format(year)) dirs.append(floc) fname = 'h{}_{}_{:03}_corefit'.format(probe, year, doy) fnames.append(fname) extension = '.csv' local_base_dir = path.Path(helios_dir) remote_base_url = 'http://helios-data.ssl.berkeley.edu/data/' def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): remote_url = '{}{}'.format(remote_base_url, directory) util.load(fname + extension, local_base_dir / directory, remote_url) def processing_func(f): return pd.read_csv(f, parse_dates=['Time']) return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, try_download, units=units)
def _mfi(starttime, endtime, version, units=None): """ Import mfi magnetic field data products from WIND. Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : DataFrame """ # Directory relative to main WIND data directory relative_dir = path.Path('mfi') / ('mfi_' + version) # Get directories and filenames dirs = [] fnames = [] daylist = util._daysplitinterval(starttime, endtime) for day in daylist: date = day[0] # Absolute path to local directory for this data file local_dir = relative_dir / str(day[0].year) dirs.append(local_dir) filename = 'wi_' + version + '_mfi_' +\ str(date.year) +\ str(date.month).zfill(2) +\ str(date.day).zfill(2) +\ '_v05' fnames.append(filename) extension = '.cdf' local_base_dir = wind_dir remote_base_url = remote_wind_dir def download_func(remote_base_url, local_base_dir, directory, fname, extension): remote_url = '{}{}'.format(remote_base_url, directory) util.load(fname + extension, local_base_dir / directory, remote_url, guessversion=True) def processing_func(cdf): epoch_dict = {'h0': 'Epoch3', 'h2': 'Epoch'} mag_dict = {'h0': 'B3GSE', 'h2': 'BGSE'} epoch_key = epoch_dict[version] mag_key = mag_dict[version] keys = {mag_key: ['Bx_gse', 'By_gse', 'Bz_gse'], epoch_key: 'Time'} badvalues = {'Bx_gse': -1e+31, 'By_gse': -1e+31, 'Bz_gse': -1e+31} df = util.cdf2df(cdf, index_key=epoch_key, keys=keys, badvalues=badvalues) return df return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units)
def mag_4hz(probe, starttime, endtime, try_download=True): """ Read in 4Hz magnetic field data. Parameters ---------- probe : int, string Helios probe to import data from. Must be 1 or 2. starttime : datetime Interval start time endtime : datetime Interval end time try_download : bool, optional If ``False`` don't try to download data if it is missing locally. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` 4Hz magnetic field data set """ probe = _check_probe(probe) local_base_dir = (path.Path(helios_dir) / 'E2_experiment' / 'Data_Cologne_Nov2016_bestdata' / 'HR' / 'helios{}'.format(probe)) remote_base_url = 'apollo.ssl.berkeley.edu' extension = '.asc' dirs = [] fnames = [] units = OrderedDict([('Bx', u.nT), ('By', u.nT), ('Bz', u.nT), ('|B|', u.nT)]) daylist = util._daysplitinterval(starttime, endtime) for [day, _, _] in daylist: dirs.append('') doy = int(day.strftime('%j')) year = day.year fnames.append('he{}1s{}{:03}'.format(probe, year - 1900, doy)) def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): local_dir = local_base_dir / directory remote_dir = ('pub/helios-data/E2_experiment/' 'Data_Cologne_Nov2016_bestdata/' 'HR/helios{}'.format(probe)) remote_url = 'ftp://' + remote_base_url + '/' + remote_dir original_fname = fname fname = None # Because the filename contains a number between 0 and 24 at the end, # get a list of all the filenames and compare them to the filename # we want with FTP(remote_base_url) as ftp: ftp.login() remote_fnames = ftp.nlst(remote_dir) for remote_fname in remote_fnames: if original_fname in remote_fname: fname = remote_fname break if fname is None: raise util.NoDataError util._download_remote(remote_url, fname, local_base_dir) # Rename to a sensible and deterministic file name downloaded_path = (local_base_dir / fname).with_suffix(extension) new_path = (local_base_dir / original_fname).with_suffix(extension) downloaded_path.rename(new_path) def processing_func(f): # Read in data headings = ['Time', 'Bx', 'By', 'Bz', '|B|'] cols = [0, 4, 5, 6, 7] data = pd.read_csv(f, names=headings, header=None, usecols=cols, delim_whitespace=True) # Convert date info to datetime data['Time'] = pd.to_datetime(data['Time'], format='%Y-%m-%dT%H:%M:%S') data = data.set_index('Time', drop=True) return data return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units, try_download=try_download)
def mag_hires(starttime, endtime, try_download=True): """ Import high resolution magnetic field from Cassini. See http://pds-ppi.igpp.ucla.edu/search/view/?f=yes&id=pds://PPI/CO-E_SW_J_S-MAG-3-RDR-FULL-RES-V1.0 for more information. Cassini Orbiter Magnetometer Calibrated MAG data at the highest time resolution available covering the period 1999-08-16 (DOY 228) to 2016-12-31 (DOY 366). The data are in RTN coordinates prior Cassini's arrival at Saturn, and Kronographic (KRTP) coordinates at Saturn (beginning 2004-05-14, DOY 135). Parameters ---------- starttime : datetime Interval start time. endtime : datetime Interval end time. Returns ------- data : :class:`~sunpy.timeseries.TimeSeries` Requested data """ remote_base_url = ('https://pds-ppi.igpp.ucla.edu/ditdos/download?id=' 'pds://PPI/CO-E_SW_J_S-MAG-3-RDR-FULL-RES-V2.0/DATA') dirs = [] fnames = [] extension = '.TAB' units = OrderedDict([('Bx', u.nT), ('By', u.nT), ('Bz', u.nT), ('coords', u.dimensionless_unscaled)]) local_base_dir = cassini_dir / 'mag' / 'hires' for [day, _, _] in util._daysplitinterval(starttime, endtime): year = day.year if calendar.isleap(year): monthstr = leapmonth2str[day.month] else: monthstr = month2str[day.month] if day < datetime.date(2004, 5, 14): coords = 'RTN' else: coords = 'KRTP' doy = day.strftime('%j') dirs.append(pathlib.Path(str(year)) / monthstr) fnames.append(str(year)[2:] + doy + '_FGM_{}'.format(coords)) def download_func(remote_base_url, local_base_dir, directory, fname, remote_fname, extension): url = remote_base_url + '/' + str(directory) util._download_remote(url, fname + extension, local_base_dir / directory) def processing_func(f): if 'error_message' in f.readline(): f.close() os.remove(f.name) raise util.NoDataError() df = pd.read_csv(f, names=['Time', 'Bx', 'By', 'Bz'], delim_whitespace=True, parse_dates=[0], index_col=0) return df return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, units=units, try_download=try_download)
def corefit(probe, starttime, endtime, verbose=False, try_download=True): """ Read in merged data set Parameters ---------- probe : int, string Helios probe to import data from. Must be 1 or 2. starttime : datetime Interval start time endtime : datetime Interval end time verbose : bool, optional If ``True``, print information as data is loading. Default is ``False``. Returns ------- data : DataFrame Data set """ probe = _check_probe(probe) dirs = [] fnames = [] daylist = util._daysplitinterval(starttime, endtime) for day in daylist: this_date = day[0] # Check that data for this day exists if probe == '1': if this_date < date(1974, 12, 12) or this_date > date(1985, 9, 4): continue if probe == '2': if this_date < date(1976, 1, 17) or this_date > date(1980, 3, 8): continue doy = int(this_date.strftime('%j')) year = this_date.year floc = os.path.join('E1_experiment', 'New_proton_corefit_data_2017', 'ascii', 'helios{}'.format(probe), '{}'.format(year)) dirs.append(floc) fname = 'h{}_{}_{:03}_corefit'.format(probe, year, doy) fnames.append(fname) extension = '.csv' local_base_dir = helios_dir remote_base_url = 'ftp://apollo.ssl.berkeley.edu/pub/helios-data/' def download_func(remote_base_url, local_base_dir, directory, fname, extension): remote_url = '{}{}'.format(remote_base_url, directory) util.load(fname + extension, os.path.join(local_base_dir, directory), remote_url) def processing_func(local_dir, fname): fname = os.path.join(local_dir, fname) return pd.read_csv(fname, parse_dates=['Time']) return util.process(dirs, fnames, extension, local_base_dir, remote_base_url, download_func, processing_func, starttime, endtime, try_download)