def _process_cdas(starttime, endtime, identifier, dataset, base_dir,
units=None, badvalues=None, warn_missing_units=True):
"""
Generic method for downloading cdas data.
"""
relative_dir = pathlib.Path(identifier)
# Directory relative to main WIND data directory
daylist = util._daysplitinterval(starttime, endtime)
dirs = []
fnames = []
dates = []
extension = '.cdf'
for day in daylist:
date = day[0]
dates.append(date)
filename = '{}_{}_{}{:02}{:02}'.format(
dataset, identifier, date.year, date.month, date.day)
fnames.append(filename)
this_relative_dir = relative_dir / str(date.year)
dirs.append(this_relative_dir)
def download_func(remote_base_url, local_base_dir,
directory, fname, remote_fname, extension, date):
return get_data(identifier, date)
def processing_func(cdf):
return util.cdf2df(cdf, index_key='Epoch',
badvalues=badvalues)
return util.process(dirs, fnames, extension, base_dir, '',
download_func, processing_func, starttime,
endtime, units=units, download_info=dates,
warn_missing_units=warn_missing_units)
def mag320ms(probe, starttime, endtime, try_download=True):
"""
Import 320ms cadence magnetic field data.
Parameters
----------
probe : string
Probe number.
starttime : datetime
Start of interval.
endtime : datetime
End of interval.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
Requested data.
"""
fnames = []
dirs = []
extension = '.cdf'
dtimes = util._daysplitinterval(starttime, endtime)
# Loop through years
for dtime in dtimes:
date = dtime[0]
intervalstring = str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2)
filename = 'i8_320msec_mag_' + intervalstring + '_v01'
fnames.append(filename)
# Location of file relative to local directory or remote url
relative_loc = 'imp' + probe + '/mag/mag_320msec_cdaweb/' +\
str(date.year)
dirs.append(relative_loc)
local_base_dir = imp_dir
remote_base_url = imp_url
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
remote_url = remote_base_url + str(directory)
filename = fname + extension
local_dir = local_base_dir / directory
util._download_remote(remote_url, filename, local_dir)
def processing_func(f):
thisdata = util.cdf2df(f, 'Epoch')
thisdata.index.name = 'Time'
return thisdata
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
try_download=try_download)
def _load_wind_cdf(starttime,
endtime,
instrument,
data_product,
fname,
badvalues={}):
relative_dir = path.Path(instrument) / data_product
# Get directories and filenames
dirs = []
fnames = []
daylist = util._daysplitinterval(starttime, endtime)
for day in daylist:
date = day[0]
filename = 'wi_{}_{}{:02}{:02}_v01'.format(fname, date.year,
date.month, date.day)
fnames.append(filename)
local_dir = relative_dir / str(date.year)
dirs.append(local_dir)
extension = '.cdf'
local_base_dir = wind_dir
remote_base_url = remote_wind_dir
def download_func(remote_base_url, local_base_dir, directory, fname,
extension):
remote_url = '{}{}'.format(remote_base_url, directory)
util.load(fname + extension, local_base_dir / directory, remote_url)
def processing_func(cdf):
return util.cdf2df(cdf, 'Epoch', badvalues=badvalues)
return util.process(dirs, fnames, extension, local_base_dir,
remote_base_url, download_func, processing_func,
starttime, endtime)
def fgm_hires(starttime, endtime, try_download=True):
"""
Import high resolution fluxgate magnetometer data.
Parameters
----------
starttime : datetime
Start of interval
endtime : datetime
End of interval
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
Requested data
"""
dtimes = util._daysplitinterval(starttime, endtime)
dirs = []
fnames = []
extension = '.ASC'
local_base_dir = ulysses_dir / 'fgm' / 'hires'
remote_base_url = ulysses_url
units = OrderedDict([('Bx', u.nT), ('By', u.nT),
('Bz', u.nT), ('|B|', u.nT)])
def download_func(remote_base_url, local_base_dir,
directory, fname, remote_fname, extension):
local_dir = local_base_dir / directory
fgm_options = url_options
fgm_options['FILE_NAME'] = fname + extension
fgm_options['FILE_PATH'] = '/ufa/HiRes/VHM-FGM/' + yearstr
remote_url = ulysses_url
for key in fgm_options:
remote_url += key + '=' + fgm_options[key] + '&'
try:
util._load_remote(remote_url, fname + extension, local_dir, 'ascii')
except Exception as err:
return
def processing_func(f):
readargs = {'names': ['year', 'doy', 'hour', 'minute', 'second',
'Bx', 'By', 'Bz', '|B|'],
'delim_whitespace': True}
thisdata = pd.read_table(f, **readargs)
thisdata = _convert_ulysses_time(thisdata)
return thisdata
for dtime in dtimes:
date = dtime[0]
yearstr = date.strftime('%Y')
filename = ('U' + yearstr[-2:] + date.strftime('%j') + 'SH')
fnames.append(filename)
this_relative_dir = local_base_dir / yearstr
dirs.append(this_relative_dir)
return util.process(
dirs, fnames, extension, local_base_dir, remote_base_url,
download_func, processing_func, starttime, endtime, units=units,
try_download=True)
def threedp_pm(starttime, endtime):
"""
Import 'pm' wind data.
3 second time resolution solar wind proton and alpha particle moments from
the PESA LOW sensor, computed on-board the spacecraft
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
"""
# Directory relative to main WIND data directory
relative_dir = path.Path('3dp') / '3dp_pm'
daylist = util._daysplitinterval(starttime, endtime)
ignore = ['TIME']
dirs = []
fnames = []
units = OrderedDict([('P_DENS', u.cm**-3), ('P_VELS', u.km / u.s),
('P_TEMP', u.eV), ('A_DENS', u.cm**-3),
('A_VELS', u.km / u.s), ('A_TEMP', u.eV),
('GAP', u.dimensionless_unscaled), ('E_RANGE', u.eV),
('VALID', u.dimensionless_unscaled),
('VC', u.dimensionless_unscaled),
('SPIN', u.dimensionless_unscaled)])
extension = '.cdf'
for day in daylist:
date = day[0]
this_relative_dir = relative_dir / str(day[0].year)
filename = 'wi_pm_3dp_' +\
str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2) +\
'_v[0-9][0-9]'
fnames.append(filename)
dirs.append(this_relative_dir)
local_base_dir = wind_dir
remote_base_url = remote_wind_dir
def download_func(*args):
util._download_remote_unknown_version(*args)
def processing_func(cdf):
return util.cdf2df(cdf, 'Epoch', ignore=ignore)
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
units=units)
def swe_h3(starttime, endtime):
"""
Import 'h3' solar wind electron data product from WIND.
Electron pitch angle files providing electron fluxes at 30 directional bins
relative to the instantaneous magnetic field direction at 13 different
energy levels
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
"""
relative_dir = os.path.join('swe', 'swe_h3')
# Get directories and filenames
dirs = []
fnames = []
daylist = util._daysplitinterval(starttime, endtime)
for day in daylist:
date = day[0]
filename = 'wi_h3_swe_{}{:02}{:02}_v01'.format(date.year, date.month,
date.day)
fnames.append(filename)
local_dir = os.path.join(relative_dir, str(date.year))
dirs.append(local_dir)
extension = '.cdf'
local_base_dir = wind_dir
remote_base_url = remote_wind_dir
def download_func(remote_base_url, local_base_dir, directory, fname,
extension):
remote_url = '{}{}'.format(remote_base_url, directory)
util.load(fname + extension, os.path.join(local_base_dir, directory),
remote_url)
def processing_func(local_dir, fname):
distkeys = []
for i in range(0, 13):
distkeys.append('f_pitch_E' + str(i).zfill(2))
anglelabels = []
for i in range(0, 30):
anglelabels.append((i + 0.5) * np.pi / 30)
timekey = 'Epoch'
energykey = 'Ve'
cdf = util.load(fname, local_dir, '')
df = util.pitchdist_cdf2df(cdf, distkeys, energykey, timekey,
anglelabels)
return df
return util.process(dirs, fnames, extension, local_base_dir,
remote_base_url, download_func, processing_func,
starttime, endtime)
def fgm_survey(probe, starttime, endtime):
"""
Import fgm survey mode magnetic field data.
Parameters
----------
probe : string
Probe number, must be 1, 2, 3, or 4
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
Imported data.
"""
# Directory relative to main MMS data directory
relative_dir = path.Path('mms' + probe) / 'fgm' / 'srvy' / 'l2'
daylist = util._daysplitinterval(starttime, endtime)
dirs = []
fnames = []
extension = '.cdf'
units = OrderedDict([('mms{}_fgm_mode_srvy_l2'.format(probe),
u.dimensionless_unscaled)])
data = []
for day in daylist:
date = day[0]
this_relative_dir = (relative_dir / str(date.year) /
str(date.month).zfill(2))
filename = 'mms{}_fgm_srvy_l2_{}{:02}{:02}_v4.18.0'.format(
probe, date.year, date.month, date.day)
fnames.append(filename)
dirs.append(this_relative_dir)
remote_base_url = remote_mms_dir
local_base_dir = mms_dir
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
remote_url = remote_base_url + str(directory)
util.load(fname + extension, local_base_dir / directory, remote_url)
def processing_func(cdf):
df = util.cdf2df(cdf, 'Epoch')
return df
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
units=units)
def mag_rtn(starttime, endtime, try_download=True):
"""
Import magnetic field in RTN coordinates from Messenger.
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
"""
# Directory relative to main WIND data directory
relative_dir = path.Path('rtn')
fnames = []
dirs = []
extension = '.cdf'
data = []
ignore = ['Quality_Flag']
daylist = util._daysplitinterval(starttime, endtime)
for day in daylist:
date = day[0]
this_relative_dir = relative_dir / str(date.year)
filename = 'messenger_mag_rtn_' +\
str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2) +\
'_v01'
fnames.append(filename)
dirs.append(this_relative_dir)
local_base_dir = mess_dir
remote_base_url = remote_mess_dir
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
remote_url = remote_base_url + str(directory)
filename = fname + extension
local_dir = local_base_dir / directory
util._download_remote(remote_url, filename, local_dir)
def processing_func(cdf):
thisdata = util.cdf2df(cdf, 'Epoch', ignore=ignore)
return thisdata
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
try_download=try_download)
def swe_h3(starttime, endtime):
"""
Import 'h3' solar wind electron data product from WIND.
Electron pitch angle files providing electron fluxes at 30 directional bins
relative to the instantaneous magnetic field direction at 13 different
energy levels
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
"""
warnings.warn(_deprecation_msg, DeprecationWarning)
relative_dir = path.Path('swe') / 'swe_h3'
# Get directories and filenames
dirs = []
fnames = []
units = OrderedDict([('Angle', u .deg),
('Energy', u.eV),
('df', u.cm/u.s)])
daylist = util._daysplitinterval(starttime, endtime)
for day in daylist:
date = day[0]
filename = 'wi_h3_swe_{}{:02}{:02}_v[0-9][0-9]'.format(
date.year, date.month, date.day)
fnames.append(filename)
local_dir = relative_dir / str(date.year)
dirs.append(local_dir)
extension = '.cdf'
local_base_dir = wind_dir
remote_base_url = remote_wind_dir
distkeys = []
for i in range(0, 13):
distkeys.append('f_pitch_E' + str(i).zfill(2))
anglelabels = []
for i in range(0, 30):
anglelabels.append((i + 0.5) * np.pi / 30)
timekey = 'Epoch'
energykey = 'Ve'
def download_func(*args):
util._download_remote_unknown_version(*args)
def processing_func(cdf):
df = util.pitchdist_cdf2df(cdf, distkeys, energykey, timekey,
anglelabels)
df = df.reset_index(level=['Energy', 'Angle'])
return df
return util.process(dirs, fnames, extension, local_base_dir,
remote_base_url, download_func, processing_func,
starttime, endtime, units=units)
def _ace(starttime,
endtime,
instrument,
product,
fname,
keys=None,
version='01',
badvalues={}):
"""
Generic method for downloading ACE data from cdaweb.
"""
# Directory relative to main WIND data directory
relative_dir = os.path.join(instrument, 'level_2_cdaweb', product)
daylist = util._daysplitinterval(starttime, endtime)
dirs = []
fnames = []
extension = '.cdf'
for day in daylist:
date = day[0]
filename = 'ac_{}_{}{:02}{:02}_v{}'.format(fname, date.year,
date.month, date.day,
version)
fnames.append(filename)
this_relative_dir = os.path.join(relative_dir, str(date.year))
dirs.append(this_relative_dir)
def download_func(remote_base_url, local_base_dir, directory, fname,
extension):
def check_exists():
# Because the version might be different to the one we guess, work
# out the downloaded filename
for f in os.listdir(os.path.join(local_base_dir, directory)):
if (f[:-6] == (fname + extension)[:-6]):
# Return filename with '.cdf' stripped off the end
return f[:-4]
if check_exists() is not None:
return check_exists()
# Now load remotely
util.load(fname + extension,
os.path.join(local_base_dir, directory),
remote_base_url + directory,
guessversion=True)
if check_exists() is not None:
return check_exists()
def processing_func(local_dir, local_fname):
cdf = util.load(local_fname, local_dir, '')
return util.cdf2df(cdf,
index_key='Epoch',
keys=keys,
badvalues=badvalues)
return util.process(dirs, fnames, extension, ace_dir, remote_ace_dir,
download_func, processing_func, starttime, endtime)
def _load(probe, starttime, endtime, instrument, product_id, product_list,
try_download):
dirs = []
fnames = []
download_info = []
for day in util._daysplitinterval(starttime, endtime):
date = day[0]
year = str(date.year)
month = str(date.month).zfill(2)
day = str(date.day).zfill(2)
dirs.append(year)
local_fname = 'C' + probe + '_' + product_id + '__' +\
year + month + day
fnames.append(local_fname)
thisstart = datetime.combine(date, time.min)
thisend = datetime.combine(date, time.max)
download_info.append((thisstart, thisend))
extension = '.cdf'
local_base_dir = cluster_dir / ('c' + probe) / instrument
remote_base_url = csa_url
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension, download_info):
starttime, endtime = download_info
_download(probe, starttime, endtime, instrument, product_id)
def processing_func(file):
for non_empty_var in list(file.cdf_info().keys()):
if 'variable' in non_empty_var.lower():
if len(file.cdf_info()[non_empty_var]) > 0:
var_list = non_empty_var
break
for key in file.cdf_info()[var_list]:
if 'CDF_EPOCH' in file.varget(key, expand=True).values():
index_key = key
break
return util.cdf2xr(file, starttime, endtime, index_key, product_list)
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
try_download=try_download,
units=None,
download_info=download_info)
def _swics(starttime, endtime, names, product, units=None, try_download=True):
data = []
dtimes = util._daysplitinterval(starttime, endtime)
dirs = []
fnames = []
extension = '.dat'
local_base_dir = ulysses_dir / 'swics'
remote_base_url = ulysses_url
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
local_dir = local_base_dir / directory
swics_options = url_options
swics_options['FILE_NAME'] = fname + extension
swics_options['FILE_PATH'] = '/ufa/HiRes/data/swics'
for key in swics_options:
remote_base_url += key + '=' + swics_options[key] + '&'
try:
util._load_remote(remote_base_url, fname + extension, local_dir,
'ascii')
# f = util.load('', local_dir, remote_base_url)
except Exception as err:
return
def processing_func(f):
readargs = {
'names': names,
'delim_whitespace': True,
'na_values': ['******']
}
thisdata = pd.read_csv(f, **readargs)
thisdata = _convert_ulysses_time(thisdata)
return thisdata
# Loop through years
for year in range(starttime.year, endtime.year + 1):
fname = '{}{}'.format(product, str(year)[-2:])
# Local locaiton to download to
dirs.append('')
fnames.append(fname)
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
units=units,
try_download=True)
def mag_h0(starttime, endtime):
"""
Imports magnetic field data from DSCOVR Spacecraft.
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
"""
dirs = []
fnames = []
extension = '.cdf'
ignore = ['Time1_PB5']
daylist = util._daysplitinterval(starttime, endtime)
for day in daylist:
date = day[0]
filename = "dscovr_h0_mag_{}{:02}{:02}_v01".format(
date.year, date.month, date.day)
fnames.append(filename)
this_relative_dir = 'h0/mag/' + str(date.year)
dirs.append(this_relative_dir)
local_base_dir = dscovr_dir
remote_base_url = dscovr_url
def download_func(remote_base_url, local_base_dir,
directory, fname, remote_fname, extension):
remote_url = remote_base_url + str(directory)
filename = fname + extension
local_dir = local_base_dir / directory
util.load(fname + extension,
local_base_dir / directory,
remote_url)
def processing_func(cdf):
df = util.cdf2df(cdf, 'Epoch1', ignore=ignore)
df.index.name = 'Time'
return df
return util.process(dirs, fnames, extension, local_base_dir,
remote_base_url, download_func, processing_func,
starttime, endtime)
def _ace(starttime,
endtime,
instrument,
product,
fname,
units=None,
version='01',
badvalues={}):
"""
Generic method for downloading ACE data from cdaweb.
"""
# Directory relative to main WIND data directory
relative_dir = path.Path(instrument) / 'level_2_cdaweb' / product
daylist = util._daysplitinterval(starttime, endtime)
dirs = []
fnames = []
extension = '.cdf'
for day in daylist:
date = day[0]
filename = 'ac_{}_{}{:02}{:02}_v[0-9][0-9]'.format(
fname, date.year, date.month, date.day)
fnames.append(filename)
this_relative_dir = relative_dir / str(date.year)
dirs.append(this_relative_dir)
def download_func(*args):
util._download_remote_unknown_version(*args)
def processing_func(cdf):
return util.cdf2df(cdf, index_key='Epoch', badvalues=badvalues)
return util.process(dirs,
fnames,
extension,
ace_dir,
remote_ace_dir,
download_func,
processing_func,
starttime,
endtime,
units=units)
def download_files(probe, instrument, data_rate, starttime, endtime,
verbose=True, product_string='', warn_missing_units=True):
"""
Download MMS files.
Parameters
----------
probe : int or str
MMS probe number. Must be in 1-4 inclusive.
instrument : str
MMS instrument. Must be in ``['afg', 'aspoc', 'dfg', 'dsp', 'edi',
'edp', 'fields', 'scm', 'sdp']``
data_rate : str
Data rate. Must be in ``['slow', 'fast', 'brst', 'srvy']``
starttime : ~datetime.datetime
Start time.
endtime : ~datetime.datetime
End time.
verbose : bool
If ``True``, show a progress bar while downloading.
product_string : str
If not empty, this string must be in the filename for it to be
downloaded.
warn_missing_units : bool
If ``True``, warnings will be shown for each variable that does not
have associated units.
Returns
-------
df : :class:`~sunpy.timeseries.GenericTimeSeries`
Requested data.
"""
_validate_instrument(instrument)
probe = _validate_probe(probe)
dirs = []
fnames = []
daylist = util._daysplitinterval(starttime, endtime)
for date, stime, etime in daylist:
files = available_files(probe, instrument, starttime, endtime,
data_rate, product_string)
for file in files:
fname = pathlib.Path(file).stem
if product_string in fname and len(fname):
fnames.append(fname)
dirs.append('')
extension = '.cdf'
local_base_dir = mms_dir / probe / instrument / data_rate
remote_base_url = dl_url
def download_func(remote_base_url, local_base_dir,
directory, fname, remote_fname, extension):
url = remote_base_url + '?file=' + fname + extension
local_fname = os.path.join(local_base_dir, fname + extension)
with requests.get(url, stream=True) as request:
with open(local_fname, 'wb') as fd:
for chunk in tqdm(
request.iter_content(chunk_size=128)):
fd.write(chunk)
def processing_func(cdf):
return util.cdf2df(cdf, index_key='Epoch')
return util.process(dirs, fnames, extension, local_base_dir,
remote_base_url, download_func, processing_func,
starttime, endtime,
warn_missing_units=warn_missing_units)
def mitplasma_h0(probe, starttime, endtime, try_download=True):
"""
Import mit h0 plasma data.
Parameters
----------
probe : string
Probe number.
starttime : datetime
Start of interval.
endtime : datetime
End of interval.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
Requested data.
"""
dirs = []
fnames = []
extension = '.cdf'
units = OrderedDict([('mode', u.dimensionless_unscaled),
('Region', u.dimensionless_unscaled),
('Spacecraft', u.dimensionless_unscaled)])
for date, _, _ in util._daysplitinterval(starttime, endtime):
intervalstring = str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2)
filename = 'i' + probe + '_h0_mitplasma_' + intervalstring + '_v01'
fnames.append(filename)
# Location of file relative to local directory or remote url
relative_loc = 'imp{}/plasma_mit/mitplasma_h0/{}'.format(
probe, date.year)
dirs.append(relative_loc)
local_base_dir = imp_dir
remote_base_url = imp_url
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
remote_url = remote_base_url + str(directory)
filename = fname + extension
local_dir = local_base_dir / directory
util._download_remote(remote_url, filename, local_dir)
def processing_func(f):
thisdata = util.cdf2df(f, 'Epoch')
thisdata.index.name = 'Time'
return thisdata
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
units=units,
try_download=try_download)
def swoops_ions(starttime, endtime, try_download=True):
"""
Import SWOOPS ion data.
Parameters
----------
starttime : datetime
Start of interval
endtime : datetime
End of interval
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
Requested data
"""
dirs = []
fnames = []
extension = '.dat'
local_base_dir = ulysses_dir / 'swoops' / 'ions'
remote_base_url = ulysses_url
units = OrderedDict([('T_p_large', u.K), ('T_p_small', u.K),
('v_t', u.km / u.s), ('v_r', u.km / u.s),
('v_n', u.km / u.s), ('r', u.au), ('n_a', u.cm**-3),
('n_p', u.cm**-3), ('hlat', u.deg), ('hlon', u.deg),
('iqual', u.dimensionless_unscaled)])
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
local_dir = local_base_dir / directory
swoops_options = url_options
year = fname[1:3]
# doy = fname[5:8]
swoops_options['FILE_NAME'] = fname + extension
swoops_options['FILE_PATH'] =\
('/ufa/stageIngestArea/swoops/ions/bamion{}.zip_files'.format(year))
for key in swoops_options:
remote_base_url += key + '=' + swoops_options[key] + '&'
try:
util._load_remote(remote_base_url, fname + extension, local_dir,
'ascii')
# f = util.load('', local_dir, remote_base_url)
except Exception as err:
return
def processing_func(f):
readargs = {
'names': [
'year', 'doy', 'hour', 'minute', 'second', 'r', 'hlat', 'hlon',
'n_p', 'n_a', 'T_p_large', 'T_p_small', 'v_r', 'v_t', 'v_n',
'iqual'
],
'delim_whitespace':
True
}
thisdata = pd.read_csv(f, **readargs)
thisdata = _convert_ulysses_time(thisdata)
return thisdata
# Loop through years
for year in range(starttime.year, endtime.year + 1):
if year == starttime.year:
start_month = starttime.month
else:
start_month = 1
if year == endtime.year:
end_month = endtime.month
else:
end_month = 12
for month in range(start_month, end_month + 1):
doy = datetime(year, month, 1).strftime('%j')
fanme = ('u{}{}bam'.format(str(year)[2:], doy))
# Local locaiton to download to
dirs.append('{}'.format(year))
fnames.append(fanme)
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
units=units,
try_download=try_download)
def mag15s(probe, starttime, endtime, verbose=False, try_download=True):
"""
Import 15s cadence magnetic field data.
Parameters
----------
probe : string
Probe number.
starttime : datetime
Start of interval.
endtime : datetime
End of interval.
verbose : bool, optional
If ``True``, print information whilst loading. Default is ``False``.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
Requested data.
"""
fnames = []
dirs = []
extension = '.asc'
units = OrderedDict([('n points', u.dimensionless_unscaled),
('Source flag', u.dimensionless_unscaled),
('x gse', u.earthRad), ('y gse', u.earthRad),
('z gse', u.earthRad), ('y gsm', u.earthRad),
('z gsm', u.earthRad), ('|B|', u.nT),
('Bx gse', u.nT), ('By gse', u.nT), ('Bz gse', u.nT),
('By gsm', u.nT), ('Bz gsm', u.nT),
('Bxx gse', u.nT**2), ('Byy gse', u.nT**2),
('Bzz gse', u.nT**2), ('Byx gse', u.nT**2),
('Bzx gse', u.nT**2), ('Bzy gse', u.nT**2),
('Time shift', u.s),
('sw flag', u.dimensionless_unscaled)])
dtimes = util._daysplitinterval(starttime, endtime)
# Loop through years
for dtime in dtimes:
startdt = dtime[0]
year = startdt.year
doy = util.dtime2doy(startdt)
if verbose:
print('Loading IMP 15s mag probe {}, {:03d}/{}'.format(
probe, doy, year))
filename = '{}{:03d}_imp{}_mag_15s_v3'.format(year, doy, probe)
fnames.append(filename)
# Location of file relative to local directory or remote url
relative_loc = os.path.join('imp{}'.format(probe), 'mag',
'15s_ascii_v3', str(year))
dirs.append(relative_loc)
local_base_dir = imp_dir
remote_base_url = imp_url
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
remote_url = remote_base_url + str(directory)
filename = fname + extension
local_dir = local_base_dir / directory
util._download_remote(remote_url, filename, local_dir)
# Read in data
def processing_func(f):
readargs = {
'names': [
'Year', 'doy', 'Second', 'Source flag', 'n points', 'x gse',
'y gse', 'z gse', 'y gsm', 'z gsm', '|B|', 'Bx gse', 'By gse',
'Bz gse', 'By gsm', 'Bz gsm', 'Bxx gse', 'Byy gse', 'Bzz gse',
'Byx gse', 'Bzx gse', 'Bzy gse', 'Time shift', 'sw flag'
],
'na_values': [
'9999', '999', '99', '9', '999', '99.99', '99.99', '99.99',
'99.99', '99.99', '9999.99', '9999.99', '9999.99', '9999.99',
'9999.99', '9999.99', '9999.99', '9999.99', '9999.99',
'9999.99', '9999.99', '9999.99', '999.9', '9'
],
'delim_whitespace':
True
}
thisdata = pd.read_table(f, **readargs)
thisdata['Time'] = (pd.to_datetime(thisdata['Year'], format='%Y') +
pd.to_timedelta(thisdata['doy'] - 1, unit='d') +
pd.to_timedelta(thisdata['Second'], unit='s'))
thisdata = thisdata.set_index('Time', drop=False)
thisdata = thisdata.drop(['Year', 'doy', 'Second'], 1)
return thisdata
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
units=units,
try_download=try_download)
def mag_1min(starttime, endtime, coords):
"""
Import 1 minute magnetic field from Cassini.
See http://pds-ppi.igpp.ucla.edu/search/view/?f=yes&id=pds://PPI/CO-E_SW_J_S-MAG-4-SUMM-1MINAVG-V1.0
for more information.
Cassini Orbiter Magnetometer Calibrated MAG data in 1 minute averages
available covering the period 1999-08-16 (DOY 228) to 2016-12-31 (DOY 366).
The data are provided in RTN coordinates throughout the mission, with
Earth, Jupiter, and Saturn centered coordinates for the respective
flybys of those planets.
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
coords : strings
Requested coordinate system. Must be one of
``['KRTP', 'KSM', 'KSO', 'RTN']``
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
Requested data
"""
valid_coords = ['KRTP', 'KSM', 'KSO', 'RTN']
if coords not in valid_coords:
raise ValueError('coords must be one of {}'.format(valid_coords))
base_url = ('http://pds-ppi.igpp.ucla.edu/ditdos/download?'
'id=pds://PPI/CO-E_SW_J_S-MAG-4-SUMM-1MINAVG-V1.0/DATA')
Rs = u.def_unit('saturnRad', 60268 * u.km)
if (coords == 'KRTP'):
units = OrderedDict([('Bx', u.nT), ('By', u.nT), ('Bz', u.nT),
('X', Rs), ('|B|', u.nT), ('Y', u.deg),
('Z', u.deg),
('Local hour', u.dimensionless_unscaled),
('n points', u.dimensionless_unscaled)])
if (coords == 'RTN'):
units = OrderedDict([('Bx', u.nT), ('By', u.nT), ('Bz', u.nT),
('X', u.AU), ('Y', u.AU), ('Z', u.AU),
('|B|', u.nT),
('Local hour', u.dimensionless_unscaled),
('n points', u.dimensionless_unscaled)])
if (coords == 'KSM' or coords == 'KSO'):
units = OrderedDict([('Bx', u.nT), ('By', u.nT), ('Bz', u.nT),
('X', Rs), ('Y', Rs), ('Z', Rs), ('|B|', u.nT),
('Local hour', u.dimensionless_unscaled),
('n points', u.dimensionless_unscaled)])
local_base_dir = cassini_dir / 'mag' / '1min'
dirs = []
fnames = []
extension = '.TAB'
for year in starttime.year, endtime.year:
dirs.append('{}'.format(year))
fnames.append('{}_FGM_{}_1M'.format(year, coords))
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
url = '{}/{}'.format(base_url, year)
util._download_remote(url, fname + extension,
local_base_dir / directory)
def processing_func(f):
if 'error_message' in f.readline():
location = f.name
f.close()
os.remove(f.name)
raise util.NoDataError()
data = pd.read_table(f,
names=[
'Time', 'Bx', 'By', 'Bz', '|B|', 'X', 'Y',
'Z', 'Local hour', 'n points'
],
delim_whitespace=True,
parse_dates=[0],
index_col=0)
f.close()
return data
return util.process(dirs,
fnames,
extension,
local_base_dir,
base_url,
download_func,
processing_func,
starttime,
endtime,
units=units)
def threedp_sfpd(starttime, endtime):
"""
Import 'sfpd' wind data.
12 second energetic electron pitch-angle energy spectra from the foil SST
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
"""
# Directory relative to main WIND data directory
relative_dir = path.Path('3dp') / '3dp_sfpd'
daylist = util._daysplitinterval(starttime, endtime)
data = []
fnames = []
dirs = []
units = OrderedDict([('Energy', u.eV),
('Bx', u.nT),
('By', u.nT),
('Bz', u.nT),
('Pitch angle', u.deg),
('Flux', (u.cm**2 * u.sr * u.eV * u.s)**-1)])
extension = '.cdf'
for (date, _, _) in daylist:
this_relative_dir = relative_dir / str(date.year)
dirs.append(this_relative_dir)
filename = 'wi_sfpd_3dp_{:{dfmt}}_v02'.format(
date, dfmt='%Y%m%d')
fnames.append(filename)
local_base_dir = wind_dir
remote_base_url = remote_wind_dir
def download_func(*args):
util._download_remote_unknown_version(*args)
def processing_func(cdf):
data_today = []
# Loop through each timestamp to build up fluxes
for non_empty_var in list(cdf.cdf_info().keys()):
if 'variable' in non_empty_var.lower():
if len(cdf.cdf_info()[non_empty_var]) > 0:
var_list = non_empty_var
break
index_ = cdf.varget('Epoch')[...]
index_ = cdflib.cdfepoch.breakdown(index_)
index_ = np.asarray([dt.datetime(*x) for x in index_])
energies_ = cdf.varget('ENERGY')[...]
angles_ = cdf.varget('PANGLE')[...]
fluxes_ = cdf.varget('FLUX')[...]
magfield_ = cdf.varget('MAGF')[...]
for i, time in enumerate(index_):
energies = energies_[i, :]
angles = angles_[i, :]
fluxes = fluxes_[i, :, :]
magfield = magfield_[i, :]
index = pd.MultiIndex.from_product(
([time], energies, angles),
names=['Time', 'Energy', 'Pitch angle'])
df = pd.DataFrame(fluxes.ravel(), index=index, columns=['Flux'])
df = df.reset_index(level=['Energy', 'Pitch angle'])
df['Bx'] = magfield[0]
df['By'] = magfield[1]
df['Bz'] = magfield[2]
data_today.append(df)
data_today = pd.concat(data_today)
data_today = data_today.sort_index()
return data_today
return util.process(dirs, fnames, extension, local_base_dir,
remote_base_url, download_func, processing_func,
starttime, endtime, units=units)
def merged(probe, starttime, endtime, try_download=True):
"""
Import merged plasma data. See
ftp://cdaweb.gsfc.nasa.gov/pub/data/imp/imp8/merged/00readme.txt
for information on variables.
Parameters
----------
probe : string
Probe number.
starttime : datetime
Start of interval.
endtime : datetime
End of interval.
verbose : bool, optional
If ``True``, print information whilst loading. Default is
``False``.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
Requested data.
"""
_check_probe(probe, ['8'])
dirs = []
fnames = []
extension = '.asc'
units = OrderedDict([
('sw_flag', u.dimensionless_unscaled), ('x_gse', u.R_earth),
('y_gse', u.R_earth), ('z_gse', u.R_earth), ('y_gsm', u.R_earth),
('z_gsm', u.R_earth),
('Nm', u.dimensionless_unscaled), ('<|B|>', u.nT), ('|<B>|', u.nT),
('<B_lat>', u.nT), ('<B_long>', u.nT), ('Bx_gse', u.nT),
('By_gse', u.nT), ('Bz_gse', u.nT), ('By_gsm', u.nT), ('Bz_gsm', u.nT),
('sigma|B|', u.nT), ('sigma B', u.nT), ('sigma B_x', u.nT),
('sigma B_y', u.nT), ('sigma B_z', u.nT),
('plas_reg', u.dimensionless_unscaled),
('Npp', u.dimensionless_unscaled), ('v_fit', u.km / u.s),
('vx_fit_gse', u.km / u.s), ('vy_fit_gse', u.km / u.s),
('vz_fit_gse', u.km / u.s), ('vlong_fit', u.deg), ('vlat_fit', u.deg),
('np_fit', u.cm**-3), ('Tp_fit', u.K), ('v_mom', u.km / u.s),
('vx_mom_gse', u.km / u.s), ('vy_mom_gse', u.km / u.s),
('vz_mom_gse', u.km / u.s), ('vlong_mom', u.deg), ('vlat_mom', u.deg),
('np_mom', u.cm**-3), ('Tp_mom', u.K),
('FCp', u.dimensionless_unscaled), ('DWp', u.dimensionless_unscaled)
])
local_base_dir = imp_dir / 'imp{}'.format(probe) / 'merged'
remote_base_url = imp_url + 'imp{}/merged'.format(probe)
# Populate directories and filenames
startyear = starttime.year
endyear = endtime.year
for year in range(startyear, endyear + 1):
if year == startyear:
startmonth = starttime.month
else:
startmonth = 1
if year == endyear:
endmonth = endtime.month
else:
endmonth = 12
for month in range(startmonth, endmonth + 1):
intervalstring = str(year) + str(month).zfill(2)
fname = 'imp_min_merge' + intervalstring
fnames.append(fname)
dirs.append('')
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
filename = fname + extension
local_dir = path.Path(local_base_dir) / directory
util._download_remote(remote_base_url, filename, local_dir)
def processing_func(f):
readargs = {
'names': [
'Year', 'doy', 'Hour', 'Minute', 'sw_flag', 'x_gse', 'y_gse',
'z_gse', 'y_gsm', 'z_gsm', 'Nm', 'FCm', 'DWm', '<|B|>',
'|<B>|', '<B_lat>', '<B_long>', 'Bx_gse', 'By_gse', 'Bz_gse',
'By_gsm', 'Bz_gsm', 'sigma|B|', 'sigma B', 'sigma B_x',
'sigma B_y', 'sigma B_z', 'plas_reg', 'Npp', 'FCp', 'DWp',
'v_fit', 'vx_fit_gse', 'vy_fit_gse', 'vz_fit_gse', 'vlong_fit',
'vlat_fit', 'np_fit', 'Tp_fit', 'v_mom', 'vx_mom_gse',
'vy_mom_gse', 'vz_mom_gse', 'vlong_mom', 'vlat_mom', 'np_mom',
'Tp_mom'
],
'na_values': [
'9999', '999', '99', '99', '9', '9999.99', '9999.99',
'9999.99', '9999.99', '9999.99', '9', '99', '9.99', '9999.99',
'9999.99', '9999.99', '9999.99', '9999.99', '9999.99',
'9999.99', '9999.99', '9999.99', '9999.99', '9999.99',
'9999.99', '9999.99', '9999.99', '9', '9', '99', '9.99',
'9999.9', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9',
'9999.9', '9999999.', '9999.9', '9999.9', '9999.9', '9999.9',
'9999.9', '9999.9', '9999.9', '9999999.'
],
'delim_whitespace':
True
}
# Read in data
data = pd.read_table(f, **readargs)
data['Time'] = (pd.to_datetime(data['Year'], format='%Y') +
pd.to_timedelta(data['doy'] - 1, unit='d') +
pd.to_timedelta(data['Hour'], unit='h') +
pd.to_timedelta(data['Minute'], unit='m'))
data = data.drop(['Year', 'doy', 'Hour', 'Minute', 'FCm', 'DWm'],
axis=1)
data = data.set_index('Time', drop=True)
return data
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
units=units,
try_download=try_download)
def download_files(probe,
instrument,
data_rate,
starttime,
endtime,
product_list=None,
verbose=True,
product_string='',
warn_missing_units=True,
want_xr=False):
"""
Download MMS files.
Parameters
----------
probe : int or str
MMS probe number. Must be in 1-4 inclusive.
instrument : str
MMS instrument. Must be in ``['afg', 'aspoc', 'dfg', 'dsp', 'edi',
'edp', 'fields', 'scm', 'sdp']``
data_rate : str
Data rate. Must be in ``['slow', 'fast', 'brst', 'srvy']``
starttime : ~datetime.datetime
Start time.
endtime : ~datetime.datetime
End time.
verbose : bool, optional
If ``True``, show a progress bar while downloading.
product_string : str, optional
If not empty, this string must be in the filename for it to be
downloaded.
warn_missing_units : bool, optional
If ``True``, warnings will be shown for each variable that does not
have associated units.
Returns
-------
df : :class:`~sunpy.timeseries.GenericTimeSeries`
Requested data.
"""
_validate_instrument(instrument)
probe = _validate_probe(probe)
dirs = []
fnames = []
daylist = util._daysplitinterval(starttime, endtime)
for date, stime, etime in daylist:
files = available_files(probe, instrument, starttime, endtime,
data_rate)
for file in files:
fname = pathlib.Path(file).stem
# Make sure that only the needed files will be loaded (i.e., in the queried time interval)
namestr = [
j for i, j in enumerate(fname.split('_'))
if j.startswith(endtime.strftime('%Y%m%d'))
]
# Select only one 'mec' (metadata) file (here 'epht89d') to avoid redundancy
if instrument == 'mec':
if 'epht89d' in fname:
fnames.append(fname)
dirs.append('')
elif product_string in fname and len(
fname
) and namestr and namestr[0] < endtime.strftime('%Y%m%d%H%M%S'):
fnames.append(fname)
dirs.append('')
else:
pass
extension = '.cdf'
local_base_dir = mms_dir / probe / instrument / data_rate
remote_base_url = dl_url
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
url = remote_base_url + '?file=' + fname + extension
local_fname = os.path.join(local_base_dir, fname + extension)
with requests.get(url, stream=True) as request:
with open(local_fname, 'wb') as fd:
for chunk in tqdm(request.iter_content(chunk_size=128)):
fd.write(chunk)
def processing_func(cdf):
if want_xr:
return util.cdf2xr(cdf, starttime, endtime, 'Epoch', product_list)
else:
return util.cdf2df(cdf, starttime, endtime, 'Epoch', product_list)
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
want_xr,
warn_missing_units=warn_missing_units)
def fgm(probe, rate, coords, starttime, endtime):
"""
Import fgm magnetic field data from THEMIS.
Parameters
----------
probe : string
Alowed values are [a, b, c, d, e].
rate : string
Date rate to return. Allowed values are [e, h, l, s].
coords : string
Magnetic field co-ordinate system. Allowed values are
[dsl, gse, gsm, ssl]. NOTE: Add link to co-ordinate system
descriptions.
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
"""
valid_rates = ['e', 'h', 'l', 's']
valid_coords = ['dsl', 'gse', 'gsm', 'ssl']
_validate_probe(probe)
if rate not in valid_rates:
raise ValueError(('rate argument %s is not in list of allowed'
'rates: %s') % (rate, valid_rates))
if coords not in valid_coords:
raise ValueError(('coords argument %s is not in list of allowed'
'co-ordinate systems: %s') % (rate, valid_rates))
# Directory relative to main THEMIS data directory
fgm_dir = path.Path('th' + probe) / 'l2' / 'fgm'
daylist = util._daysplitinterval(starttime, endtime)
dirs = []
fnames = []
extension = '.cdf'
units = OrderedDict([('|B|', u.nT), ('Bz_dsl', u.nT), ('By_dsl', u.nT),
('Bx_dsl', u.nT),
('th{}_fgm_fg{}_quality'.format(probe, rate),
u.dimensionless_unscaled)])
for day in daylist:
date = day[0]
filename = 'th{}_l2_fgm_{}{:02}{:02}_v01'.format(
probe, date.year, date.month, date.day)
fnames.append(filename)
this_relative_dir = fgm_dir / str(date.year)
dirs.append(this_relative_dir)
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
remote_url = remote_base_url + str(directory)
# Now load remotely
util.load(fname + extension, local_base_dir / directory, remote_url)
def processing_func(cdf, **kwargs):
probe = kwargs.pop('probe')
rate = kwargs.pop('rate')
kwargs.pop('coords')
probestr = 'th' + probe
ratestr = '_fg' + rate + '_'
df = util.cdf2df(cdf, probestr + ratestr + 'time', dtimeindex=False)
df = df.set_index(pd.to_datetime(df.index.values, unit='s'))
df.index.name = 'Time'
return df
processing_kwargs = {'probe': probe, 'rate': rate, 'coords': coords}
return util.process(dirs,
fnames,
extension,
artemis_dir,
remote_themis_dir,
download_func,
processing_func,
starttime,
endtime,
units=units,
processing_kwargs=processing_kwargs)
def mag_ness(probe, starttime, endtime, try_download=True):
"""
Read in 6 second magnetic field data.
Parameters
----------
probe : int, string
Helios probe to import data from. Must be 1 or 2.
starttime : datetime
Interval start time
endtime : datetime
Interval end time
try_download : bool, optional
If ``False`` don't try to download data if it is missing locally.
Returns
-------
data : DataFrame
6 second magnetic field data set
"""
probe = _check_probe(probe)
remote_base_url = ('http://helios-data.ssl.berkeley.edu/data/'
'E3_experiment/helios{}_6sec_ness/'.format(probe))
local_base_dir = (path.Path(helios_dir) / 'E3_experiment' /
'helios{}_6sec_ness'.format(probe))
dirs = []
fnames = []
extension = '.asc'
units = OrderedDict([('probe', u.dimensionless_unscaled),
('naverage', u.dimensionless_unscaled), ('Bx', u.nT),
('By', u.nT), ('Bz', u.nT), ('|B|', u.nT),
('sigma_Bx', u.nT), ('sigma_By', u.nT),
('sigma_Bz', u.nT)])
daylist = util._daysplitinterval(starttime, endtime)
for [day, _, _] in daylist:
year = day.year
doy = int(day.strftime('%j'))
dirs.append('{}'.format(day.year))
fnames.append('h{}{}{:03}'.format(probe, year - 1900, doy))
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
remote_url = remote_base_url + str(directory)
local_dir = local_base_dir / directory
filename = fname + extension
try:
util._download_remote(remote_url, filename, local_dir)
except URLError:
raise util.NoDataError
def processing_func(f):
# Read in data
headings = [
'probe', 'year', 'doy', 'hour', 'minute', 'second', 'naverage',
'Bx', 'By', 'Bz', '|B|', 'sigma_Bx', 'sigma_By', 'sigma_Bz'
]
colspecs = [(1, 2), (2, 4), (4, 7), (7, 9), (9, 11), (11, 13),
(13, 15), (15, 22), (22, 29), (29, 36), (36, 42), (42, 48),
(48, 54), (54, 60)]
data = pd.read_fwf(f, names=headings, header=None, colspecs=colspecs)
# Process data
data['year'] += 1900
# Convert date info to datetime
data['Time'] = pd.to_datetime(data['year'], format='%Y') + \
pd.to_timedelta(data['doy'] - 1, unit='d') + \
pd.to_timedelta(data['hour'], unit='h') + \
pd.to_timedelta(data['minute'], unit='m') + \
pd.to_timedelta(data['second'], unit='s')
data = data.drop(['year', 'doy', 'hour', 'minute', 'second'], axis=1)
data = data.set_index('Time', drop=False)
return data
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
units=units,
try_download=try_download)
def low(starttime, endtime, try_download=True):
"""
Import data from OMNI Web Interface.
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
"""
# Directory relative to main OMNI data directory
local_base_dir = omni_dir / 'low'
remote_base_url = omni_url + '/low_res_omni'
dirs = []
fnames = []
names = ['Year', 'Decimal Day', 'Hour', 'Bartels Rotation Number',
'ID IMF Spacecraft', 'ID SW Plasma Spacecraft',
'points(IMF Average)', 'points(Plasma Average)',
'|B|', 'Magnitude of Avg Field Vector',
'Lat. Angle of Aver. Field Vector',
'Long. Angle of Aver. Field Vector', 'Bx GSE, GSM', 'By GSE',
'Bz GSE', 'By GSM', 'Bz GSM', 'sigma |B|', 'sigma B', 'sigma Bx',
'sigma By', 'sigma Bz', 'Proton Temperature',
'Proton Density', 'Plasma Flow Speed', 'Plasma Flow Long. Angle',
'Plasma Flow Lat. Angle', 'Na/Np', 'Flow Pressure', 'sigma T',
'sigma N', 'sigma V', 'sigma phi V', 'sigma theta V',
'sigma Na/Np', 'Electric Field', 'Plasma Beta',
'Alfven Mach Number', 'Kp', 'R', 'DST Index', 'AE Index',
'Proton Flux > 1MeV', 'Proton Flux > 2MeV',
'Proton Flux > 4MeV', 'Proton Flux > 10MeV',
'Proton Flux > 30MeV',
'Proton Flux > 60MeV', 'flag', 'ap index',
'f10.7 index', 'PC(N) index', 'AL index (Kyoto)',
'AU index (Kyoto)', 'Magnetosonic Mach No.']
badvalues = [np.nan, np.nan, np.nan, 9999, 99, 99, 999, 999, 999.9, 999.9,
999.9, 999.9, 999.9, 999.9, 999.9, 999.9, 999.9, 999.9, 999.9,
999.9, 999.9, 999.9, 9999999., 999.9, 9999., 999.9, 999.9,
9.999, 99.99, 9999999., 999.9, 9999., 999.9, 999.9, 9.999,
999.99, 999.99, 999.9, 99, 999, 99999, 9999, 999999.99,
99999.99, 99999.99, 99999.99, 99999.99, 99999.99, np.nan,
999, 999.9, 999.9, 99999, 99999, 99.9]
sfu = u.def_unit('sfu', 10**-22 * u.m**-2 * u.Hz**-1)
units = OrderedDict([('Bartels Rotation Number', u.dimensionless_unscaled),
('ID IMF Spacecraft', u.dimensionless_unscaled),
('ID SW Plasma Spacecraft', u.dimensionless_unscaled),
('points(IMF Average)', u.dimensionless_unscaled),
('points(Plasma Average)', u.dimensionless_unscaled),
('|B|', u.nT),
('Magnitude of Avg Field Vector', u.nT),
('Lat. Angle of Aver. Field Vector', u.deg),
('Long. Angle of Aver. Field Vector', u.deg),
('Bx GSE, GSM', u.nT),
('By GSE', u.nT),
('Bz GSE', u.nT),
('By GSM', u.nT),
('Bz GSM', u.nT),
('sigma |B|', u.nT),
('sigma B', u.nT),
('sigma Bx', u.nT),
('sigma By', u.nT),
('sigma Bz', u.nT),
('Proton Temperature', u.K),
('Proton Density', u.cm**-3),
('Plasma Flow Speed', u.km / u.s),
('Plasma Flow Long. Angle', u.deg),
('Plasma Flow Lat. Angle', u.deg),
('Na/Np', u.dimensionless_unscaled),
('Flow Pressure', u.nPa),
('sigma T', u.K),
('sigma N', u.cm**-3),
('sigma V', u.km / u.s),
('sigma phi V', u.deg),
('sigma theta V', u.deg),
('sigma Na/Np', u.dimensionless_unscaled),
('Electric Field', u.mV / u.m),
('Plasma Beta', u.dimensionless_unscaled),
('Alfven Mach Number', u.dimensionless_unscaled),
('Kp', u.dimensionless_unscaled),
('R', u.dimensionless_unscaled),
('AE Index', u.nT),
('DST Index', u.nT),
('Proton Flux > 1MeV', u.cm**-2),
('Proton Flux > 2MeV', u.cm**-2),
('Proton Flux > 4MeV', u.cm**-2),
('Proton Flux > 10MeV', u.cm**-2),
('Proton Flux > 30MeV', u.cm**-2),
('Proton Flux > 60MeV', u.cm**-2),
('flag', u.dimensionless_unscaled),
('ap index', u.nT),
('PC(N) index', u.dimensionless_unscaled),
('AL index (Kyoto)', u.nT),
('AU index (Kyoto)', u.nT),
('Magnetosonic Mach No.', u.dimensionless_unscaled),
('f10.7 index', sfu)])
extension = '.dat'
for year in range(starttime.year, endtime.year + 1):
fnames.append("omni2_{}".format(year))
dirs.append(local_base_dir)
def download_func(remote_base_url, local_base_dir,
directory, fname, remote_fname, extension):
url = '{}'.format(remote_base_url)
util._download_remote(url,
fname + extension,
local_base_dir / directory)
def processing_func(file):
thisdata = pd.read_csv(file, names=names,
delim_whitespace=True)
for name, bad_value in zip(names, badvalues):
if name in ['Year', 'Decimal Day', 'Hour']:
continue
thisdata[name] = thisdata[name].replace(bad_value, np.nan)
year = thisdata['Year'][0]
day_list = list(thisdata['Decimal Day'])
hour_list = list(thisdata['Hour'])
len_ = len(thisdata)
time_index = convert_datetime(year, day_list, hour_list, len_)
thisdata['Time'] = pd.to_datetime(time_index)
thisdata = thisdata.set_index('Time')
thisdata = thisdata.drop(['Year', 'Decimal Day', 'Hour'], axis=1)
return thisdata
def convert_datetime(year, day_list, hour_list, len_):
datetime_index = []
base_date = dt.datetime(year, 1, 1, 0, 0, 0)
for x in range(0, len_):
time_delta = dt.timedelta(days=day_list[x] - 1, hours=hour_list[x])
datetime_index.append(base_date + time_delta)
return datetime_index
return util.process(dirs, fnames, extension, local_base_dir,
remote_base_url, download_func, processing_func,
starttime, endtime, units=units)
def corefit(probe, starttime, endtime, try_download=True):
"""
Read in merged data set
Parameters
----------
probe : int, string
Helios probe to import data from. Must be 1 or 2.
starttime : datetime
Interval start time
endtime : datetime
Interval end time
try_download : bool, optional
If ``False`` don't try to download data if it is missing locally.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
Data set
"""
probe = _check_probe(probe)
dirs = []
fnames = []
units = OrderedDict([('B instrument', u.dimensionless_unscaled),
('Bx', u.nT), ('By', u.nT), ('Bz', u.nT),
('sigma B', u.nT),
('Ion instrument', u.dimensionless_unscaled),
('Status', u.dimensionless_unscaled), ('Tp_par', u.K),
('Tp_perp', u.K),
('carrot', u.dimensionless_unscaled), ('r_sun', u.AU),
('clat', u.deg), ('clong', u.deg),
('earth_he_angle', u.deg), ('n_p', u.cm**-3),
('vp_x', u.km / u.s), ('vp_y', u.km / u.s),
('vp_z', u.km / u.s), ('vth_p_par', u.km / u.s),
('vth_p_perp', u.km / u.s)])
daylist = util._daysplitinterval(starttime, endtime)
for day in daylist:
this_date = day[0]
# Check that data for this day exists
if probe == '1':
if this_date < date(1974, 12, 12) or this_date > date(1985, 9, 4):
continue
if probe == '2':
if this_date < date(1976, 1, 17) or this_date > date(1980, 3, 8):
continue
doy = int(this_date.strftime('%j'))
year = this_date.year
floc = (path.Path('E1_experiment') / 'New_proton_corefit_data_2017' /
'ascii' / 'helios{}'.format(probe) / '{}'.format(year))
dirs.append(floc)
fname = 'h{}_{}_{:03}_corefit'.format(probe, year, doy)
fnames.append(fname)
extension = '.csv'
local_base_dir = path.Path(helios_dir)
remote_base_url = 'http://helios-data.ssl.berkeley.edu/data/'
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
remote_url = '{}{}'.format(remote_base_url, directory)
util.load(fname + extension, local_base_dir / directory, remote_url)
def processing_func(f):
return pd.read_csv(f, parse_dates=['Time'])
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
try_download,
units=units)
def _mfi(starttime, endtime, version, units=None):
"""
Import mfi magnetic field data products from WIND.
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
"""
# Directory relative to main WIND data directory
relative_dir = path.Path('mfi') / ('mfi_' + version)
# Get directories and filenames
dirs = []
fnames = []
daylist = util._daysplitinterval(starttime, endtime)
for day in daylist:
date = day[0]
# Absolute path to local directory for this data file
local_dir = relative_dir / str(day[0].year)
dirs.append(local_dir)
filename = 'wi_' + version + '_mfi_' +\
str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2) +\
'_v05'
fnames.append(filename)
extension = '.cdf'
local_base_dir = wind_dir
remote_base_url = remote_wind_dir
def download_func(remote_base_url, local_base_dir, directory, fname,
extension):
remote_url = '{}{}'.format(remote_base_url, directory)
util.load(fname + extension,
local_base_dir / directory,
remote_url,
guessversion=True)
def processing_func(cdf):
epoch_dict = {'h0': 'Epoch3', 'h2': 'Epoch'}
mag_dict = {'h0': 'B3GSE', 'h2': 'BGSE'}
epoch_key = epoch_dict[version]
mag_key = mag_dict[version]
keys = {mag_key: ['Bx_gse', 'By_gse', 'Bz_gse'], epoch_key: 'Time'}
badvalues = {'Bx_gse': -1e+31, 'By_gse': -1e+31, 'Bz_gse': -1e+31}
df = util.cdf2df(cdf,
index_key=epoch_key,
keys=keys,
badvalues=badvalues)
return df
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
units=units)
def mag_4hz(probe, starttime, endtime, try_download=True):
"""
Read in 4Hz magnetic field data.
Parameters
----------
probe : int, string
Helios probe to import data from. Must be 1 or 2.
starttime : datetime
Interval start time
endtime : datetime
Interval end time
try_download : bool, optional
If ``False`` don't try to download data if it is missing locally.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
4Hz magnetic field data set
"""
probe = _check_probe(probe)
local_base_dir = (path.Path(helios_dir) / 'E2_experiment' /
'Data_Cologne_Nov2016_bestdata' / 'HR' /
'helios{}'.format(probe))
remote_base_url = 'apollo.ssl.berkeley.edu'
extension = '.asc'
dirs = []
fnames = []
units = OrderedDict([('Bx', u.nT), ('By', u.nT), ('Bz', u.nT),
('|B|', u.nT)])
daylist = util._daysplitinterval(starttime, endtime)
for [day, _, _] in daylist:
dirs.append('')
doy = int(day.strftime('%j'))
year = day.year
fnames.append('he{}1s{}{:03}'.format(probe, year - 1900, doy))
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
local_dir = local_base_dir / directory
remote_dir = ('pub/helios-data/E2_experiment/'
'Data_Cologne_Nov2016_bestdata/'
'HR/helios{}'.format(probe))
remote_url = 'ftp://' + remote_base_url + '/' + remote_dir
original_fname = fname
fname = None
# Because the filename contains a number between 0 and 24 at the end,
# get a list of all the filenames and compare them to the filename
# we want
with FTP(remote_base_url) as ftp:
ftp.login()
remote_fnames = ftp.nlst(remote_dir)
for remote_fname in remote_fnames:
if original_fname in remote_fname:
fname = remote_fname
break
if fname is None:
raise util.NoDataError
util._download_remote(remote_url, fname, local_base_dir)
# Rename to a sensible and deterministic file name
downloaded_path = (local_base_dir / fname).with_suffix(extension)
new_path = (local_base_dir / original_fname).with_suffix(extension)
downloaded_path.rename(new_path)
def processing_func(f):
# Read in data
headings = ['Time', 'Bx', 'By', 'Bz', '|B|']
cols = [0, 4, 5, 6, 7]
data = pd.read_csv(f,
names=headings,
header=None,
usecols=cols,
delim_whitespace=True)
# Convert date info to datetime
data['Time'] = pd.to_datetime(data['Time'], format='%Y-%m-%dT%H:%M:%S')
data = data.set_index('Time', drop=True)
return data
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
units=units,
try_download=try_download)
def mag_hires(starttime, endtime, try_download=True):
"""
Import high resolution magnetic field from Cassini.
See http://pds-ppi.igpp.ucla.edu/search/view/?f=yes&id=pds://PPI/CO-E_SW_J_S-MAG-3-RDR-FULL-RES-V1.0
for more information.
Cassini Orbiter Magnetometer Calibrated MAG data at the highest time
resolution available covering the period 1999-08-16 (DOY 228) to
2016-12-31 (DOY 366).
The data are in RTN coordinates prior Cassini's arrival at Saturn, and
Kronographic (KRTP) coordinates at Saturn (beginning 2004-05-14, DOY 135).
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
Requested data
"""
remote_base_url = ('https://pds-ppi.igpp.ucla.edu/ditdos/download?id='
'pds://PPI/CO-E_SW_J_S-MAG-3-RDR-FULL-RES-V2.0/DATA')
dirs = []
fnames = []
extension = '.TAB'
units = OrderedDict([('Bx', u.nT), ('By', u.nT), ('Bz', u.nT),
('coords', u.dimensionless_unscaled)])
local_base_dir = cassini_dir / 'mag' / 'hires'
for [day, _, _] in util._daysplitinterval(starttime, endtime):
year = day.year
if calendar.isleap(year):
monthstr = leapmonth2str[day.month]
else:
monthstr = month2str[day.month]
if day < datetime.date(2004, 5, 14):
coords = 'RTN'
else:
coords = 'KRTP'
doy = day.strftime('%j')
dirs.append(pathlib.Path(str(year)) / monthstr)
fnames.append(str(year)[2:] + doy + '_FGM_{}'.format(coords))
def download_func(remote_base_url, local_base_dir,
directory, fname, remote_fname, extension):
url = remote_base_url + '/' + str(directory)
util._download_remote(url, fname + extension,
local_base_dir / directory)
def processing_func(f):
if 'error_message' in f.readline():
f.close()
os.remove(f.name)
raise util.NoDataError()
df = pd.read_csv(f, names=['Time', 'Bx', 'By', 'Bz'],
delim_whitespace=True,
parse_dates=[0], index_col=0)
return df
return util.process(dirs, fnames, extension, local_base_dir,
remote_base_url, download_func, processing_func,
starttime, endtime, units=units,
try_download=try_download)
def corefit(probe, starttime, endtime, verbose=False, try_download=True):
"""
Read in merged data set
Parameters
----------
probe : int, string
Helios probe to import data from. Must be 1 or 2.
starttime : datetime
Interval start time
endtime : datetime
Interval end time
verbose : bool, optional
If ``True``, print information as data is loading.
Default is ``False``.
Returns
-------
data : DataFrame
Data set
"""
probe = _check_probe(probe)
dirs = []
fnames = []
daylist = util._daysplitinterval(starttime, endtime)
for day in daylist:
this_date = day[0]
# Check that data for this day exists
if probe == '1':
if this_date < date(1974, 12, 12) or this_date > date(1985, 9, 4):
continue
if probe == '2':
if this_date < date(1976, 1, 17) or this_date > date(1980, 3, 8):
continue
doy = int(this_date.strftime('%j'))
year = this_date.year
floc = os.path.join('E1_experiment', 'New_proton_corefit_data_2017',
'ascii', 'helios{}'.format(probe),
'{}'.format(year))
dirs.append(floc)
fname = 'h{}_{}_{:03}_corefit'.format(probe, year, doy)
fnames.append(fname)
extension = '.csv'
local_base_dir = helios_dir
remote_base_url = 'ftp://apollo.ssl.berkeley.edu/pub/helios-data/'
def download_func(remote_base_url, local_base_dir, directory, fname,
extension):
remote_url = '{}{}'.format(remote_base_url, directory)
util.load(fname + extension, os.path.join(local_base_dir, directory),
remote_url)
def processing_func(local_dir, fname):
fname = os.path.join(local_dir, fname)
return pd.read_csv(fname, parse_dates=['Time'])
return util.process(dirs, fnames, extension, local_base_dir,
remote_base_url, download_func, processing_func,
starttime, endtime, try_download)
