def _process_cdas(starttime, endtime, identifier, dataset, base_dir,
units=None, badvalues=None, warn_missing_units=True):
"""
Generic method for downloading cdas data.
"""
relative_dir = pathlib.Path(identifier)
# Directory relative to main WIND data directory
daylist = util._daysplitinterval(starttime, endtime)
dirs = []
fnames = []
dates = []
extension = '.cdf'
for day in daylist:
date = day[0]
dates.append(date)
filename = '{}_{}_{}{:02}{:02}'.format(
dataset, identifier, date.year, date.month, date.day)
fnames.append(filename)
this_relative_dir = relative_dir / str(date.year)
dirs.append(this_relative_dir)
def download_func(remote_base_url, local_base_dir,
directory, fname, remote_fname, extension, date):
return get_data(identifier, date)
def processing_func(cdf):
return util.cdf2df(cdf, index_key='Epoch',
badvalues=badvalues)
return util.process(dirs, fnames, extension, base_dir, '',
download_func, processing_func, starttime,
endtime, units=units, download_info=dates,
warn_missing_units=warn_missing_units)
def _load_wind_cdf(starttime,
endtime,
instrument,
data_product,
fname,
badvalues={}):
relative_dir = path.Path(instrument) / data_product
# Get directories and filenames
dirs = []
fnames = []
daylist = util._daysplitinterval(starttime, endtime)
for day in daylist:
date = day[0]
filename = 'wi_{}_{}{:02}{:02}_v01'.format(fname, date.year,
date.month, date.day)
fnames.append(filename)
local_dir = relative_dir / str(date.year)
dirs.append(local_dir)
extension = '.cdf'
local_base_dir = wind_dir
remote_base_url = remote_wind_dir
def download_func(remote_base_url, local_base_dir, directory, fname,
extension):
remote_url = '{}{}'.format(remote_base_url, directory)
util.load(fname + extension, local_base_dir / directory, remote_url)
def processing_func(cdf):
return util.cdf2df(cdf, 'Epoch', badvalues=badvalues)
return util.process(dirs, fnames, extension, local_base_dir,
remote_base_url, download_func, processing_func,
starttime, endtime)
def mag_hires(starttime, endtime):
"""
Import high resolution magnetic field from Cassini.
See http://pds-ppi.igpp.ucla.edu/search/view/?f=yes&id=pds://PPI/CO-E_SW_J_S-MAG-3-RDR-FULL-RES-V1.0
for more information.
Cassini Orbiter Magnetometer Calibrated MAG data at the highest time
resolution available covering the period 1999-08-16 (DOY 228) to
2016-12-31 (DOY 366).
The data are in RTN coordinates prior Cassini's arrival at Saturn, and
Kronographic (KRTP) coordinates at Saturn (beginning 2004-05-14, DOY 135).
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
Requested data
"""
base_url = ('http://pds-ppi.igpp.ucla.edu/ditdos/download?id='
'pds://PPI/CO-E_SW_J_S-MAG-3-RDR-FULL-RES-V1.0/DATA')
daylist = util._daysplitinterval(starttime, endtime)
data = []
for [day, stime, etime] in daylist:
year = day.year
url = '{}/{}'.format(base_url, year)
if calendar.isleap(year):
monthstr = leapmonth2str[day.month]
else:
monthstr = month2str[day.month]
url = '{}/{}'.format(url, monthstr)
doy = day.strftime('%j')
local_dir = cassini_dir / 'mag' / 'hires' / str(year)
# No way to work out co-ordinates, so guess Kronian and then RTN
try:
coords = 'KRTP'
df = _mag_hires_helper(year, doy, local_dir, url, coords)
except RuntimeError:
try:
coords = 'RTN'
df = _mag_hires_helper(year, doy, local_dir, url, coords)
except RuntimeError:
continue
df['coords'] = coords
data.append(df)
return util.timefilter(data, starttime, endtime)
def _load(probe, starttime, endtime, instrument, product_id, cdfkeys):
daylist = util._daysplitinterval(starttime, endtime)
data = []
for day in daylist:
date = day[0]
year = str(date.year)
month = str(date.month).zfill(2)
day = str(date.day).zfill(2)
local_dir = cluster_dir / ('c' + probe) / instrument / year
local_fname = 'C' + probe + '_' + product_id + '__' +\
year + month + day + '.cdf'
local_file = local_dir / local_fname
# If we don't have local file download it
if not local_file.exists():
thisstart = datetime.combine(date, time.min)
thisend = datetime.combine(date, time.max)
try:
_download(probe, thisstart, thisend, instrument, product_id)
except Exception as err:
print(str(err), '\n')
continue
from pycdf import pycdf
cdf = pycdf.CDF(os.path.join(local_dir, local_fname))
for key, value in cdfkeys.items():
if value == 'Time':
index_key = key
break
data.append(util.cdf2df(cdf, index_key, cdfkeys))
if len(data) == 0:
raise RuntimeError('No data available to download during requested '
'times')
return util.timefilter(data, starttime, endtime)
def mitplasma_h0(probe, starttime, endtime):
"""
Import mit h0 plasma data.
Parameters
----------
probe : string
Probe number.
starttime : datetime
Start of interval.
endtime : datetime
End of interval.
Returns
-------
data : DataFrame
Requested data.
"""
data = []
dtimes = util._daysplitinterval(starttime, endtime)
# Loop through years
for dtime in dtimes:
date = dtime[0]
intervalstring = str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2)
filename = 'i' + probe + '_h0_mitplasma_' + intervalstring + '_v01.cdf'
# Location of file relative to local directory or remote url
relative_loc = 'imp' + probe + '/plasma_mit/mitplasma_h0/' +\
str(date.year)
local_dir = os.path.join(imp_dir, relative_loc)
remote_url = imp_url + relative_loc
cdf = util.load(filename, local_dir, remote_url)
keys = {
'EW_flowangle_best': 'EW_flowangle_best',
'EW_flowangle_mom': 'EW_flowangle_best',
'Epoch': 'Time',
'Flow_elevation_thresh': 'Flow_elevation_thresh',
'Flow_elevation_threshsp': 'Flow_elevation_threshsp',
'Region': 'Region',
'V_fit': 'V_fit',
'V_mom': 'V_fit',
'mode': 'mode',
'protonV_thermal_fit': 'protonV_thermal_fit',
'protonV_thermal_mom': 'protonV_thermal_fit',
'proton_density_fit': 'proton_density_fit',
'proton_density_mom': 'proton_density_mom',
'xyzgse': ['x_gse', 'y_gse', 'z_gse'],
'ygsm': 'ygsm',
'zgsm': 'zgsm'
}
thisdata = util.cdf2df(cdf, 'Epoch', keys)
data.append(thisdata)
data = pd.concat(data)
data = data[(data['Time'] > starttime) & (data['Time'] < endtime)]
data.index.name = 'Time'
return data
def mag320ms(probe, starttime, endtime, try_download=True):
"""
Import 320ms cadence magnetic field data.
Parameters
----------
probe : string
Probe number.
starttime : datetime
Start of interval.
endtime : datetime
End of interval.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
Requested data.
"""
fnames = []
dirs = []
extension = '.cdf'
dtimes = util._daysplitinterval(starttime, endtime)
# Loop through years
for dtime in dtimes:
date = dtime[0]
intervalstring = str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2)
filename = 'i8_320msec_mag_' + intervalstring + '_v01'
fnames.append(filename)
# Location of file relative to local directory or remote url
relative_loc = 'imp' + probe + '/mag/mag_320msec_cdaweb/' +\
str(date.year)
dirs.append(relative_loc)
local_base_dir = imp_dir
remote_base_url = imp_url
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
remote_url = remote_base_url + str(directory)
filename = fname + extension
local_dir = local_base_dir / directory
util._download_remote(remote_url, filename, local_dir)
def processing_func(f):
thisdata = util.cdf2df(f, 'Epoch')
thisdata.index.name = 'Time'
return thisdata
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
try_download=try_download)
def threedp_pm(starttime, endtime):
"""
Import 'pm' wind data.
3 second time resolution solar wind proton and alpha particle moments from
the PESA LOW sensor, computed on-board the spacecraft
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
"""
# Directory relative to main WIND data directory
relative_dir = path.Path('3dp') / '3dp_pm'
daylist = util._daysplitinterval(starttime, endtime)
ignore = ['TIME']
dirs = []
fnames = []
units = OrderedDict([('P_DENS', u.cm**-3), ('P_VELS', u.km / u.s),
('P_TEMP', u.eV), ('A_DENS', u.cm**-3),
('A_VELS', u.km / u.s), ('A_TEMP', u.eV),
('GAP', u.dimensionless_unscaled), ('E_RANGE', u.eV),
('VALID', u.dimensionless_unscaled),
('VC', u.dimensionless_unscaled),
('SPIN', u.dimensionless_unscaled)])
extension = '.cdf'
for day in daylist:
date = day[0]
this_relative_dir = relative_dir / str(day[0].year)
filename = 'wi_pm_3dp_' +\
str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2) +\
'_v[0-9][0-9]'
fnames.append(filename)
dirs.append(this_relative_dir)
local_base_dir = wind_dir
remote_base_url = remote_wind_dir
def download_func(*args):
util._download_remote_unknown_version(*args)
def processing_func(cdf):
return util.cdf2df(cdf, 'Epoch', ignore=ignore)
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
units=units)
def swe_h3(starttime, endtime):
"""
Import 'h3' solar wind electron data product from WIND.
Electron pitch angle files providing electron fluxes at 30 directional bins
relative to the instantaneous magnetic field direction at 13 different
energy levels
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
"""
relative_dir = os.path.join('swe', 'swe_h3')
# Get directories and filenames
dirs = []
fnames = []
daylist = util._daysplitinterval(starttime, endtime)
for day in daylist:
date = day[0]
filename = 'wi_h3_swe_{}{:02}{:02}_v01'.format(date.year, date.month,
date.day)
fnames.append(filename)
local_dir = os.path.join(relative_dir, str(date.year))
dirs.append(local_dir)
extension = '.cdf'
local_base_dir = wind_dir
remote_base_url = remote_wind_dir
def download_func(remote_base_url, local_base_dir, directory, fname,
extension):
remote_url = '{}{}'.format(remote_base_url, directory)
util.load(fname + extension, os.path.join(local_base_dir, directory),
remote_url)
def processing_func(local_dir, fname):
distkeys = []
for i in range(0, 13):
distkeys.append('f_pitch_E' + str(i).zfill(2))
anglelabels = []
for i in range(0, 30):
anglelabels.append((i + 0.5) * np.pi / 30)
timekey = 'Epoch'
energykey = 'Ve'
cdf = util.load(fname, local_dir, '')
df = util.pitchdist_cdf2df(cdf, distkeys, energykey, timekey,
anglelabels)
return df
return util.process(dirs, fnames, extension, local_base_dir,
remote_base_url, download_func, processing_func,
starttime, endtime)
def fgm_hires(starttime, endtime, try_download=True):
"""
Import high resolution fluxgate magnetometer data.
Parameters
----------
starttime : datetime
Start of interval
endtime : datetime
End of interval
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
Requested data
"""
dtimes = util._daysplitinterval(starttime, endtime)
dirs = []
fnames = []
extension = '.ASC'
local_base_dir = ulysses_dir / 'fgm' / 'hires'
remote_base_url = ulysses_url
units = OrderedDict([('Bx', u.nT), ('By', u.nT),
('Bz', u.nT), ('|B|', u.nT)])
def download_func(remote_base_url, local_base_dir,
directory, fname, remote_fname, extension):
local_dir = local_base_dir / directory
fgm_options = url_options
fgm_options['FILE_NAME'] = fname + extension
fgm_options['FILE_PATH'] = '/ufa/HiRes/VHM-FGM/' + yearstr
remote_url = ulysses_url
for key in fgm_options:
remote_url += key + '=' + fgm_options[key] + '&'
try:
util._load_remote(remote_url, fname + extension, local_dir, 'ascii')
except Exception as err:
return
def processing_func(f):
readargs = {'names': ['year', 'doy', 'hour', 'minute', 'second',
'Bx', 'By', 'Bz', '|B|'],
'delim_whitespace': True}
thisdata = pd.read_table(f, **readargs)
thisdata = _convert_ulysses_time(thisdata)
return thisdata
for dtime in dtimes:
date = dtime[0]
yearstr = date.strftime('%Y')
filename = ('U' + yearstr[-2:] + date.strftime('%j') + 'SH')
fnames.append(filename)
this_relative_dir = local_base_dir / yearstr
dirs.append(this_relative_dir)
return util.process(
dirs, fnames, extension, local_base_dir, remote_base_url,
download_func, processing_func, starttime, endtime, units=units,
try_download=True)
def fgm_survey(probe, starttime, endtime):
"""
Import fgm survey mode magnetic field data.
Parameters
----------
probe : string
Probe number, must be 1, 2, 3, or 4
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
Imported data.
"""
# Directory relative to main MMS data directory
relative_dir = path.Path('mms' + probe) / 'fgm' / 'srvy' / 'l2'
daylist = util._daysplitinterval(starttime, endtime)
dirs = []
fnames = []
extension = '.cdf'
units = OrderedDict([('mms{}_fgm_mode_srvy_l2'.format(probe),
u.dimensionless_unscaled)])
data = []
for day in daylist:
date = day[0]
this_relative_dir = (relative_dir / str(date.year) /
str(date.month).zfill(2))
filename = 'mms{}_fgm_srvy_l2_{}{:02}{:02}_v4.18.0'.format(
probe, date.year, date.month, date.day)
fnames.append(filename)
dirs.append(this_relative_dir)
remote_base_url = remote_mms_dir
local_base_dir = mms_dir
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
remote_url = remote_base_url + str(directory)
util.load(fname + extension, local_base_dir / directory, remote_url)
def processing_func(cdf):
df = util.cdf2df(cdf, 'Epoch')
return df
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
units=units)
def mag320ms(probe, startTime, endTime):
"""
Import 320ms cadence magnetic field data.
Parameters
----------
probe : string
Probe number.
starttime : datetime
Start of interval.
endtime : datetime
End of interval.
Returns
-------
data : DataFrame
Requested data.
"""
data = []
dtimes = util._daysplitinterval(startTime, endTime)
# Loop through years
for dtime in dtimes:
date = dtime[0]
intervalstring = str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2)
filename = 'i8_320msec_mag_' + intervalstring + '_v01.cdf'
hdffname = filename[:-3] + '.hdf'
# Location of file relative to local directory or remote url
relative_loc = 'imp' + probe + '/mag/mag_320msec_cdaweb/' +\
str(date.year)
local_dir = os.path.join(imp_dir, relative_loc)
hdffile = os.path.join(local_dir, hdffname)
if os.path.exists(hdffile):
thisdata = pd.read_hdf(hdffile)
data.append(thisdata)
continue
remote_url = imp_url + relative_loc
cdf = util.load(filename, local_dir, remote_url)
keys = {
'B': '|B|',
'BX': 'Bx',
'BY': 'By',
'BZ': 'Bz',
'Epoch': 'Time'
}
thisdata = util.cdf2df(cdf, 'Epoch', keys)
data.append(thisdata)
if use_hdf:
thisdata.to_hdf(hdffile, key='merged', mode='w')
data = pd.concat(data)
data = data[(data['Time'] > startTime) & (data['Time'] < endTime)]
data = data.drop(columns='Time')
data.index.name = 'Time'
return data
def mag_rtn(starttime, endtime, try_download=True):
"""
Import magnetic field in RTN coordinates from Messenger.
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
"""
# Directory relative to main WIND data directory
relative_dir = path.Path('rtn')
fnames = []
dirs = []
extension = '.cdf'
data = []
ignore = ['Quality_Flag']
daylist = util._daysplitinterval(starttime, endtime)
for day in daylist:
date = day[0]
this_relative_dir = relative_dir / str(date.year)
filename = 'messenger_mag_rtn_' +\
str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2) +\
'_v01'
fnames.append(filename)
dirs.append(this_relative_dir)
local_base_dir = mess_dir
remote_base_url = remote_mess_dir
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
remote_url = remote_base_url + str(directory)
filename = fname + extension
local_dir = local_base_dir / directory
util._download_remote(remote_url, filename, local_dir)
def processing_func(cdf):
thisdata = util.cdf2df(cdf, 'Epoch', ignore=ignore)
return thisdata
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
try_download=try_download)
def swe_h3(starttime, endtime):
"""
Import 'h3' solar wind electron data product from WIND.
Electron pitch angle files providing electron fluxes at 30 directional bins
relative to the instantaneous magnetic field direction at 13 different
energy levels
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
"""
warnings.warn(_deprecation_msg, DeprecationWarning)
relative_dir = path.Path('swe') / 'swe_h3'
# Get directories and filenames
dirs = []
fnames = []
units = OrderedDict([('Angle', u .deg),
('Energy', u.eV),
('df', u.cm/u.s)])
daylist = util._daysplitinterval(starttime, endtime)
for day in daylist:
date = day[0]
filename = 'wi_h3_swe_{}{:02}{:02}_v[0-9][0-9]'.format(
date.year, date.month, date.day)
fnames.append(filename)
local_dir = relative_dir / str(date.year)
dirs.append(local_dir)
extension = '.cdf'
local_base_dir = wind_dir
remote_base_url = remote_wind_dir
distkeys = []
for i in range(0, 13):
distkeys.append('f_pitch_E' + str(i).zfill(2))
anglelabels = []
for i in range(0, 30):
anglelabels.append((i + 0.5) * np.pi / 30)
timekey = 'Epoch'
energykey = 'Ve'
def download_func(*args):
util._download_remote_unknown_version(*args)
def processing_func(cdf):
df = util.pitchdist_cdf2df(cdf, distkeys, energykey, timekey,
anglelabels)
df = df.reset_index(level=['Energy', 'Angle'])
return df
return util.process(dirs, fnames, extension, local_base_dir,
remote_base_url, download_func, processing_func,
starttime, endtime, units=units)
def _ace(starttime,
endtime,
instrument,
product,
fname,
keys=None,
version='01',
badvalues={}):
"""
Generic method for downloading ACE data from cdaweb.
"""
# Directory relative to main WIND data directory
relative_dir = os.path.join(instrument, 'level_2_cdaweb', product)
daylist = util._daysplitinterval(starttime, endtime)
dirs = []
fnames = []
extension = '.cdf'
for day in daylist:
date = day[0]
filename = 'ac_{}_{}{:02}{:02}_v{}'.format(fname, date.year,
date.month, date.day,
version)
fnames.append(filename)
this_relative_dir = os.path.join(relative_dir, str(date.year))
dirs.append(this_relative_dir)
def download_func(remote_base_url, local_base_dir, directory, fname,
extension):
def check_exists():
# Because the version might be different to the one we guess, work
# out the downloaded filename
for f in os.listdir(os.path.join(local_base_dir, directory)):
if (f[:-6] == (fname + extension)[:-6]):
# Return filename with '.cdf' stripped off the end
return f[:-4]
if check_exists() is not None:
return check_exists()
# Now load remotely
util.load(fname + extension,
os.path.join(local_base_dir, directory),
remote_base_url + directory,
guessversion=True)
if check_exists() is not None:
return check_exists()
def processing_func(local_dir, local_fname):
cdf = util.load(local_fname, local_dir, '')
return util.cdf2df(cdf,
index_key='Epoch',
keys=keys,
badvalues=badvalues)
return util.process(dirs, fnames, extension, ace_dir, remote_ace_dir,
download_func, processing_func, starttime, endtime)
def fgm_hires(starttime, endtime):
"""
Import high resolution fluxgate magnetometer data.
Parameters
----------
starttime : datetime
Start of interval
endtime : datetime
End of interval
Returns
-------
data : DataFrame
Requested data
"""
fgm_options = url_options
readargs = {
'names':
['year', 'doy', 'hour', 'minute', 'second', 'Bx', 'By', 'Bz', '|B|'],
'delim_whitespace':
True
}
data = []
dtimes = util._daysplitinterval(starttime, endtime)
# Loop through years
for dtime in dtimes:
date = dtime[0]
yearstr = date.strftime('%Y')
fgm_options['FILE_NAME'] = ('U' + yearstr[-2:] + date.strftime('%j') +
'SH.ASC')
# Local locaiton to download to
local_dir = ulysses_dir / 'fgm' / 'hires' / yearstr
local_file = local_dir / fgm_options['FILE_NAME']
local_hdf = local_file.with_suffix('.hdf')
# If we have already saved a hdf file
if local_hdf.exists():
thisdata = pd.read_hdf(local_hdf)
else:
# Put together remote url
fgm_options['FILE_PATH'] = '/ufa/HiRes/VHM-FGM/' + yearstr
remote_url = ulysses_url
for key in fgm_options:
remote_url += key + '=' + fgm_options[key] + '&'
f = util.load(fgm_options['FILE_NAME'], local_dir, remote_url)
# Read in data
thisdata = pd.read_table(f, **readargs)
# Process data/time
thisdata = _convert_ulysses_time(thisdata)
if use_hdf:
thisdata.to_hdf(local_hdf, 'fgm_hires')
data.append(thisdata)
return util.timefilter(data, starttime, endtime)
def _load(probe, starttime, endtime, instrument, product_id, product_list,
try_download):
dirs = []
fnames = []
download_info = []
for day in util._daysplitinterval(starttime, endtime):
date = day[0]
year = str(date.year)
month = str(date.month).zfill(2)
day = str(date.day).zfill(2)
dirs.append(year)
local_fname = 'C' + probe + '_' + product_id + '__' +\
year + month + day
fnames.append(local_fname)
thisstart = datetime.combine(date, time.min)
thisend = datetime.combine(date, time.max)
download_info.append((thisstart, thisend))
extension = '.cdf'
local_base_dir = cluster_dir / ('c' + probe) / instrument
remote_base_url = csa_url
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension, download_info):
starttime, endtime = download_info
_download(probe, starttime, endtime, instrument, product_id)
def processing_func(file):
for non_empty_var in list(file.cdf_info().keys()):
if 'variable' in non_empty_var.lower():
if len(file.cdf_info()[non_empty_var]) > 0:
var_list = non_empty_var
break
for key in file.cdf_info()[var_list]:
if 'CDF_EPOCH' in file.varget(key, expand=True).values():
index_key = key
break
return util.cdf2xr(file, starttime, endtime, index_key, product_list)
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
try_download=try_download,
units=None,
download_info=download_info)
def _swics(starttime, endtime, names, product, units=None, try_download=True):
data = []
dtimes = util._daysplitinterval(starttime, endtime)
dirs = []
fnames = []
extension = '.dat'
local_base_dir = ulysses_dir / 'swics'
remote_base_url = ulysses_url
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
local_dir = local_base_dir / directory
swics_options = url_options
swics_options['FILE_NAME'] = fname + extension
swics_options['FILE_PATH'] = '/ufa/HiRes/data/swics'
for key in swics_options:
remote_base_url += key + '=' + swics_options[key] + '&'
try:
util._load_remote(remote_base_url, fname + extension, local_dir,
'ascii')
# f = util.load('', local_dir, remote_base_url)
except Exception as err:
return
def processing_func(f):
readargs = {
'names': names,
'delim_whitespace': True,
'na_values': ['******']
}
thisdata = pd.read_csv(f, **readargs)
thisdata = _convert_ulysses_time(thisdata)
return thisdata
# Loop through years
for year in range(starttime.year, endtime.year + 1):
fname = '{}{}'.format(product, str(year)[-2:])
# Local locaiton to download to
dirs.append('')
fnames.append(fname)
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
units=units,
try_download=True)
def fgm_survey(probe, starttime, endtime):
"""
Import fgm survey mode magnetic field data.
Parameters
----------
probe : string
Probe number, must be 1, 2, 3, or 4
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
Imported data.
"""
# Directory relative to main MMS data directory
relative_dir = os.path.join('mms' + probe, 'fgm', 'srvy', 'l2')
daylist = util._daysplitinterval(starttime, endtime)
data = []
for day in daylist:
date = day[0]
this_relative_dir = os.path.join(relative_dir, str(date.year),
str(date.month).zfill(2))
filename = 'mms{}_fgm_srvy_l2_{}{:02}{:02}_v4.18.0.cdf'.format(
probe, date.year, date.month, date.day)
# Absolute path to local directory for this data file
local_dir = os.path.join(mms_dir, this_relative_dir)
util._checkdir(local_dir)
remote_url = remote_mms_dir + this_relative_dir
# Load cdf file
cdf = util.load(filename, local_dir, remote_url)
# Convert cdf to dataframe
keys = {
'mms' + probe + '_fgm_b_gsm_srvy_l2': ['Bx', 'By', 'Bz', 'Br'],
'Epoch': 'Time'
}
df = util.cdf2df(cdf, 'Epoch', keys)
data.append(df)
return util.timefilter(data, starttime, endtime)
def mag_h0(starttime, endtime):
"""
Imports magnetic field data from DSCOVR Spacecraft.
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
"""
dirs = []
fnames = []
extension = '.cdf'
ignore = ['Time1_PB5']
daylist = util._daysplitinterval(starttime, endtime)
for day in daylist:
date = day[0]
filename = "dscovr_h0_mag_{}{:02}{:02}_v01".format(
date.year, date.month, date.day)
fnames.append(filename)
this_relative_dir = 'h0/mag/' + str(date.year)
dirs.append(this_relative_dir)
local_base_dir = dscovr_dir
remote_base_url = dscovr_url
def download_func(remote_base_url, local_base_dir,
directory, fname, remote_fname, extension):
remote_url = remote_base_url + str(directory)
filename = fname + extension
local_dir = local_base_dir / directory
util.load(fname + extension,
local_base_dir / directory,
remote_url)
def processing_func(cdf):
df = util.cdf2df(cdf, 'Epoch1', ignore=ignore)
df.index.name = 'Time'
return df
return util.process(dirs, fnames, extension, local_base_dir,
remote_base_url, download_func, processing_func,
starttime, endtime)
def _ace(starttime,
endtime,
instrument,
product,
fname,
units=None,
version='01',
badvalues={}):
"""
Generic method for downloading ACE data from cdaweb.
"""
# Directory relative to main WIND data directory
relative_dir = path.Path(instrument) / 'level_2_cdaweb' / product
daylist = util._daysplitinterval(starttime, endtime)
dirs = []
fnames = []
extension = '.cdf'
for day in daylist:
date = day[0]
filename = 'ac_{}_{}{:02}{:02}_v[0-9][0-9]'.format(
fname, date.year, date.month, date.day)
fnames.append(filename)
this_relative_dir = relative_dir / str(date.year)
dirs.append(this_relative_dir)
def download_func(*args):
util._download_remote_unknown_version(*args)
def processing_func(cdf):
return util.cdf2df(cdf, index_key='Epoch', badvalues=badvalues)
return util.process(dirs,
fnames,
extension,
ace_dir,
remote_ace_dir,
download_func,
processing_func,
starttime,
endtime,
units=units)
def _swics(starttime, endtime, names, product):
swics_options = url_options
readargs = {'names': names, 'delim_whitespace': True}
data = []
dtimes = util._daysplitinterval(starttime, endtime)
# Loop through years
for year in range(starttime.year, endtime.year + 1):
swics_options['FILE_NAME'] = '{}{}.dat'.format(product, str(year)[-2:])
# Local locaiton to download to
local_dir = os.path.join(ulysses_dir, 'swics')
local_file = os.path.join(local_dir, swics_options['FILE_NAME'])
local_hdf = local_file[:-4] + '.hdf'
# If we have already saved a hdf file
if os.path.exists(local_hdf):
thisdata = pd.read_hdf(local_hdf)
else:
# Put together remote url
swics_options['FILE_PATH'] = '/ufa/HiRes/data/swics'
remote_url = ulysses_url
for key in swics_options:
remote_url += key + '=' + swics_options[key] + '&'
f = util.load(swics_options['FILE_NAME'], local_dir, remote_url)
if f is None:
print('File {}/{} not available\n'.format(
remote_url, filename))
continue
# Read in data
thisdata = pd.read_table(f, **readargs)
# Process data/time
thisdata = _convert_ulysses_time(thisdata)
if use_hdf:
thisdata.to_hdf(local_hdf, 'swics')
data.append(thisdata)
return util.timefilter(data, starttime, endtime)
def mag_hires(starttime, endtime, try_download=True):
"""
Import high resolution magnetic field from Cassini.
See http://pds-ppi.igpp.ucla.edu/search/view/?f=yes&id=pds://PPI/CO-E_SW_J_S-MAG-3-RDR-FULL-RES-V1.0
for more information.
Cassini Orbiter Magnetometer Calibrated MAG data at the highest time
resolution available covering the period 1999-08-16 (DOY 228) to
2016-12-31 (DOY 366).
The data are in RTN coordinates prior Cassini's arrival at Saturn, and
Kronographic (KRTP) coordinates at Saturn (beginning 2004-05-14, DOY 135).
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
Requested data
"""
remote_base_url = ('https://pds-ppi.igpp.ucla.edu/ditdos/download?id='
'pds://PPI/CO-E_SW_J_S-MAG-3-RDR-FULL-RES-V2.0/DATA')
dirs = []
fnames = []
extension = '.TAB'
units = OrderedDict([('Bx', u.nT), ('By', u.nT), ('Bz', u.nT),
('coords', u.dimensionless_unscaled)])
local_base_dir = cassini_dir / 'mag' / 'hires'
for [day, _, _] in util._daysplitinterval(starttime, endtime):
year = day.year
if calendar.isleap(year):
monthstr = leapmonth2str[day.month]
else:
monthstr = month2str[day.month]
if day < datetime.date(2004, 5, 14):
coords = 'RTN'
else:
coords = 'KRTP'
doy = day.strftime('%j')
dirs.append(pathlib.Path(str(year)) / monthstr)
fnames.append(str(year)[2:] + doy + '_FGM_{}'.format(coords))
def download_func(remote_base_url, local_base_dir,
directory, fname, remote_fname, extension):
url = remote_base_url + '/' + str(directory)
util._download_remote(url, fname + extension,
local_base_dir / directory)
def processing_func(f):
if 'error_message' in f.readline():
f.close()
os.remove(f.name)
raise util.NoDataError()
df = pd.read_csv(f, names=['Time', 'Bx', 'By', 'Bz'],
delim_whitespace=True,
parse_dates=[0], index_col=0)
return df
return util.process(dirs, fnames, extension, local_base_dir,
remote_base_url, download_func, processing_func,
starttime, endtime, units=units,
try_download=try_download)
def threedp_sfpd(starttime, endtime):
"""
Import 'sfpd' wind data.
12 second energetic electron pitch-angle energy spectra from the foil SST
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
"""
# Directory relative to main WIND data directory
relative_dir = path.Path('3dp') / '3dp_sfpd'
daylist = util._daysplitinterval(starttime, endtime)
data = []
mag = []
for (date, _, _) in daylist:
this_relative_dir = relative_dir / str(date.year)
# Absolute path to local directory for this data file
local_dir = wind_dir / this_relative_dir
filename = 'wi_sfpd_3dp_{:{dfmt}}_v02'.format(date, dfmt='%Y%m%d')
hdfname = filename + '.hdf'
hdfloc = local_dir / hdfname
if hdfloc.exists():
df = pd.read_hdf(hdfloc)
data.append(df)
continue
util._checkdir(local_dir)
remote_url = remote_wind_dir + str(this_relative_dir)
cdf = util.load(filename + '.cdf',
local_dir,
remote_url,
guessversion=True)
if cdf is None:
print('File {}/{} not available\n'.format(remote_url, filename))
continue
data_today = []
# Loop through each timestamp to build up fluxes
for i, time in enumerate(cdf['Epoch'][...]):
energies = cdf['ENERGY'][i, :]
angles = cdf['PANGLE'][i, :]
fluxes = cdf['FLUX'][i, :, :]
magfield = cdf['MAGF'][i, :]
index = pd.MultiIndex.from_product(
([time], energies, angles),
names=['Time', 'Energy', 'Pitch angle'])
df = pd.DataFrame(fluxes.ravel(), index=index, columns=['Flux'])
df['Bx'] = magfield[0]
df['By'] = magfield[1]
df['Bz'] = magfield[2]
data_today.append(df)
data_today = pd.concat(data_today)
data_today = data_today.sort_index()
if use_hdf:
data_today.to_hdf(hdfloc, 'sfpd', mode='w')
data.append(data_today)
data = util.timefilter(data, starttime, endtime)
data = data.reset_index(level=['Energy', 'Pitch angle'])
return data
def threedp_pm(starttime, endtime):
"""
Import 'pm' wind data.
3 second time resolution solar wind proton and alpha particle moments from
the PESA LOW sensor, computed on-board the spacecraft
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
"""
# Directory relative to main WIND data directory
relative_dir = path.Path('3dp') / '3dp_pm'
daylist = util._daysplitinterval(starttime, endtime)
data = []
for day in daylist:
date = day[0]
this_relative_dir = relative_dir / str(day[0].year)
# Absolute path to local directory for this data file
local_dir = wind_dir / this_relative_dir
filename = 'wi_pm_3dp_' +\
str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2) +\
'_v05.cdf'
hdfname = filename[:-4] + 'hdf'
hdfloc = local_dir / hdfname
if hdfloc.exists():
df = pd.read_hdf(hdfloc)
data.append(df)
continue
util._checkdir(local_dir)
remote_url = remote_wind_dir + str(this_relative_dir)
cdf = util.load(filename, local_dir, remote_url, guessversion=True)
if cdf is None:
print('File {}/{} not available\n'.format(remote_url, filename))
continue
keys = {
'A_DENS': 'n_a',
'A_TEMP': 'T_a',
'A_VELS': ['va_x', 'va_y', 'va_z'],
'P_DENS': 'n_p',
'P_TEMP': 'T_p',
'P_VELS': ['vp_x', 'vp_y', 'vp_z'],
'Epoch': 'Time'
}
df = util.cdf2df(cdf, index_key='Epoch', keys=keys)
if use_hdf:
df.to_hdf(hdfloc, 'pm', mode='w')
data.append(df)
return util.timefilter(data, starttime, endtime)
def _mfi(starttime, endtime, version, units=None):
"""
Import mfi magnetic field data products from WIND.
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
"""
# Directory relative to main WIND data directory
relative_dir = path.Path('mfi') / ('mfi_' + version)
# Get directories and filenames
dirs = []
fnames = []
daylist = util._daysplitinterval(starttime, endtime)
for day in daylist:
date = day[0]
# Absolute path to local directory for this data file
local_dir = relative_dir / str(day[0].year)
dirs.append(local_dir)
filename = 'wi_' + version + '_mfi_' +\
str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2) +\
'_v05'
fnames.append(filename)
extension = '.cdf'
local_base_dir = wind_dir
remote_base_url = remote_wind_dir
def download_func(remote_base_url, local_base_dir, directory, fname,
extension):
remote_url = '{}{}'.format(remote_base_url, directory)
util.load(fname + extension,
local_base_dir / directory,
remote_url,
guessversion=True)
def processing_func(cdf):
epoch_dict = {'h0': 'Epoch3', 'h2': 'Epoch'}
mag_dict = {'h0': 'B3GSE', 'h2': 'BGSE'}
epoch_key = epoch_dict[version]
mag_key = mag_dict[version]
keys = {mag_key: ['Bx_gse', 'By_gse', 'Bz_gse'], epoch_key: 'Time'}
badvalues = {'Bx_gse': -1e+31, 'By_gse': -1e+31, 'Bz_gse': -1e+31}
df = util.cdf2df(cdf,
index_key=epoch_key,
keys=keys,
badvalues=badvalues)
return df
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
units=units)
def threedp_sfpd(starttime, endtime):
"""
Import 'sfpd' wind data.
12 second energetic electron pitch-angle energy spectra from the foil SST
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
"""
# Directory relative to main WIND data directory
relative_dir = path.Path('3dp') / '3dp_sfpd'
daylist = util._daysplitinterval(starttime, endtime)
data = []
fnames = []
dirs = []
units = OrderedDict([('Energy', u.eV),
('Bx', u.nT),
('By', u.nT),
('Bz', u.nT),
('Pitch angle', u.deg),
('Flux', (u.cm**2 * u.sr * u.eV * u.s)**-1)])
extension = '.cdf'
for (date, _, _) in daylist:
this_relative_dir = relative_dir / str(date.year)
dirs.append(this_relative_dir)
filename = 'wi_sfpd_3dp_{:{dfmt}}_v02'.format(
date, dfmt='%Y%m%d')
fnames.append(filename)
local_base_dir = wind_dir
remote_base_url = remote_wind_dir
def download_func(*args):
util._download_remote_unknown_version(*args)
def processing_func(cdf):
data_today = []
# Loop through each timestamp to build up fluxes
for non_empty_var in list(cdf.cdf_info().keys()):
if 'variable' in non_empty_var.lower():
if len(cdf.cdf_info()[non_empty_var]) > 0:
var_list = non_empty_var
break
index_ = cdf.varget('Epoch')[...]
index_ = cdflib.cdfepoch.breakdown(index_)
index_ = np.asarray([dt.datetime(*x) for x in index_])
energies_ = cdf.varget('ENERGY')[...]
angles_ = cdf.varget('PANGLE')[...]
fluxes_ = cdf.varget('FLUX')[...]
magfield_ = cdf.varget('MAGF')[...]
for i, time in enumerate(index_):
energies = energies_[i, :]
angles = angles_[i, :]
fluxes = fluxes_[i, :, :]
magfield = magfield_[i, :]
index = pd.MultiIndex.from_product(
([time], energies, angles),
names=['Time', 'Energy', 'Pitch angle'])
df = pd.DataFrame(fluxes.ravel(), index=index, columns=['Flux'])
df = df.reset_index(level=['Energy', 'Pitch angle'])
df['Bx'] = magfield[0]
df['By'] = magfield[1]
df['Bz'] = magfield[2]
data_today.append(df)
data_today = pd.concat(data_today)
data_today = data_today.sort_index()
return data_today
return util.process(dirs, fnames, extension, local_base_dir,
remote_base_url, download_func, processing_func,
starttime, endtime, units=units)
def _download(probe, starttime, endtime, instrument, product_id):
if cda_cookie == 'none':
raise RuntimeError('Cluster download cookie not set')
daylist = util._daysplitinterval(starttime, endtime)
for day in daylist:
date = day[0]
start = datetime.combine(date, time.min)
end = datetime.combine(date, time.max)
# Add start and end time to request dictionary
request_dict = generic_dict
request_dict['START_DATE'] = start.strftime(cda_time_fmt)
request_dict['END_DATE'] = end.strftime(cda_time_fmt)
# Create request string
request_str = ''
request_str += 'DATASET_ID' + '='
request_str += 'C' + probe + '_' + product_id
for item in request_dict:
request_str += '&'
request_str += item
request_str += '='
request_str += request_dict[item]
# Create request url
request_str += '&NON_BROWSER'
request_url = csa_url + request_str
# Work out local directory to download to
year = str(starttime.year)
month = str(starttime.month).zfill(2)
day = str(starttime.day).zfill(2)
local_dir = cluster_dir / ('c' + probe) / instrument / year
local_fname = 'C' + probe + '_' + product_id + '__' +\
year + month + day + '.cdf'
local_file = local_dir / local_fname
print(request_url)
# Download data
util._checkdir(local_dir)
urlreq.urlretrieve(request_url,
filename=local_file,
reporthook=util._reporthook)
print('\n')
# Extract tar.gz file
tar = tarfile.open(local_file)
tar.extractall(local_dir)
# Delete tar.gz file
os.remove(local_file)
# The CSA timpstamps the downloaded file by when it is downloaded,
# so manually list and retrieve the folder name
dirlist = os.listdir(local_dir)
for d in dirlist:
if d[:13] == 'CSA_Download_':
download_dir = local_dir / d / ('C' + probe + '_' + product_id)
break
# Remove request times from filename
dirlist = os.listdir(download_dir)
# Move to data folder
cutoff = 3 + len(product_id) + 10
for f in dirlist:
os.rename(download_dir / f, local_dir / (f[:cutoff] + '.cdf'))
# Delte extra folders created by tar.gz file
os.rmdir(download_dir)
os.rmdir(os.path.join(local_dir, d))
def fgm(probe, rate, coords, starttime, endtime):
"""
Import fgm magnetic field data from THEMIS.
Parameters
----------
probe : string
Alowed values are [a, b, c, d, e].
rate : string
Date rate to return. Allowed values are [e, h, l, s].
coords : string
Magnetic field co-ordinate system. Allowed values are
[dsl, gse, gsm, ssl]. NOTE: Add link to co-ordinate system
descriptions.
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
"""
valid_rates = ['e', 'h', 'l', 's']
valid_coords = ['dsl', 'gse', 'gsm', 'ssl']
_validate_probe(probe)
if rate not in valid_rates:
raise ValueError(('rate argument %s is not in list of allowed'
'rates: %s') % (rate, valid_rates))
if coords not in valid_coords:
raise ValueError(('coords argument %s is not in list of allowed'
'co-ordinate systems: %s') % (rate, valid_rates))
# Directory relative to main THEMIS data directory
fgm_dir = path.Path('th' + probe) / 'l2' / 'fgm'
daylist = util._daysplitinterval(starttime, endtime)
dirs = []
fnames = []
extension = '.cdf'
units = OrderedDict([('|B|', u.nT), ('Bz_dsl', u.nT), ('By_dsl', u.nT),
('Bx_dsl', u.nT),
('th{}_fgm_fg{}_quality'.format(probe, rate),
u.dimensionless_unscaled)])
for day in daylist:
date = day[0]
filename = 'th{}_l2_fgm_{}{:02}{:02}_v01'.format(
probe, date.year, date.month, date.day)
fnames.append(filename)
this_relative_dir = fgm_dir / str(date.year)
dirs.append(this_relative_dir)
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
remote_url = remote_base_url + str(directory)
# Now load remotely
util.load(fname + extension, local_base_dir / directory, remote_url)
def processing_func(cdf, **kwargs):
probe = kwargs.pop('probe')
rate = kwargs.pop('rate')
kwargs.pop('coords')
probestr = 'th' + probe
ratestr = '_fg' + rate + '_'
df = util.cdf2df(cdf, probestr + ratestr + 'time', dtimeindex=False)
df = df.set_index(pd.to_datetime(df.index.values, unit='s'))
df.index.name = 'Time'
return df
processing_kwargs = {'probe': probe, 'rate': rate, 'coords': coords}
return util.process(dirs,
fnames,
extension,
artemis_dir,
remote_themis_dir,
download_func,
processing_func,
starttime,
endtime,
units=units,
processing_kwargs=processing_kwargs)
def mag15s(probe, starttime, endtime, verbose=False, try_download=True):
"""
Import 15s cadence magnetic field data.
Parameters
----------
probe : string
Probe number.
starttime : datetime
Start of interval.
endtime : datetime
End of interval.
verbose : bool, optional
If ``True``, print information whilst loading. Default is ``False``.
Returns
-------
data : :class:`~sunpy.timeseries.TimeSeries`
Requested data.
"""
fnames = []
dirs = []
extension = '.asc'
units = OrderedDict([('n points', u.dimensionless_unscaled),
('Source flag', u.dimensionless_unscaled),
('x gse', u.earthRad), ('y gse', u.earthRad),
('z gse', u.earthRad), ('y gsm', u.earthRad),
('z gsm', u.earthRad), ('|B|', u.nT),
('Bx gse', u.nT), ('By gse', u.nT), ('Bz gse', u.nT),
('By gsm', u.nT), ('Bz gsm', u.nT),
('Bxx gse', u.nT**2), ('Byy gse', u.nT**2),
('Bzz gse', u.nT**2), ('Byx gse', u.nT**2),
('Bzx gse', u.nT**2), ('Bzy gse', u.nT**2),
('Time shift', u.s),
('sw flag', u.dimensionless_unscaled)])
dtimes = util._daysplitinterval(starttime, endtime)
# Loop through years
for dtime in dtimes:
startdt = dtime[0]
year = startdt.year
doy = util.dtime2doy(startdt)
if verbose:
print('Loading IMP 15s mag probe {}, {:03d}/{}'.format(
probe, doy, year))
filename = '{}{:03d}_imp{}_mag_15s_v3'.format(year, doy, probe)
fnames.append(filename)
# Location of file relative to local directory or remote url
relative_loc = os.path.join('imp{}'.format(probe), 'mag',
'15s_ascii_v3', str(year))
dirs.append(relative_loc)
local_base_dir = imp_dir
remote_base_url = imp_url
def download_func(remote_base_url, local_base_dir, directory, fname,
remote_fname, extension):
remote_url = remote_base_url + str(directory)
filename = fname + extension
local_dir = local_base_dir / directory
util._download_remote(remote_url, filename, local_dir)
# Read in data
def processing_func(f):
readargs = {
'names': [
'Year', 'doy', 'Second', 'Source flag', 'n points', 'x gse',
'y gse', 'z gse', 'y gsm', 'z gsm', '|B|', 'Bx gse', 'By gse',
'Bz gse', 'By gsm', 'Bz gsm', 'Bxx gse', 'Byy gse', 'Bzz gse',
'Byx gse', 'Bzx gse', 'Bzy gse', 'Time shift', 'sw flag'
],
'na_values': [
'9999', '999', '99', '9', '999', '99.99', '99.99', '99.99',
'99.99', '99.99', '9999.99', '9999.99', '9999.99', '9999.99',
'9999.99', '9999.99', '9999.99', '9999.99', '9999.99',
'9999.99', '9999.99', '9999.99', '999.9', '9'
],
'delim_whitespace':
True
}
thisdata = pd.read_table(f, **readargs)
thisdata['Time'] = (pd.to_datetime(thisdata['Year'], format='%Y') +
pd.to_timedelta(thisdata['doy'] - 1, unit='d') +
pd.to_timedelta(thisdata['Second'], unit='s'))
thisdata = thisdata.set_index('Time', drop=False)
thisdata = thisdata.drop(['Year', 'doy', 'Second'], 1)
return thisdata
return util.process(dirs,
fnames,
extension,
local_base_dir,
remote_base_url,
download_func,
processing_func,
starttime,
endtime,
units=units,
try_download=try_download)
def download_files(probe, instrument, data_rate, starttime, endtime,
verbose=True, product_string='', warn_missing_units=True):
"""
Download MMS files.
Parameters
----------
probe : int or str
MMS probe number. Must be in 1-4 inclusive.
instrument : str
MMS instrument. Must be in ``['afg', 'aspoc', 'dfg', 'dsp', 'edi',
'edp', 'fields', 'scm', 'sdp']``
data_rate : str
Data rate. Must be in ``['slow', 'fast', 'brst', 'srvy']``
starttime : ~datetime.datetime
Start time.
endtime : ~datetime.datetime
End time.
verbose : bool
If ``True``, show a progress bar while downloading.
product_string : str
If not empty, this string must be in the filename for it to be
downloaded.
warn_missing_units : bool
If ``True``, warnings will be shown for each variable that does not
have associated units.
Returns
-------
df : :class:`~sunpy.timeseries.GenericTimeSeries`
Requested data.
"""
_validate_instrument(instrument)
probe = _validate_probe(probe)
dirs = []
fnames = []
daylist = util._daysplitinterval(starttime, endtime)
for date, stime, etime in daylist:
files = available_files(probe, instrument, starttime, endtime,
data_rate, product_string)
for file in files:
fname = pathlib.Path(file).stem
if product_string in fname and len(fname):
fnames.append(fname)
dirs.append('')
extension = '.cdf'
local_base_dir = mms_dir / probe / instrument / data_rate
remote_base_url = dl_url
def download_func(remote_base_url, local_base_dir,
directory, fname, remote_fname, extension):
url = remote_base_url + '?file=' + fname + extension
local_fname = os.path.join(local_base_dir, fname + extension)
with requests.get(url, stream=True) as request:
with open(local_fname, 'wb') as fd:
for chunk in tqdm(
request.iter_content(chunk_size=128)):
fd.write(chunk)
def processing_func(cdf):
return util.cdf2df(cdf, index_key='Epoch')
return util.process(dirs, fnames, extension, local_base_dir,
remote_base_url, download_func, processing_func,
starttime, endtime,
warn_missing_units=warn_missing_units)
