def mag_hires(starttime, endtime):
"""
Import high resolution magnetic field from Cassini.
See http://pds-ppi.igpp.ucla.edu/search/view/?f=yes&id=pds://PPI/CO-E_SW_J_S-MAG-3-RDR-FULL-RES-V1.0
for more information.
Cassini Orbiter Magnetometer Calibrated MAG data at the highest time
resolution available covering the period 1999-08-16 (DOY 228) to
2016-12-31 (DOY 366).
The data are in RTN coordinates prior Cassini's arrival at Saturn, and
Kronographic (KRTP) coordinates at Saturn (beginning 2004-05-14, DOY 135).
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
Requested data
"""
base_url = ('http://pds-ppi.igpp.ucla.edu/ditdos/download?id='
'pds://PPI/CO-E_SW_J_S-MAG-3-RDR-FULL-RES-V1.0/DATA')
daylist = util._daysplitinterval(starttime, endtime)
data = []
for [day, stime, etime] in daylist:
year = day.year
url = '{}/{}'.format(base_url, year)
if calendar.isleap(year):
monthstr = leapmonth2str[day.month]
else:
monthstr = month2str[day.month]
url = '{}/{}'.format(url, monthstr)
doy = day.strftime('%j')
local_dir = cassini_dir / 'mag' / 'hires' / str(year)
# No way to work out co-ordinates, so guess Kronian and then RTN
try:
coords = 'KRTP'
df = _mag_hires_helper(year, doy, local_dir, url, coords)
except RuntimeError:
try:
coords = 'RTN'
df = _mag_hires_helper(year, doy, local_dir, url, coords)
except RuntimeError:
continue
df['coords'] = coords
data.append(df)
return util.timefilter(data, starttime, endtime)
def _load(probe, starttime, endtime, instrument, product_id, cdfkeys):
daylist = util._daysplitinterval(starttime, endtime)
data = []
for day in daylist:
date = day[0]
year = str(date.year)
month = str(date.month).zfill(2)
day = str(date.day).zfill(2)
local_dir = cluster_dir / ('c' + probe) / instrument / year
local_fname = 'C' + probe + '_' + product_id + '__' +\
year + month + day + '.cdf'
local_file = local_dir / local_fname
# If we don't have local file download it
if not local_file.exists():
thisstart = datetime.combine(date, time.min)
thisend = datetime.combine(date, time.max)
try:
_download(probe, thisstart, thisend, instrument, product_id)
except Exception as err:
print(str(err), '\n')
continue
from pycdf import pycdf
cdf = pycdf.CDF(os.path.join(local_dir, local_fname))
for key, value in cdfkeys.items():
if value == 'Time':
index_key = key
break
data.append(util.cdf2df(cdf, index_key, cdfkeys))
if len(data) == 0:
raise RuntimeError('No data available to download during requested '
'times')
return util.timefilter(data, starttime, endtime)
def fgm_hires(starttime, endtime):
"""
Import high resolution fluxgate magnetometer data.
Parameters
----------
starttime : datetime
Start of interval
endtime : datetime
End of interval
Returns
-------
data : DataFrame
Requested data
"""
fgm_options = url_options
readargs = {
'names':
['year', 'doy', 'hour', 'minute', 'second', 'Bx', 'By', 'Bz', '|B|'],
'delim_whitespace':
True
}
data = []
dtimes = util._daysplitinterval(starttime, endtime)
# Loop through years
for dtime in dtimes:
date = dtime[0]
yearstr = date.strftime('%Y')
fgm_options['FILE_NAME'] = ('U' + yearstr[-2:] + date.strftime('%j') +
'SH.ASC')
# Local locaiton to download to
local_dir = ulysses_dir / 'fgm' / 'hires' / yearstr
local_file = local_dir / fgm_options['FILE_NAME']
local_hdf = local_file.with_suffix('.hdf')
# If we have already saved a hdf file
if local_hdf.exists():
thisdata = pd.read_hdf(local_hdf)
else:
# Put together remote url
fgm_options['FILE_PATH'] = '/ufa/HiRes/VHM-FGM/' + yearstr
remote_url = ulysses_url
for key in fgm_options:
remote_url += key + '=' + fgm_options[key] + '&'
f = util.load(fgm_options['FILE_NAME'], local_dir, remote_url)
# Read in data
thisdata = pd.read_table(f, **readargs)
# Process data/time
thisdata = _convert_ulysses_time(thisdata)
if use_hdf:
thisdata.to_hdf(local_hdf, 'fgm_hires')
data.append(thisdata)
return util.timefilter(data, starttime, endtime)
def fgm_survey(probe, starttime, endtime):
"""
Import fgm survey mode magnetic field data.
Parameters
----------
probe : string
Probe number, must be 1, 2, 3, or 4
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
Imported data.
"""
# Directory relative to main MMS data directory
relative_dir = os.path.join('mms' + probe, 'fgm', 'srvy', 'l2')
daylist = util._daysplitinterval(starttime, endtime)
data = []
for day in daylist:
date = day[0]
this_relative_dir = os.path.join(relative_dir, str(date.year),
str(date.month).zfill(2))
filename = 'mms{}_fgm_srvy_l2_{}{:02}{:02}_v4.18.0.cdf'.format(
probe, date.year, date.month, date.day)
# Absolute path to local directory for this data file
local_dir = os.path.join(mms_dir, this_relative_dir)
util._checkdir(local_dir)
remote_url = remote_mms_dir + this_relative_dir
# Load cdf file
cdf = util.load(filename, local_dir, remote_url)
# Convert cdf to dataframe
keys = {
'mms' + probe + '_fgm_b_gsm_srvy_l2': ['Bx', 'By', 'Bz', 'Br'],
'Epoch': 'Time'
}
df = util.cdf2df(cdf, 'Epoch', keys)
data.append(df)
return util.timefilter(data, starttime, endtime)
def _swics(starttime, endtime, names, product):
swics_options = url_options
readargs = {'names': names, 'delim_whitespace': True}
data = []
dtimes = util._daysplitinterval(starttime, endtime)
# Loop through years
for year in range(starttime.year, endtime.year + 1):
swics_options['FILE_NAME'] = '{}{}.dat'.format(product, str(year)[-2:])
# Local locaiton to download to
local_dir = os.path.join(ulysses_dir, 'swics')
local_file = os.path.join(local_dir, swics_options['FILE_NAME'])
local_hdf = local_file[:-4] + '.hdf'
# If we have already saved a hdf file
if os.path.exists(local_hdf):
thisdata = pd.read_hdf(local_hdf)
else:
# Put together remote url
swics_options['FILE_PATH'] = '/ufa/HiRes/data/swics'
remote_url = ulysses_url
for key in swics_options:
remote_url += key + '=' + swics_options[key] + '&'
f = util.load(swics_options['FILE_NAME'], local_dir, remote_url)
if f is None:
print('File {}/{} not available\n'.format(
remote_url, filename))
continue
# Read in data
thisdata = pd.read_table(f, **readargs)
# Process data/time
thisdata = _convert_ulysses_time(thisdata)
if use_hdf:
thisdata.to_hdf(local_hdf, 'swics')
data.append(thisdata)
return util.timefilter(data, starttime, endtime)
def integrated_dists(probe, starttime, endtime, verbose=False):
"""
Returns the integrated distributions from experiments i1a and i1b in Helios
distribution function files.
The distributions are integrated over all angles and given as a function
of proton velocity.
Parameters
----------
probe : int
Helios probe to import data from. Must be 1 or 2.
starttime : datetime
Start of interval
endtime : datetime
End of interval
verbose : bool, optional
If ``True``, print information whilst loading. Default is ``False``.
Returns
-------
distinfo : Series
Infromation stored in the top of distribution function files.
"""
extensions = ['hdm.0', 'hdm.1', 'ndm.0', 'ndm.1']
distlist = {'a': [], 'b': []}
starttime_orig = starttime
# Loop through each day
while starttime < endtime:
year = starttime.year
doy = starttime.strftime('%j')
# Directory for today's distribution files
dist_dir = _dist_file_dir(probe, year, doy)
# Locaiton of hdf file to save to/load from
hdffile = 'h' + probe + str(year) + str(doy).zfill(3) +\
'integrated_dists.hdf'
hdffile = os.path.join(dist_dir, hdffile)
todays_dists = {'a': [], 'b': []}
# Check if data is already saved
if os.path.isfile(hdffile):
for key in todays_dists:
todays_dists[key] = pd.read_hdf(hdffile, key=key)
distlist[key].append(todays_dists[key])
starttime += timedelta(days=1)
continue
# If not saved, generate a derived file
else:
# Get every distribution function file present for this day
for f in os.listdir(dist_dir):
path = os.path.join(dist_dir, f)
# Check for distribution function
if path[-5:] in extensions:
hour, minute, second = _dist_filename_to_hms(path)
try:
a, b = integrated_dists_single(probe, year, doy, hour,
minute, second)
except RuntimeError as err:
strerr = 'No ion distribution function data in file'
if str(err) == strerr:
continue
raise err
t = datetime.combine(starttime.date(),
time(hour, minute, second))
if verbose:
print(t)
dists = {'a': a, 'b': b}
for key in dists:
dist = dists[key]
dist['Time'] = t
dist = dist.set_index(['Time', 'v'], drop=True)
todays_dists[key].append(dist)
# Go through a and b and concat all the data
for key in todays_dists:
todays_dists[key] = pd.concat(todays_dists[key])
if use_hdf:
todays_dists[key].to_hdf(hdffile, key=key, mode='a')
distlist[key].append(todays_dists[key])
starttime += timedelta(days=1)
for key in distlist:
distlist[key] = util.timefilter(distlist[key], starttime_orig, endtime)
return distlist
def mag_rtn(starttime, endtime):
"""
Import magnetic field in RTN coordinates from Messenger.
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
"""
# Directory relative to main WIND data directory
relative_dir = 'rtn'
daylist = util._daysplitinterval(starttime, endtime)
data = []
for day in daylist:
date = day[0]
this_relative_dir = os.path.join(relative_dir, str(date.year))
hdffile = 'messenger_mag_rtn_' +\
str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2) +\
'_v01.hdf'
hdfloc = os.path.join(mess_dir, this_relative_dir, hdffile)
# Try to load hdf file
if os.path.isfile(hdfloc):
df = pd.read_hdf(hdfloc)
data.append(df)
continue
filename = hdffile[:-4] + '.cdf'
# Absolute path to local directory for this data file
local_dir = os.path.join(mess_dir, this_relative_dir)
util._checkdir(local_dir)
remote_url = os.path.join(remote_mess_dir, this_relative_dir)
cdf = util.load(filename, local_dir, remote_url, guessversion=True)
if cdf is None:
print('File {}/{} not available\n'.format(remote_url, filename))
continue
keys = {
'B_normal': 'Bn',
'B_radial': 'Br',
'B_tangential': 'Bt',
'Epoch': 'Time',
'azimuth_ecliptic': 'sc_Az',
'latitude_ecliptic': 'sc_Lat',
'radialDistance': 'sc_r',
'MissionElapsedTime': 'mission_time'
}
df = util.cdf2df(cdf, index_key='Epoch', keys=keys)
if use_hdf:
hdffile = filename[:-4] + '.hdf'
df.to_hdf(hdfloc, key='data', mode='w')
data.append(df)
return util.timefilter(data, starttime, endtime)
def mag_1min(starttime, endtime, coords):
"""
Import 1 minute magnetic field from Cassini.
See http://pds-ppi.igpp.ucla.edu/search/view/?f=yes&id=pds://PPI/CO-E_SW_J_S-MAG-4-SUMM-1MINAVG-V1.0
for more information.
Cassini Orbiter Magnetometer Calibrated MAG data in 1 minute averages
available covering the period 1999-08-16 (DOY 228) to 2016-12-31 (DOY 366).
The data are provided in RTN coordinates throughout the mission, with
Earth, Jupiter, and Saturn centered coordinates for the respective
flybys of those planets.
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
coords : strings
Requested coordinate system. Must be one of
``['KRTP', 'KSM', 'KSO', 'RTN']``
Returns
-------
data : DataFrame
Requested data
"""
valid_coords = ['KRTP', 'KSM', 'KSO', 'RTN']
if coords not in valid_coords:
raise ValueError('coords must be one of {}'.format(valid_coords))
base_url = ('http://pds-ppi.igpp.ucla.edu/ditdos/download?'
'id=pds://PPI/CO-E_SW_J_S-MAG-4-SUMM-1MINAVG-V1.0/DATA')
data = []
for year in starttime.year, endtime.year:
url = '{}/{}'.format(base_url, year)
local_dir = os.path.join(cassini_dir, 'mag', '1min')
fname = '{}_FGM_{}_1M'.format(year, coords)
hdfloc = os.path.join(local_dir, fname + '.hdf')
if os.path.isfile(hdfloc):
df = pd.read_hdf(hdfloc)
data.append(df)
continue
f = util.load(fname + '.TAB', local_dir, url)
if 'error_message' in f.readline():
f.close()
os.remove(os.path.join(local_dir, fname + '.TAB'))
continue
df = pd.read_table(f,
names=[
'Time', 'Bx', 'By', 'Bz', '|B|', 'X', 'Y', 'Z',
'Local hour', 'n points'
],
delim_whitespace=True,
parse_dates=[0],
index_col=0)
data.append(df)
if use_hdf:
df.to_hdf(hdfloc, key='data', mode='w')
return util.timefilter(data, starttime, endtime)
def merged(probe, starttime, endtime, verbose=False):
"""
Import merged plasma data. See
ftp://cdaweb.gsfc.nasa.gov/pub/data/imp/imp8/merged/00readme.txt
for information on variables.
Parameters
----------
probe : string
Probe number.
starttime : datetime
Start of interval.
endtime : datetime
End of interval.
verbose : bool, optional
If ``True``, print information whilst loading. Default is
``False``.
Returns
-------
data : DataFrame
Requested data.
"""
_check_probe(probe, ['8'])
data = []
startyear = starttime.year
endyear = endtime.year
# Loop through years
for year in range(startyear, endyear + 1):
if year == startyear:
startmonth = starttime.month
else:
startmonth = 1
if year == endyear:
endmonth = endtime.month
else:
endmonth = 12
# Loop through months
for month in range(startmonth, endmonth + 1):
if verbose:
print('Loading IMP merged probe {}, {:02d}/{}'.format(
probe, month, year))
intervalstring = str(year) + str(month).zfill(2)
filename = 'imp_min_merge' + intervalstring + '.asc'
# Location of file relative to local directory or remote url
relative_loc = os.path.join('imp' + probe, 'merged')
local_dir = os.path.join(imp_dir, relative_loc)
hdffile = os.path.join(local_dir, filename[:-4] + '.hdf')
if os.path.isfile(hdffile):
data.append(pd.read_hdf(hdffile))
continue
remote_url = imp_url + relative_loc
f = util.load(filename, local_dir, remote_url)
if f is None:
print('File {}/{} not available\n'.format(
remote_url, filename))
continue
readargs = {
'names': [
'Year', 'doy', 'Hour', 'Minute', 'sw_flag', 'x_gse',
'y_gse', 'z_gse', 'y_gsm', 'z_gsm', 'Nm', 'FCm', 'DWm',
'<|B|>', '|<B>|', '<B_lat>', '<B_long>', 'Bx_gse',
'By_gse', 'Bz_gse', 'By_gsm', 'Bz_gsm', 'sigma|B|',
'sigma B', 'sigma B_x', 'sigma B_y', 'sigma B_z',
'plas_reg', 'Npp', 'FCp', 'DWp', 'v_fit', 'vx_fit_gse',
'vy_fit_gse', 'vz_fit_gse', 'vlong_fit', 'vlat_fit',
'np_fit', 'Tp_fit', 'v_mom', 'vx_mom_gse', 'vy_mom_gse',
'vz_mom_gse', 'vlong_mom', 'vlat_mom', 'np_mom', 'Tp_mom'
],
'na_values': [
'9999', '999', '99', '99', '9', '9999.99', '9999.99',
'9999.99', '9999.99', '9999.99', '9', '99', '9.99',
'9999.99', '9999.99', '9999.99', '9999.99', '9999.99',
'9999.99', '9999.99', '9999.99', '9999.99', '9999.99',
'9999.99', '9999.99', '9999.99', '9999.99', '9', '9', '99',
'9.99', '9999.9', '9999.9', '9999.9', '9999.9', '9999.9',
'9999.9', '9999.9', '9999999.', '9999.9', '9999.9',
'9999.9', '9999.9', '9999.9', '9999.9', '9999.9',
'9999999.'
],
'delim_whitespace':
True
}
# Read in data
thisdata = pd.read_table(f, **readargs)
thisdata['Time'] = (
pd.to_datetime(thisdata['Year'], format='%Y') +
pd.to_timedelta(thisdata['doy'] - 1, unit='d') +
pd.to_timedelta(thisdata['Hour'], unit='h') +
pd.to_timedelta(thisdata['Minute'], unit='m'))
if use_hdf:
thisdata.to_hdf(hdffile, key='distparams', mode='w')
data.append(thisdata)
return util.timefilter(data, starttime, endtime)
def merged(probe, starttime, endtime, verbose=True, try_download=True):
"""
Read in merged data set.
Parameters
----------
probe : int, string
Helios probe to import data from. Must be 1 or 2.
starttime : datetime
Interval start time
endtime : datetime
Interval end time
verbose : bool, optional
If ``True``, print information as data is loading.
Default is ``True``.
Returns
-------
data : DataFrame
Merged data set
Notes
-----
This is an old dataset, and it is recommended to use `corefit` instead.
"""
probe = _check_probe(probe)
daylist = util._daysplitinterval(starttime, endtime)
data = []
floc = _merged_localdir(probe)
for day in daylist:
this_date = day[0]
# Check that data for this day exists
if probe == '1':
if this_date < date(1974, 12, 12) or this_date > date(1985, 9, 4):
continue
if probe == '2':
if this_date < date(1976, 1, 17) or this_date > date(1980, 3, 8):
continue
doy = int(this_date.strftime('%j'))
year = this_date.year
hdfloc = os.path.join(floc, _merged_fname(probe, year, doy) + '.hdf')
# Data not processed yet, try to process and load it
if not os.path.isfile(hdfloc):
try:
data.append(
_merged_fromascii(probe,
year,
doy,
try_download=try_download))
if verbose:
print(year, doy, 'Processed ascii file')
except (FileNotFoundError, URLError) as err:
if verbose:
print(str(err))
print(year, doy, 'No raw merged data')
else:
# Load data from already processed file
data.append(pd.read_hdf(hdfloc, 'table'))
if verbose:
print(year, doy)
if data == []:
fmt = '%d-%m-%Y'
raise ValueError('No data to import for probe ' + probe + ' between ' +
starttime.strftime(fmt) + ' and ' +
endtime.strftime(fmt))
return util.timefilter(data, starttime, endtime)
def ion_dists(probe, starttime, endtime, remove_advect=False, verbose=False):
"""
Return 3D ion distributions between *starttime* and *endtime*
Parameters
----------
probe : int
Helios probe to import data from. Must be 1 or 2.
starttime : datetime
Start of interval
endtime : datetime
End of interval
remove_advect : bool, optional
If *False*, the distribution is returned in
the spacecraft frame.
If *True*, the distribution is
returned in the solar wind frame, by subtracting the spacecraft
velocity from the velcoity of each bin. Note this significantly
slows down reading in the distribution.
verbose : bool, optional
If ``True``, print dates when loading files. Default is ``False``.
Returns
-------
distinfo : Series
Infromation stored in the top of distribution function files.
"""
extensions = ['hdm.0', 'hdm.1', 'ndm.0', 'ndm.1']
distlist = []
# Loop through each day
starttime_orig = starttime
while starttime < endtime:
year = starttime.year
doy = int(starttime.strftime('%j'))
if verbose:
print('Loading ion dists from year', year, 'doy', doy)
# Directory for today's distribution files
dist_dir = _dist_file_dir(probe, year, doy)
# If directory doesn't exist, print error and continue
if not os.path.exists(dist_dir):
print('No ion distributions available for year', year, 'doy', doy)
starttime += timedelta(days=1)
continue
# Locaiton of hdf file to save to/load from
hdffile = 'h' + probe + str(year) + str(doy).zfill(3) +\
'ion_dists.hdf'
hdffile = os.path.join(dist_dir, hdffile)
if os.path.isfile(hdffile):
todays_dist = pd.read_hdf(hdffile)
distlist.append(todays_dist)
starttime += timedelta(days=1)
continue
todays_dist = []
# Get every distribution function file present for this day
for f in os.listdir(dist_dir):
path = os.path.join(dist_dir, f)
# Check for distribution function
if path[-5:] in extensions:
hour, minute, second = _dist_filename_to_hms(path)
try:
d = ion_dist_single(probe, year, doy, hour, minute, second)
except RuntimeError as err:
strerr = 'No ion distribution function data in file'
if str(err) == strerr:
continue
raise err
t = datetime.combine(starttime.date(),
time(hour, minute, second))
d['Time'] = t
if verbose:
print(t)
todays_dist.append(d)
if todays_dist == []:
starttime += timedelta(days=1)
continue
todays_dist = pd.concat(todays_dist)
todays_dist = todays_dist.set_index('Time', append=True)
if use_hdf:
todays_dist.to_hdf(hdffile, key='ion_dist', mode='w')
distlist.append(todays_dist)
starttime += timedelta(days=1)
if distlist == []:
raise RuntimeError('No data available for times ' +
str(starttime_orig) + ' to ' + str(endtime))
return util.timefilter(distlist, starttime_orig, endtime)
def distparams(probe, starttime, endtime, verbose=False):
"""
Read in distribution parameters found in the header of distribution files.
Parameters
----------
probe : int
Helios probe to import data from. Must be 1 or 2.
starttime : datetime
Start of interval
endtime : datetime
End of interval
verbose : bool, optional
If ``True``, print information whilst loading. Default is ``False``.
Returns
-------
distinfo : Series
Infromation stored in the top of distribution function files
"""
extensions = ['hdm.0', 'hdm.1', 'ndm.0', 'ndm.1']
paramlist = []
starttime_orig = starttime
# Loop through each day
while starttime < endtime:
year = starttime.year
doy = starttime.strftime('%j')
# Directory for today's distribution files
dist_dir = _dist_file_dir(probe, year, doy)
# Locaiton of hdf file to save to/load from
hdffile = 'h' + probe + str(year) + str(doy).zfill(3) +\
'distparams.hdf'
hdffile = os.path.join(dist_dir, hdffile)
if os.path.isfile(hdffile):
todays_params = pd.read_hdf(hdffile)
elif not os.path.isdir(dist_dir):
starttime += timedelta(days=1)
continue
else:
todays_params = []
# Get every distribution function file present for this day
for f in os.listdir(dist_dir):
path = os.path.join(dist_dir, f)
# Check for distribution function
if path[-5:] in extensions:
hour, minute, second = _dist_filename_to_hms(path)
if verbose:
print(starttime.date(), hour, minute, second)
p = distparams_single(probe, year, doy, hour, minute,
second)
todays_params.append(p)
todays_params = pd.concat(todays_params, ignore_index=True,
axis=1).T
todays_params = todays_params.set_index('Time', drop=False)
# Convert columns to numeric types
todays_params = todays_params.apply(pd.to_numeric, errors='ignore')
if use_hdf:
todays_params.to_hdf(hdffile, key='distparams', mode='w')
paramlist.append(todays_params)
starttime += timedelta(days=1)
return util.timefilter(paramlist, starttime_orig, endtime)
def fpi_dis_moms(probe, mode, starttime, endtime):
"""
Import fpi ion distribution moment data.
Parameters
----------
probe : string
Probe number, must be 1, 2, 3, or 4
mode : string
Data mode, must be 'fast' or 'brst'
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
Imported data.
"""
valid_modes = ['fast', 'brst']
if mode not in valid_modes:
raise RuntimeError('Mode must be either fast or brst')
# Directory relative to main MMS data directory
relative_dir = os.path.join('mms' + probe, 'fpi', mode, 'l2', 'dis-moms')
daylist = util._daysplitinterval(starttime, endtime)
data = []
for day in daylist:
date = day[0]
starthour = day[1].hour
endhour = day[2].hour + 1
# fips fast data product has files every two hours, so get nearest two
# hour stamps
starthour -= np.mod(starthour, 2)
endhour += np.mod(endhour, 2)
for h in range(starthour, endhour, 2):
this_relative_dir = os.path.join(relative_dir,
str(date.year),
str(date.month).zfill(2))
filename = ('mms{}_fpi_{}_l2_dis-moms_'
'{}{:02}{:02}{:02}0000_v3.3.0.cdf').format(
probe, mode, date.year, date.month, date.day, h)
# Absolute path to local directory for this data file
local_dir = os.path.join(mms_dir, this_relative_dir)
util._checkdir(local_dir)
remote_url = remote_mms_dir + this_relative_dir
# Load cdf file
try:
cdf = util.load(filename, local_dir, remote_url)
except urllib.error.HTTPError as e:
if str(e) == 'HTTP Error 404: Not Found':
print('No data available for hours', str(h) + '-' +
str(h + 2), 'on', date.strftime('%d/%m/%Y'))
continue
else:
raise
probestr = 'mms' + probe + '_'
# Convert cdf to dataframe
keys = {'Epoch': 'Time',
probestr + 'dis_bulkv_gse_fast': ['bulkv_x',
'bulkv_y',
'bulkv_z'],
probestr + 'dis_heatq_gse_fast': ['heatq_x',
'heatq_y',
'heatq_z'],
probestr + 'dis_numberdensity_fast': 'n',
probestr + 'dis_temppara_fast': 'T_par',
probestr + 'dis_tempperp_fast': 'T_perp'}
df = util.cdf2df(cdf, 'Epoch', keys)
data.append(df)
return util.timefilter(data, starttime, endtime)
def threedp_pm(starttime, endtime):
"""
Import 'pm' wind data.
3 second time resolution solar wind proton and alpha particle moments from
the PESA LOW sensor, computed on-board the spacecraft
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
"""
# Directory relative to main WIND data directory
relative_dir = path.Path('3dp') / '3dp_pm'
daylist = util._daysplitinterval(starttime, endtime)
data = []
for day in daylist:
date = day[0]
this_relative_dir = relative_dir / str(day[0].year)
# Absolute path to local directory for this data file
local_dir = wind_dir / this_relative_dir
filename = 'wi_pm_3dp_' +\
str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2) +\
'_v05.cdf'
hdfname = filename[:-4] + 'hdf'
hdfloc = local_dir / hdfname
if hdfloc.exists():
df = pd.read_hdf(hdfloc)
data.append(df)
continue
util._checkdir(local_dir)
remote_url = remote_wind_dir + str(this_relative_dir)
cdf = util.load(filename, local_dir, remote_url, guessversion=True)
if cdf is None:
print('File {}/{} not available\n'.format(remote_url, filename))
continue
keys = {
'A_DENS': 'n_a',
'A_TEMP': 'T_a',
'A_VELS': ['va_x', 'va_y', 'va_z'],
'P_DENS': 'n_p',
'P_TEMP': 'T_p',
'P_VELS': ['vp_x', 'vp_y', 'vp_z'],
'Epoch': 'Time'
}
df = util.cdf2df(cdf, index_key='Epoch', keys=keys)
if use_hdf:
df.to_hdf(hdfloc, 'pm', mode='w')
data.append(df)
return util.timefilter(data, starttime, endtime)
def mag_4hz(probe, starttime, endtime, verbose=True, try_download=True):
"""
Read in 4Hz magnetic field data.
Parameters
----------
probe : int, string
Helios probe to import data from. Must be 1 or 2.
starttime : datetime
Interval start time
endtime : datetime
Interval end time
verbose : bool, optional
If ``True``, print more information as data is loading.
Default is ``True``.
try_download : bool, optional
If ``False`` don't try to download data if it is missing locally.
Default is ``False``.
Returns
-------
data : DataFrame
4Hz magnetic field data set
"""
probe = _check_probe(probe)
data = []
# Loop through years
for year in range(starttime.year, endtime.year + 1):
floc = _4hz_localdir(probe)
# Calculate start day of year
if year == starttime.year:
startdoy = int(starttime.strftime('%j'))
else:
startdoy = 1
# Calculate end day of year
if year == endtime.year:
enddoy = int(endtime.strftime('%j'))
else:
enddoy = 366
# Loop through days of year
for doy in range(startdoy, enddoy + 1):
hdfloc = os.path.join(floc,
_4hz_filename(probe, year, doy) + '.hdf')
if not os.path.isfile(hdfloc):
# Data not processed yet, try to process and load it
try:
data.append(
_fourHz_fromascii(probe,
year,
doy,
try_download=try_download))
except ValueError as err:
if 'No mag data available' in str(err):
if verbose:
print('{}/{:03} '
'4Hz mag data not available'.format(
year, doy))
else:
raise
else:
# Load data from already processed file
data.append(pd.read_hdf(hdfloc, 'table'))
if verbose:
print('{}/{:03} 4Hz mag data loaded'.format(year, doy))
if data == []:
raise ValueError('No raw 4Hz mag data available')
data = util.timefilter(data, starttime, endtime)
if data.empty:
raise ValueError('No 4Hz raw mag data available for entire interval')
return (data)
def mag15s(probe, starttime, endtime, verbose=False):
"""
Import 15s cadence magnetic field data.
Parameters
----------
probe : string
Probe number.
starttime : datetime
Start of interval.
endtime : datetime
End of interval.
verbose : bool, optional
If ``True``, print information whilst loading. Default is ``False``.
Returns
-------
data : DataFrame
Requested data.
"""
data = []
dtimes = util._daysplitinterval(starttime, endtime)
# Loop through years
for dtime in dtimes:
startdt = dtime[0]
year = startdt.year
doy = util.dtime2doy(startdt)
if verbose:
print('Loading IMP 15s mag probe {}, {:03d}/{}'.format(
probe, doy, year))
filename = '{}{:03d}_imp{}_mag_15s_v3.asc'.format(year, doy, probe)
hdffname = filename[:-3] + 'hdf'
# Location of file relative to local directory or remote url
relative_loc = os.path.join('imp{}'.format(probe), 'mag',
'15s_ascii_v3', str(year))
local_dir = os.path.join(imp_dir, relative_loc)
hdffile = os.path.join(local_dir, hdffname)
if os.path.exists(hdffile):
thisdata = pd.read_hdf(hdffile)
data.append(thisdata)
continue
remote_url = imp_url + relative_loc
f = util.load(filename, local_dir, remote_url)
readargs = {
'names': [
'Year', 'doy', 'Second', 'Source flag', 'n points', 'x gse',
'y gse', 'z gse', 'y gsm', 'z gsm', '|B|', 'Bx gse', 'By gse',
'Bz gse', 'By gsm', 'Bz gsm', 'Bxx gse', 'Byy gse', 'Bzz gse',
'Byx gse', 'Bzx gse', 'Bzy gse', 'Time shift', 'sw flag'
],
'na_values': [
'9999', '999', '99', '9', '999', '99.99', '99.99', '99.99',
'99.99', '99.99', '9999.99', '9999.99', '9999.99', '9999.99',
'9999.99', '9999.99', '9999.99', '9999.99', '9999.99',
'9999.99', '9999.99', '9999.99', '999.9', '9'
],
'delim_whitespace':
True
}
# Read in data
thisdata = pd.read_table(f, **readargs)
thisdata['Time'] = (pd.to_datetime(thisdata['Year'], format='%Y') +
pd.to_timedelta(thisdata['doy'] - 1, unit='d') +
pd.to_timedelta(thisdata['Second'], unit='s'))
thisdata = thisdata.set_index('Time', drop=False)
thisdata = thisdata.drop(['Year', 'doy', 'Second'], 1)
if use_hdf:
thisdata.to_hdf(hdffile, key='distparams', mode='w')
data.append(thisdata)
return util.timefilter(data, starttime, endtime)
def swoops_ions(starttime, endtime):
"""
Import SWOOPS ion data.
Parameters
----------
starttime : datetime
Start of interval
endtime : datetime
End of interval
Returns
-------
data : DataFrame
Requested data
"""
swoops_options = url_options
readargs = {
'names': [
'year', 'doy', 'hour', 'minute', 'second', 'r', 'hlat', 'hlon',
'n_p', 'n_a', 'T_p_large', 'T_p_small', 'v_r', 'v_t', 'v_n',
'iqual'
],
'delim_whitespace':
True
}
data = []
months_loaded = []
dtimes = util._daysplitinterval(starttime, endtime)
# Loop through individual days
for dtime in dtimes:
thisdate = dtime[0]
# Get first day of the month
first_day = date(thisdate.year, thisdate.month, 1)
# Check if this month's data already loaded
if first_day in months_loaded:
continue
doy = first_day.strftime('%j')
swoops_options['FILE_NAME'] = ('u' + first_day.strftime('%y') + doy +
'bam.dat')
swoops_options['FILE_PATH'] =\
('/ufa/stageIngestArea/swoops/ions/bamion' +
first_day.strftime('%y') + '.zip_files')
# Put together url for this days data
remote_url = ulysses_url
for key in swoops_options:
remote_url += key + '=' + swoops_options[key] + '&'
# Local locaiton to download to
local_dir = os.path.join(ulysses_dir, 'swoops', 'ions',
first_day.strftime('%Y'))
# Load data
try:
f = util.load(swoops_options['FILE_NAME'], local_dir, remote_url)
except HTTPError:
print('No SWOOPS ion data available for date %s' % first_day)
continue
# Read in data
thisdata = pd.read_table(f, **readargs)
# Process data/time
thisdata = _convert_ulysses_time(thisdata)
data.append(thisdata)
months_loaded.append(first_day)
return util.timefilter(data, starttime, endtime)
def threedp_sfpd(starttime, endtime):
"""
Import 'sfpd' wind data.
12 second energetic electron pitch-angle energy spectra from the foil SST
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
"""
# Directory relative to main WIND data directory
relative_dir = path.Path('3dp') / '3dp_sfpd'
daylist = util._daysplitinterval(starttime, endtime)
data = []
mag = []
for (date, _, _) in daylist:
this_relative_dir = relative_dir / str(date.year)
# Absolute path to local directory for this data file
local_dir = wind_dir / this_relative_dir
filename = 'wi_sfpd_3dp_{:{dfmt}}_v02'.format(date, dfmt='%Y%m%d')
hdfname = filename + '.hdf'
hdfloc = local_dir / hdfname
if hdfloc.exists():
df = pd.read_hdf(hdfloc)
data.append(df)
continue
util._checkdir(local_dir)
remote_url = remote_wind_dir + str(this_relative_dir)
cdf = util.load(filename + '.cdf',
local_dir,
remote_url,
guessversion=True)
if cdf is None:
print('File {}/{} not available\n'.format(remote_url, filename))
continue
data_today = []
# Loop through each timestamp to build up fluxes
for i, time in enumerate(cdf['Epoch'][...]):
energies = cdf['ENERGY'][i, :]
angles = cdf['PANGLE'][i, :]
fluxes = cdf['FLUX'][i, :, :]
magfield = cdf['MAGF'][i, :]
index = pd.MultiIndex.from_product(
([time], energies, angles),
names=['Time', 'Energy', 'Pitch angle'])
df = pd.DataFrame(fluxes.ravel(), index=index, columns=['Flux'])
df['Bx'] = magfield[0]
df['By'] = magfield[1]
df['Bz'] = magfield[2]
data_today.append(df)
data_today = pd.concat(data_today)
data_today = data_today.sort_index()
if use_hdf:
data_today.to_hdf(hdfloc, 'sfpd', mode='w')
data.append(data_today)
data = util.timefilter(data, starttime, endtime)
data = data.reset_index(level=['Energy', 'Pitch angle'])
return data
def mag_ness(probe, starttime, endtime, verbose=True, try_download=True):
"""
Read in 6 second magnetic field data.
Parameters
----------
probe : int, string
Helios probe to import data from. Must be 1 or 2.
starttime : datetime
Interval start time
endtime : datetime
Interval end time
verbose : bool, optional
If ``True``, print more information as data is loading. Default is
``True``.
Returns
-------
data : DataFrame
6 second magnetic field data set
"""
probe = _check_probe(probe)
startdate = starttime.date()
enddate = endtime.date()
def _check_doy(probe, year, doy):
'''
Returns False if year and doy are out of bounds for given probe
'''
if probe == '1':
minyear = 1974
mindoy = 349
maxyear = 1981
maxdoy = 167
elif probe == '2':
minyear = 1976
mindoy = 17
maxyear = 1980
maxdoy = 68
if (year == minyear and doy < mindoy) or (year < minyear):
return False
elif (year == maxyear and doy > maxdoy) or (year > maxyear):
return False
else:
return True
data = []
# Loop through years
for year in range(startdate.year, enddate.year + 1):
floc = _ness_localdir(probe, year)
# Calculate start day
startdoy = 1
if year == startdate.year:
startdoy = int(startdate.strftime('%j'))
# Calculate end day
enddoy = 366
if year == enddate.year:
enddoy = int(enddate.strftime('%j'))
# Loop through days of year
for doy in range(startdoy, enddoy + 1):
nodatastr = '{}/{:03} 6s mag data not available'.format(year, doy)
datastr = '{}/{:03} 6s mag data loaded'.format(year, doy)
if not _check_doy(probe, year, doy):
if verbose:
print(nodatastr)
continue
hdfloc = os.path.join(floc, _ness_fname(probe, year, doy) + 'hdf')
if os.path.isfile(hdfloc):
# Load data from already processed file
data.append(pd.read_hdf(hdfloc, 'table'))
print(datastr)
continue
# Data not processed yet, try to process and load it
try:
data.append(
_mag_ness_fromascii(probe,
year,
doy,
try_download=try_download))
if verbose:
print(datastr)
except ValueError:
if verbose:
print(nodatastr)
if data == []:
raise ValueError('No 6s mag data avaialble between '
'{} and {}'.format(starttime, endtime))
return util.timefilter(data, starttime, endtime)
