def _load(probe, starttime, endtime, instrument, product_id, cdfkeys):
daylist = util._daysplitinterval(starttime, endtime)
data = []
for day in daylist:
date = day[0]
year = str(date.year)
month = str(date.month).zfill(2)
day = str(date.day).zfill(2)
local_dir = cluster_dir / ('c' + probe) / instrument / year
local_fname = 'C' + probe + '_' + product_id + '__' +\
year + month + day + '.cdf'
local_file = local_dir / local_fname
# If we don't have local file download it
if not local_file.exists():
thisstart = datetime.combine(date, time.min)
thisend = datetime.combine(date, time.max)
try:
_download(probe, thisstart, thisend, instrument, product_id)
except Exception as err:
print(str(err), '\n')
continue
from pycdf import pycdf
cdf = pycdf.CDF(os.path.join(local_dir, local_fname))
for key, value in cdfkeys.items():
if value == 'Time':
index_key = key
break
data.append(util.cdf2df(cdf, index_key, cdfkeys))
if len(data) == 0:
raise RuntimeError('No data available to download during requested '
'times')
return util.timefilter(data, starttime, endtime)
def mitplasma_h0(probe, starttime, endtime):
"""
Import mit h0 plasma data.
Parameters
----------
probe : string
Probe number.
starttime : datetime
Start of interval.
endtime : datetime
End of interval.
Returns
-------
data : DataFrame
Requested data.
"""
data = []
dtimes = util._daysplitinterval(starttime, endtime)
# Loop through years
for dtime in dtimes:
date = dtime[0]
intervalstring = str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2)
filename = 'i' + probe + '_h0_mitplasma_' + intervalstring + '_v01.cdf'
# Location of file relative to local directory or remote url
relative_loc = 'imp' + probe + '/plasma_mit/mitplasma_h0/' +\
str(date.year)
local_dir = os.path.join(imp_dir, relative_loc)
remote_url = imp_url + relative_loc
cdf = util.load(filename, local_dir, remote_url)
keys = {
'EW_flowangle_best': 'EW_flowangle_best',
'EW_flowangle_mom': 'EW_flowangle_best',
'Epoch': 'Time',
'Flow_elevation_thresh': 'Flow_elevation_thresh',
'Flow_elevation_threshsp': 'Flow_elevation_threshsp',
'Region': 'Region',
'V_fit': 'V_fit',
'V_mom': 'V_fit',
'mode': 'mode',
'protonV_thermal_fit': 'protonV_thermal_fit',
'protonV_thermal_mom': 'protonV_thermal_fit',
'proton_density_fit': 'proton_density_fit',
'proton_density_mom': 'proton_density_mom',
'xyzgse': ['x_gse', 'y_gse', 'z_gse'],
'ygsm': 'ygsm',
'zgsm': 'zgsm'
}
thisdata = util.cdf2df(cdf, 'Epoch', keys)
data.append(thisdata)
data = pd.concat(data)
data = data[(data['Time'] > starttime) & (data['Time'] < endtime)]
data.index.name = 'Time'
return data
def mag320ms(probe, startTime, endTime):
"""
Import 320ms cadence magnetic field data.
Parameters
----------
probe : string
Probe number.
starttime : datetime
Start of interval.
endtime : datetime
End of interval.
Returns
-------
data : DataFrame
Requested data.
"""
data = []
dtimes = util._daysplitinterval(startTime, endTime)
# Loop through years
for dtime in dtimes:
date = dtime[0]
intervalstring = str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2)
filename = 'i8_320msec_mag_' + intervalstring + '_v01.cdf'
hdffname = filename[:-3] + '.hdf'
# Location of file relative to local directory or remote url
relative_loc = 'imp' + probe + '/mag/mag_320msec_cdaweb/' +\
str(date.year)
local_dir = os.path.join(imp_dir, relative_loc)
hdffile = os.path.join(local_dir, hdffname)
if os.path.exists(hdffile):
thisdata = pd.read_hdf(hdffile)
data.append(thisdata)
continue
remote_url = imp_url + relative_loc
cdf = util.load(filename, local_dir, remote_url)
keys = {
'B': '|B|',
'BX': 'Bx',
'BY': 'By',
'BZ': 'Bz',
'Epoch': 'Time'
}
thisdata = util.cdf2df(cdf, 'Epoch', keys)
data.append(thisdata)
if use_hdf:
thisdata.to_hdf(hdffile, key='merged', mode='w')
data = pd.concat(data)
data = data[(data['Time'] > startTime) & (data['Time'] < endTime)]
data = data.drop(columns='Time')
data.index.name = 'Time'
return data
def processing_func(cdf):
epoch_dict = {'h0': 'Epoch3', 'h2': 'Epoch'}
epoch_key = epoch_dict[version]
badvalues = {'Bx_gse': -1e+31, 'By_gse': -1e+31, 'Bz_gse': -1e+31}
df = util.cdf2df(cdf,
index_key=epoch_key,
badvalues=badvalues,
ignore=ignore)
return df
def processing_func(cdf, **kwargs):
probe = kwargs.pop('probe')
rate = kwargs.pop('rate')
kwargs.pop('coords')
probestr = 'th' + probe
ratestr = '_fg' + rate + '_'
df = util.cdf2df(cdf, probestr + ratestr + 'time', dtimeindex=False)
df = df.set_index(pd.to_datetime(df.index.values, unit='s'))
df.index.name = 'Time'
return df
def processing_func(file):
for non_empty_var in list(file.cdf_info().keys()):
if 'variable' in non_empty_var.lower():
if len(file.cdf_info()[non_empty_var]) > 0:
var_list = non_empty_var
break
for key in file.cdf_info()[var_list]:
if 'CDF_EPOCH' in file.varget(key, expand=True).values():
index_key = key
break
return util.cdf2df(file, index_key)
def processing_func(cdf):
epoch_dict = {'h0': 'Epoch3', 'h2': 'Epoch'}
mag_dict = {'h0': 'B3GSE', 'h2': 'BGSE'}
epoch_key = epoch_dict[version]
mag_key = mag_dict[version]
keys = {mag_key: ['Bx_gse', 'By_gse', 'Bz_gse'], epoch_key: 'Time'}
badvalues = {'Bx_gse': -1e+31, 'By_gse': -1e+31, 'Bz_gse': -1e+31}
df = util.cdf2df(cdf,
index_key=epoch_key,
keys=keys,
badvalues=badvalues)
return df
def fgm_survey(probe, starttime, endtime):
"""
Import fgm survey mode magnetic field data.
Parameters
----------
probe : string
Probe number, must be 1, 2, 3, or 4
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
Imported data.
"""
# Directory relative to main MMS data directory
relative_dir = os.path.join('mms' + probe, 'fgm', 'srvy', 'l2')
daylist = util._daysplitinterval(starttime, endtime)
data = []
for day in daylist:
date = day[0]
this_relative_dir = os.path.join(relative_dir, str(date.year),
str(date.month).zfill(2))
filename = 'mms{}_fgm_srvy_l2_{}{:02}{:02}_v4.18.0.cdf'.format(
probe, date.year, date.month, date.day)
# Absolute path to local directory for this data file
local_dir = os.path.join(mms_dir, this_relative_dir)
util._checkdir(local_dir)
remote_url = remote_mms_dir + this_relative_dir
# Load cdf file
cdf = util.load(filename, local_dir, remote_url)
# Convert cdf to dataframe
keys = {
'mms' + probe + '_fgm_b_gsm_srvy_l2': ['Bx', 'By', 'Bz', 'Br'],
'Epoch': 'Time'
}
df = util.cdf2df(cdf, 'Epoch', keys)
data.append(df)
return util.timefilter(data, starttime, endtime)
def processing_func(cdf, **kwargs):
probe = kwargs.pop('probe')
rate = kwargs.pop('rate')
coords = kwargs.pop('coords')
probestr = 'th' + probe
ratestr = '_fg' + rate + '_'
keys = {
probestr + ratestr + 'btotal':
'|B|',
probestr + ratestr + coords:
['Bx_' + coords, 'By_' + coords, 'Bz_' + coords]
}
df = util.cdf2df(cdf,
probestr + ratestr + 'time',
keys,
dtimeindex=False)
df = df.set_index(pd.to_datetime(df.index.values, unit='s'))
df.index.name = 'Time'
return df
def processing_func(directory, fname):
cdf = util.load(fname, directory, '')
return util.cdf2df(cdf, 'Epoch', badvalues=badvalues)
def processing_func(cdf):
return util.cdf2df(cdf, index_key='Epoch')
def processing_func(cdf):
return util.cdf2df(cdf, 'Epoch', ignore=ignore)
def mag_rtn(starttime, endtime):
"""
Import magnetic field in RTN coordinates from Messenger.
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
"""
# Directory relative to main WIND data directory
relative_dir = 'rtn'
daylist = util._daysplitinterval(starttime, endtime)
data = []
for day in daylist:
date = day[0]
this_relative_dir = os.path.join(relative_dir, str(date.year))
hdffile = 'messenger_mag_rtn_' +\
str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2) +\
'_v01.hdf'
hdfloc = os.path.join(mess_dir, this_relative_dir, hdffile)
# Try to load hdf file
if os.path.isfile(hdfloc):
df = pd.read_hdf(hdfloc)
data.append(df)
continue
filename = hdffile[:-4] + '.cdf'
# Absolute path to local directory for this data file
local_dir = os.path.join(mess_dir, this_relative_dir)
util._checkdir(local_dir)
remote_url = os.path.join(remote_mess_dir, this_relative_dir)
cdf = util.load(filename, local_dir, remote_url, guessversion=True)
if cdf is None:
print('File {}/{} not available\n'.format(remote_url, filename))
continue
keys = {
'B_normal': 'Bn',
'B_radial': 'Br',
'B_tangential': 'Bt',
'Epoch': 'Time',
'azimuth_ecliptic': 'sc_Az',
'latitude_ecliptic': 'sc_Lat',
'radialDistance': 'sc_r',
'MissionElapsedTime': 'mission_time'
}
df = util.cdf2df(cdf, index_key='Epoch', keys=keys)
if use_hdf:
hdffile = filename[:-4] + '.hdf'
df.to_hdf(hdfloc, key='data', mode='w')
data.append(df)
return util.timefilter(data, starttime, endtime)
def processing_func(cdf):
if want_xr:
return util.cdf2xr(cdf, starttime, endtime, 'Epoch', product_list)
else:
return util.cdf2df(cdf, starttime, endtime, 'Epoch', product_list)
def processing_func(cdf):
return util.cdf2df(cdf, 'Epoch', badvalues=badvalues)
def fgm(probe, rate, coords, starttime, endtime):
"""
Import fgm magnetic field data from THEMIS.
Parameters
----------
probe : string
Alowed values are [a, b, c, d, e].
rate : string
Date rate to return. Allowed values are [e, h, l, s].
coords : string
Magnetic field co-ordinate system. Allowed values are
[dsl, gse, gsm, ssl]. NOTE: Add link to co-ordinate system
descriptions.
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
"""
valid_rates = ['e', 'h', 'l', 's']
valid_coords = ['dsl', 'gse', 'gsm', 'ssl']
_validate_probe(probe)
if rate not in valid_rates:
raise ValueError(('rate argument %s is not in list of allowed'
'rates: %s') % (rate, valid_rates))
if coords not in valid_coords:
raise ValueError(('coords argument %s is not in list of allowed'
'co-ordinate systems: %s') % (rate, valid_rates))
# Directory relative to main THEMIS data directory
relative_dir = os.path.join('th' + probe, 'l2', 'fgm')
daylist = util._daysplitinterval(starttime, endtime)
data = []
for day in daylist:
date = day[0]
this_relative_dir = os.path.join(relative_dir, str(date.year))
filename = 'th' + probe + '_l2_fgm_' +\
str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2) +\
'_v01.cdf'
# Absolute path to local directory for this data file
local_dir = os.path.join(themis_dir, this_relative_dir)
util._checkdir(local_dir)
remote_url = remote_themis_dir + this_relative_dir
cdf = util.load(filename, local_dir, remote_url)
if cdf is None:
print('File {}/{} not available\n'.format(remote_url, filename))
continue
probestr = 'th' + probe
ratestr = '_fg' + rate + '_'
keys = {
probestr + ratestr + 'btotal':
'|B|',
probestr + ratestr + coords:
['Bx_' + coords, 'By_' + coords, 'Bz_' + coords],
probestr + ratestr + 'time':
'Time'
}
df = util.cdf2df(cdf,
probestr + ratestr + 'time',
keys,
dtimeindex=False)
df = df.set_index(pd.to_datetime(df.index.values, unit='s'))
df['Time'] = df.index.values
data.append(df)
data = pd.concat(data)
data = data[(data['Time'] > starttime) & (data['Time'] < endtime)]
return data
def processing_func(cdf):
df = util.cdf2df(cdf, 'Epoch')
return df
def processing_func(cdf):
thisdata = util.cdf2df(cdf, 'Epoch', ignore=ignore)
return thisdata
def processing_func(cdf):
return util.cdf2df(cdf,
index_key='Epoch',
keys=keys,
badvalues=badvalues)
def load_local_file(self, interval):
local_path = self.local_path(interval)
cdf = util._load_cdf(local_path)
return util.cdf2df(cdf, index_key='EPOCH')
def processing_func(cdf):
df = util.cdf2df(cdf, 'Epoch1', ignore=ignore)
df.index.name = 'Time'
return df
def load_local_file(self, interval):
local_path = self.local_path(interval)
cdf = util._load_cdf(local_path)
return util.cdf2df(cdf, index_key='Epoch', badvalues=self.badvalues)
def load_local_file(self, interval):
# Read in data
cdf = util._load_cdf(self.local_path(interval), )
df = util.cdf2df(cdf, 'Epoch1', ignore=['Time1_PB5'])
df.index.name = 'Time'
return df
def fpi_dis_moms(probe, mode, starttime, endtime):
"""
Import fpi ion distribution moment data.
Parameters
----------
probe : string
Probe number, must be 1, 2, 3, or 4
mode : string
Data mode, must be 'fast' or 'brst'
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
Imported data.
"""
valid_modes = ['fast', 'brst']
if mode not in valid_modes:
raise RuntimeError('Mode must be either fast or brst')
# Directory relative to main MMS data directory
relative_dir = os.path.join('mms' + probe, 'fpi', mode, 'l2', 'dis-moms')
daylist = util._daysplitinterval(starttime, endtime)
data = []
for day in daylist:
date = day[0]
starthour = day[1].hour
endhour = day[2].hour + 1
# fips fast data product has files every two hours, so get nearest two
# hour stamps
starthour -= np.mod(starthour, 2)
endhour += np.mod(endhour, 2)
for h in range(starthour, endhour, 2):
this_relative_dir = os.path.join(relative_dir,
str(date.year),
str(date.month).zfill(2))
filename = ('mms{}_fpi_{}_l2_dis-moms_'
'{}{:02}{:02}{:02}0000_v3.3.0.cdf').format(
probe, mode, date.year, date.month, date.day, h)
# Absolute path to local directory for this data file
local_dir = os.path.join(mms_dir, this_relative_dir)
util._checkdir(local_dir)
remote_url = remote_mms_dir + this_relative_dir
# Load cdf file
try:
cdf = util.load(filename, local_dir, remote_url)
except urllib.error.HTTPError as e:
if str(e) == 'HTTP Error 404: Not Found':
print('No data available for hours', str(h) + '-' +
str(h + 2), 'on', date.strftime('%d/%m/%Y'))
continue
else:
raise
probestr = 'mms' + probe + '_'
# Convert cdf to dataframe
keys = {'Epoch': 'Time',
probestr + 'dis_bulkv_gse_fast': ['bulkv_x',
'bulkv_y',
'bulkv_z'],
probestr + 'dis_heatq_gse_fast': ['heatq_x',
'heatq_y',
'heatq_z'],
probestr + 'dis_numberdensity_fast': 'n',
probestr + 'dis_temppara_fast': 'T_par',
probestr + 'dis_tempperp_fast': 'T_perp'}
df = util.cdf2df(cdf, 'Epoch', keys)
data.append(df)
return util.timefilter(data, starttime, endtime)
def processing_func(local_dir, local_fname):
cdf = util.load(local_fname, local_dir, '')
return util.cdf2df(cdf,
index_key='Epoch',
keys=keys,
badvalues=badvalues)
def threedp_pm(starttime, endtime):
"""
Import 'pm' wind data.
3 second time resolution solar wind proton and alpha particle moments from
the PESA LOW sensor, computed on-board the spacecraft
Parameters
----------
starttime : datetime
Interval start time.
endtime : datetime
Interval end time.
Returns
-------
data : DataFrame
"""
# Directory relative to main WIND data directory
relative_dir = path.Path('3dp') / '3dp_pm'
daylist = util._daysplitinterval(starttime, endtime)
data = []
for day in daylist:
date = day[0]
this_relative_dir = relative_dir / str(day[0].year)
# Absolute path to local directory for this data file
local_dir = wind_dir / this_relative_dir
filename = 'wi_pm_3dp_' +\
str(date.year) +\
str(date.month).zfill(2) +\
str(date.day).zfill(2) +\
'_v05.cdf'
hdfname = filename[:-4] + 'hdf'
hdfloc = local_dir / hdfname
if hdfloc.exists():
df = pd.read_hdf(hdfloc)
data.append(df)
continue
util._checkdir(local_dir)
remote_url = remote_wind_dir + str(this_relative_dir)
cdf = util.load(filename, local_dir, remote_url, guessversion=True)
if cdf is None:
print('File {}/{} not available\n'.format(remote_url, filename))
continue
keys = {
'A_DENS': 'n_a',
'A_TEMP': 'T_a',
'A_VELS': ['va_x', 'va_y', 'va_z'],
'P_DENS': 'n_p',
'P_TEMP': 'T_p',
'P_VELS': ['vp_x', 'vp_y', 'vp_z'],
'Epoch': 'Time'
}
df = util.cdf2df(cdf, index_key='Epoch', keys=keys)
if use_hdf:
df.to_hdf(hdfloc, 'pm', mode='w')
data.append(df)
return util.timefilter(data, starttime, endtime)
def processing_func(f):
thisdata = util.cdf2df(f, 'Epoch')
thisdata.index.name = 'Time'
return thisdata
def load_local_file(self, interval):
local_path = self.local_path(interval)
cdf = util._load_cdf(local_path)
zvars = cdf.cdf_info()['zVariables']
index_key = 'EPOCH' if 'EPOCH' in zvars else 'Epoch'
return util.cdf2df(cdf, index_key=index_key)
def processing_func(cdf):
probestr = 'mms' + probe + '_'
df = util.cdf2df(cdf, 'Epoch')
return df