def test_long_name_kp_metadata(self):
"""Test Kp metadata initialization with a long name"""
dkey = 'high_lat_Kp'
kp_ap.initialize_kp_metadata(self.testInst.meta, dkey)
assert self.testInst.meta[dkey][self.testInst.meta.labels.name] == dkey
assert(self.testInst.meta[dkey][self.testInst.meta.labels.desc]
== 'Planetary K-index')
del dkey
def test_uninit_kp_metadata(self):
"""Test Kp metadata initialization with uninitialized Metadata"""
kp_ap.initialize_kp_metadata(self.testMeta, 'Kp')
assert self.testMeta['Kp'][self.testMeta.labels.units] == ''
assert self.testMeta['Kp'][self.testMeta.labels.name] == 'Kp'
assert(self.testMeta['Kp'][self.testMeta.labels.desc]
== 'Planetary K-index')
assert self.testMeta['Kp'][self.testMeta.labels.min_val] == 0
assert self.testMeta['Kp'][self.testMeta.labels.max_val] == 9
assert self.testMeta['Kp'][self.testMeta.labels.fill_val] == -1
def test_initialize_kp_metadata(self):
"""Test default Kp metadata initialization"""
kp_ap.initialize_kp_metadata(self.testInst.meta, 'Kp')
assert self.testInst.meta['Kp'][self.testInst.meta.labels.units] == ''
assert self.testInst.meta['Kp'][self.testInst.meta.labels.name] == 'Kp'
assert self.testInst.meta['Kp'][
self.testInst.meta.labels.desc] == 'Planetary K-index'
assert self.testInst.meta['Kp'][self.testInst.meta.labels.min_val] == 0
assert self.testInst.meta['Kp'][self.testInst.meta.labels.max_val] == 9
assert self.testInst.meta['Kp'][
self.testInst.meta.labels.fill_val] == -1
def load(fnames, tag=None, inst_id=None):
"""Load Kp index files
Parameters
----------
fnames : pandas.Series
Series of filenames
tag : str or NoneType
tag or None (default=None)
inst_id : str or NoneType
satellite id or None (default=None)
Returns
-------
data : pandas.DataFrame
Object containing satellite data
meta : pysat.Meta
Object containing metadata such as column names and units
Note
----
Called by pysat. Not intended for direct use by user.
"""
meta = pysat.Meta()
if tag == '':
# Kp data stored monthly, need to return data daily. The daily date is
# attached to filename. Parse off the last date, load month of data,
# and downselect to the desired day
data = pds.DataFrame()
# Set up fixed width format for these files
colspec = [(0, 2), (2, 4), (4, 6), (7, 10), (10, 13), (13, 16),
(16, 19), (19, 23), (23, 26), (26, 29), (29, 32), (32, 50)]
all_data = []
for filename in fnames:
# The daily date is attached to the filename. Parse off the last
# date, load month of data, downselect to the desired day
fname = filename[0:-11]
date = dt.datetime.strptime(filename[-10:], '%Y-%m-%d')
temp = pds.read_fwf(fname,
colspecs=colspec,
skipfooter=4,
header=None,
parse_dates=[[0, 1, 2]],
date_parser=parse_date,
index_col='0_1_2')
idx, = np.where((temp.index >= date)
& (temp.index < date + pds.DateOffset(days=1)))
temp = temp.iloc[idx, :]
all_data.append(temp)
# Combine data together
data = pds.concat(all_data, axis=0, sort=True)
# Drop last column as it has data that belongs to a different inst
data = data.iloc[:, 0:-1]
# Each column increments UT by three hours. Produce a single data
# series that has Kp value monotonically increasing in time with
# appropriate datetime indices
data_series = pds.Series(dtype='float64')
for i in np.arange(8):
tind = data.index + pds.DateOffset(hours=int(3 * i))
temp = pds.Series(data.iloc[:, i].values, index=tind)
data_series = data_series.append(temp)
data_series = data_series.sort_index()
data_series.index.name = 'time'
# Kp comes in non-user friendly values like 2-, 2o, and 2+. Relate
# these to 1.667, 2.0, 2.333 for processing and user friendliness
first = np.array([float(x[0]) for x in data_series])
flag = np.array([x[1] for x in data_series])
ind, = np.where(flag == '+')
first[ind] += 1.0 / 3.0
ind, = np.where(flag == '-')
first[ind] -= 1.0 / 3.0
result = pds.DataFrame(first, columns=['Kp'], index=data_series.index)
fill_val = np.nan
elif tag == 'forecast':
# Load forecast data
all_data = []
for fname in fnames:
result = pds.read_csv(fname, index_col=0, parse_dates=True)
all_data.append(result)
result = pds.concat(all_data)
fill_val = -1
elif tag == 'recent':
# Load recent Kp data
all_data = []
for fname in fnames:
result = pds.read_csv(fname, index_col=0, parse_dates=True)
all_data.append(result)
result = pds.concat(all_data)
fill_val = -1
# Initalize the meta data
for kk in result.keys():
kp_ap.initialize_kp_metadata(meta, kk, fill_val)
return result, meta
def test_fill_kp_metadata(self):
"""Test Kp metadata initialization with user-specified fill value"""
kp_ap.initialize_kp_metadata(self.testInst.meta, 'Kp', fill_val=666)
assert self.testInst.meta['Kp'][
self.testInst.meta.labels.fill_val] == 666
def load(fnames, tag=None, inst_id=None):
"""Load Kp index files
Parameters
----------
fnames : pandas.Series
Series of filenames
tag : str or NoneType
tag or None (default=None)
inst_id : str or NoneType
satellite id or None (default=None)
Returns
-------
data : pandas.DataFrame
Object containing satellite data
meta : pysat.Meta
Object containing metadata such as column names and units
Note
----
Called by pysat. Not intended for direct use by user.
"""
meta = pysat.Meta()
if tag == '':
# Kp data stored monthly, need to return data daily. The daily date is
# attached to filename. Parse off the last date, load month of data,
# and downselect to the desired day
data = pds.DataFrame()
# Set up fixed width format for these files, only selecting the date
# and daily 3-hour Kp values
date_slice = slice(0, 6)
kp_slice = [
slice(7, 10),
slice(10, 13),
slice(13, 16),
slice(16, 19),
slice(19, 23),
slice(23, 26),
slice(26, 29),
slice(29, 32)
]
# These are monthly files, if a date range is desired, test here.
# Does not assume an ordered list, but the date range must be continous
# within a given month.
unique_fnames = dict()
for filename in fnames:
fname = filename[0:-11]
fdate = dt.datetime.strptime(filename[-10:], '%Y-%m-%d')
if fname not in unique_fnames.keys():
unique_fnames[fname] = [fdate]
else:
unique_fnames[fname].append(fdate)
# Load all of the desired filenames
all_data = []
for fname in unique_fnames.keys():
# The daily date is attached to the filename. Parse off the last
# date, load month of data, downselect to the desired day
fdate = min(unique_fnames[fname])
with open(fname, 'r') as fin:
temp = fin.readlines()
if len(temp) == 0:
logger.warn('Empty file: {:}'.format(fname))
continue
# This file has a different format if it is historic or a file that
# is being actively updated. In either case, this line will
# remove the appropriate number of summmary lines.
ilast = -1 if temp[-1].find('Mean') > 0 else -4
temp = np.array(temp[:ilast])
# Re-format the time data
temp_index = np.array([
dt.datetime.strptime(tline[date_slice], '%y%m%d')
for tline in temp
])
idx, = np.where((temp_index >= fdate)
& (temp_index < max(unique_fnames[fname]) +
dt.timedelta(days=1)))
temp_data = list()
for tline in temp[idx]:
temp_data.append(list())
for col in kp_slice:
temp_data[-1].append(tline[col].strip())
# Select the desired times and add to data list
all_data.append(pds.DataFrame(temp_data, index=temp_index[idx]))
# Combine data together
data = pds.concat(all_data, axis=0, sort=True)
# Each column increments UT by three hours. Produce a single data
# series that has Kp value monotonically increasing in time with
# appropriate datetime indices
data_series = pds.Series(dtype='float64')
for i in np.arange(8):
tind = data.index + pds.DateOffset(hours=int(3 * i))
temp = pds.Series(data.iloc[:, i].values, index=tind)
data_series = data_series.append(temp)
data_series = data_series.sort_index()
data_series.index.name = 'time'
# Kp comes in non-user friendly values like 2-, 2o, and 2+. Relate
# these to 1.667, 2.0, 2.333 for processing and user friendliness
first = np.array([float(str_val[0]) for str_val in data_series])
flag = np.array([str_val[1] for str_val in data_series])
ind, = np.where(flag == '+')
first[ind] += 1.0 / 3.0
ind, = np.where(flag == '-')
first[ind] -= 1.0 / 3.0
result = pds.DataFrame(first, columns=['Kp'], index=data_series.index)
fill_val = np.nan
elif tag == 'forecast':
# Load forecast data
all_data = []
for fname in fnames:
result = pds.read_csv(fname, index_col=0, parse_dates=True)
all_data.append(result)
result = pds.concat(all_data)
fill_val = -1
elif tag == 'recent':
# Load recent Kp data
all_data = []
for fname in fnames:
result = pds.read_csv(fname, index_col=0, parse_dates=True)
all_data.append(result)
result = pds.concat(all_data)
fill_val = -1
# Initalize the meta data
for kk in result.keys():
kp_ap.initialize_kp_metadata(meta, kk, fill_val)
return result, meta