def test_inequality_with_higher_order_meta3(self):
self.meta = pysat.Meta()
meta = pysat.Meta()
meta['dm'] = {'units': 'hey', 'long_name': 'boo'}
meta['rpa'] = {'units': 'crazy', 'long_name': 'boo_whoo'}
self.meta['higher'] = meta
self.meta['lower'] = {'units': 'yoyooy'}
meta2 = pysat.Meta()
meta2['dm'] = {'units': 'hey', 'long_name': 'boo'}
meta2['rpa'] = {'units': 'crazy', 'long_name': 'boo_whoo'}
meta3 = pysat.Meta()
meta3['higher'] = meta2
# print ('meta3', meta3)
# print ('self.meta', self.meta)
# for key in meta3.keys():
# print ('meta3 key', meta3[key])
# print ('self key', self.meta[key])
# for key in meta3.keys_nD():
# for key2 in meta3[key].keys():
#
# print (meta3[key].children[key2])
# print (self.meta[key].children[key2])
assert not (meta3 == self.meta)
assert not (self.meta == meta3)
def test_change_Units_and_Name_case_w_ho_wrong_case(self):
self.meta = pysat.Meta(units_label='units', name_label='long_Name')
meta2 = pysat.Meta(units_label='units', name_label='long_Name')
meta2['new21'] = {'units': 'hey2', 'long_name': 'boo2'}
self.meta['new'] = {'units': 'hey', 'long_name': 'boo'}
self.meta['new2'] = meta2
self.meta.units_label = 'Units'
self.meta.name_label = 'Long_Name'
assert ((self.meta['new'].units == 'hey') &
(self.meta['new'].long_name == 'boo') &
(self.meta['new2'].children['new21'].units == 'hey2') &
(self.meta['new2'].children['new21'].long_name == 'boo2'))
def test_basic_concat_w_ho_collision_strict(self):
self.meta['new1'] = {'units': 'hey1', 'long_name': 'crew'}
self.meta['new2'] = {'units': 'hey', 'long_name': 'boo',
'description': 'boohoo'}
meta2 = pysat.Meta()
meta2['new31'] = {'units': 'hey3', 'long_name': 'crew_brew'}
self.meta['new3'] = meta2
meta3 = pysat.Meta()
meta3['new31'] = {'units': 'hey4', 'long_name': 'crew_brew',
'bob_level': 'max'}
meta2['new3'] = meta3
self.meta = self.meta.concat(meta2, strict=True)
def test_create_new_metadata_from_old(self):
meta = pysat.Meta()
meta['dm'] = {'units': 'hey', 'long_name': 'boo'}
meta['rpa'] = {'units': 'crazy', 'long_name': 'boo_whoo'}
self.meta[['higher', 'lower', 'lower2']] = {
'meta': [meta, None, meta],
'units': [None, 'boo', None],
'long_name': [None, 'boohoo', None]
}
meta2 = pysat.Meta(metadata=self.meta.data)
check1 = np.all(meta2['lower'] == self.meta['lower'])
assert check1
def test_equality_with_higher_order_meta(self):
self.meta = pysat.Meta()
meta = pysat.Meta()
meta['dm'] = {'units': 'hey', 'long_name': 'boo'}
meta['rpa'] = {'units': 'crazy', 'long_name': 'boo_whoo'}
self.meta['higher'] = meta
meta2 = pysat.Meta()
meta2['dm'] = {'units': 'hey', 'long_name': 'boo'}
meta2['rpa'] = {'units': 'crazy', 'long_name': 'boo_whoo'}
meta3 = pysat.Meta()
meta3['higher'] = meta2
assert meta3 == self.meta
assert self.meta == meta3
def test_inequality_with_higher_order_meta(self):
self.meta = pysat.Meta()
meta = pysat.Meta()
meta['dm'] = {'units': 'hey', 'long_name': 'boo', 'radn': 'raiden'}
meta['rpa'] = {'units': 'crazy', 'long_name': 'boo_whoo'}
self.meta['higher'] = meta
meta2 = pysat.Meta()
meta2['dm'] = {'units': 'hey', 'long_name': 'boo'}
meta2['rpa'] = {'units': 'crazy', 'long_name': 'boo_whoo'}
meta3 = pysat.Meta()
meta3['higher'] = meta2
assert not (meta3 == self.meta)
assert not (self.meta == meta3)
def load_orig(fnames, tag=None):
import pydarn
if len(fnames) <= 0:
return pysat.DataFrame(None), pysat.Meta(None)
elif len(fnames) == 1:
b = pydarn.sdio.sdDataOpen(pysat.datetime(1980, 1, 1),
src='local',
eTime=pysat.datetime(2050, 1, 1),
fileName=fnames[0])
data_list = pydarn.sdio.sdDataReadAll(b)
sys.stdout.flush()
in_dict = []
for info in data_list:
arr = np.arange(len(info.stid))
drift_frame = pds.DataFrame(
info.vector.__dict__,
#index=[info.vector.mlon, info.vector.mlat])
index=info.vector.index)
drift_frame.index.name = 'index'
drift_frame.sort(inplace=True)
#drift_frame.index.names=['mlon', 'mlat']
for i in arr:
nvec = info.nvec[i]
in_frame = drift_frame.iloc[0:nvec]
drift_frame = drift_frame.iloc[nvec:]
in_dict.append({
'stid': info.stid[i],
'channel': info.channel[i],
'noisemean': info.noisemean[i],
'noisesd': info.noisesd[i],
'gsct': info.gsct[i],
'nvec': info.nvec[i],
'pmax': info.pmax[i],
'vector': in_frame,
'start_time': info.sTime,
'end_time': info.eTime,
'vemax': info.vemax[i],
'vemin': info.vemin[i],
'pmin': info.pmin[i],
'programid': info.programid[i],
'wmax': info.wmax[i],
'wmin': info.wmin[i],
'freq': info.freq[i]
})
output = pds.DataFrame(in_dict)
output.index = output.start_time
output.drop('start_time', axis=1, inplace=True)
return output, pysat.Meta()
else:
raise ValueError('Only one filename currently supported.')
def test_basic_concat_w_ho(self):
self.meta['new1'] = {'units': 'hey1', 'long_name': 'crew'}
self.meta['new2'] = {'units': 'hey', 'long_name': 'boo',
'description': 'boohoo'}
meta2 = pysat.Meta()
meta2['new3'] = {'units': 'hey3', 'long_name': 'crew_brew'}
meta3 = pysat.Meta()
meta3['new41'] = {'units': 'hey4', 'long_name': 'crew_brew',
'bob_level': 'max'}
meta2['new4'] = meta3
self.meta = self.meta.concat(meta2)
assert (self.meta['new3'].units == 'hey3')
assert (self.meta['new4'].children['new41'].units == 'hey4')
def test_basic_concat_w_ho_collision_not_strict(self):
self.meta['new1'] = {'units': 'hey1', 'long_name': 'crew'}
self.meta['new2'] = {'units': 'hey', 'long_name': 'boo',
'description': 'boohoo'}
meta2 = pysat.Meta()
meta2['new3'] = {'units': 'hey3', 'long_name': 'crew_brew'}
meta3 = pysat.Meta()
meta3['new41'] = {'units': 'hey4', 'long_name': 'crew_brew',
'bob_level': 'max'}
meta2['new3'] = meta3
self.meta = self.meta.concat(meta2, strict=False)
assert self.meta['new3'].children['new41'].units == 'hey4'
assert self.meta['new3'].children['new41'].bob_level == 'max'
assert self.meta['new2'].units == 'hey'
def load(cosmicFiles, tag=None, sat_id=None, altitude_bin=None):
"""
cosmic data load routine, called by pysat
"""
num = len(cosmicFiles)
# make sure there are files to read
if num != 0:
# call separate load_files routine, segemented for possible
# multiprocessor load, not included and only benefits about 20%
output = pysat.DataFrame(
load_files(cosmicFiles,
tag=tag,
sat_id=sat_id,
altitude_bin=altitude_bin))
output.index = pysat.utils.create_datetime_index(
year=output.year,
month=output.month,
day=output.day,
uts=output.hour * 3600. + output.minute * 60. + output.second)
# make sure UTS strictly increasing
output.sort_index(inplace=True)
# use the first available file to pick out meta information
profile_meta = pysat.Meta()
meta = pysat.Meta()
ind = 0
repeat = True
while repeat:
try:
data = netcdf_file(cosmicFiles[ind], mode='r', mmap=False)
keys = data.variables.keys()
for key in keys:
profile_meta[key] = {
'units': data.variables[key].units,
'long_name': data.variables[key].long_name
}
# ncattrsList = data.ncattrs()
ncattrsList = data._attributes.keys()
for d in ncattrsList:
meta[d] = {'units': '', 'long_name': d}
repeat = False
except RuntimeError:
# file was empty, try the next one by incrementing ind
ind += 1
meta['profiles'] = profile_meta
return output, meta
else:
# no data
return pysat.DataFrame(None), pysat.Meta()
def setup(self):
"""Runs before every method to create a clean testing setup"""
# Load a test instrument with 3hr ap data
self.testInst = pysat.Instrument()
self.testInst.data = pds.DataFrame(
{'3hr_ap': [0, 2, 3, 4, 5, 6, 7, 9, 12, 15]},
index=[
pysat.datetime(2009, 1, 1) + pds.DateOffset(hours=3 * i)
for i in range(10)
])
self.testInst.meta = pysat.Meta()
self.meta_dict = {
self.testInst.meta.units_label: '',
self.testInst.meta.name_label: 'ap',
self.testInst.meta.desc_label:
"3-hour ap (equivalent range) index",
self.testInst.meta.plot_label: "ap",
self.testInst.meta.axis_label: "ap",
self.testInst.meta.scale_label: 'linear',
self.testInst.meta.min_label: 0,
self.testInst.meta.max_label: 400,
self.testInst.meta.fill_label: np.nan,
self.testInst.meta.notes_label: 'test ap'
}
self.testInst.meta.__setitem__('3hr_ap', self.meta_dict)
def test_inst_assign_from_meta_w_ho(self):
self.testInst.load(2009, 1)
frame = pds.DataFrame(
{
'dummy_frame1': np.arange(10),
'dummy_frame2': np.arange(10)
},
columns=['dummy_frame1', 'dummy_frame2'])
meta = pysat.Meta()
meta['dummy_frame1'] = {'units': 'A'}
meta['dummy_frame2'] = {'desc': 'nothing'}
self.testInst['help'] = {
'data': [frame] * len(self.testInst.data.index),
'units': 'V',
'long_name': 'The Doors',
'meta': meta
}
self.testInst['help2'] = self.testInst['help']
self.testInst.meta['help2'] = self.testInst.meta['help']
assert self.testInst.meta['help'].children['dummy_frame1',
'units'] == 'A'
assert self.testInst.meta['help2', 'long_name'] == 'The Doors'
assert 'dummy_frame1' in self.testInst.meta.ho_data['help2']
assert 'dummy_frame2' in self.testInst.meta.ho_data['help2']
assert 'dummy_frame1' in self.testInst.meta['help2']['children']
assert 'dummy_frame2' in self.testInst.meta['help2']['children']
assert self.testInst.meta['help2']['children'].has_attr('units')
assert self.testInst.meta['help2']['children'].has_attr('desc')
assert self.testInst.meta['help2']['children']['dummy_frame1',
'desc'] == ''
assert self.testInst.meta['help2']['children']['dummy_frame2',
'desc'] == 'nothing'
assert 'children' not in self.testInst.meta.data.columns
def test_basic_pops(self):
self.meta['new1'] = {'units': 'hey1', 'long_name': 'crew',
'value_min': 0, 'value_max': 1}
self.meta['new2'] = {'units': 'hey', 'long_name': 'boo',
'description': 'boohoo', 'fill': 1,
'value_min': 0, 'value_max': 1}
# create then assign higher order meta data
meta2 = pysat.Meta(name_label='long_name')
meta2['new31'] = {'units': 'hey3', 'long_name': 'crew_brew', 'fill': 1,
'value_min': 0, 'value_max': 1}
self.meta['new3'] = meta2
aa = self.meta.pop('new3')
assert np.all(aa['children'] == meta2)
# ensure lower metadata created when ho data assigned
assert aa['units'] == ''
assert aa['long_name'] == 'new3'
m1 = self.meta['new2']
m2 = self.meta.pop('new2')
assert m1['children'] is None
assert m2['children'] is None
for key in m1.index:
if key not in ['children']:
assert m1[key] == m2[key]
# make sure both have the same indexes
assert np.all(m1.index == m2.index)
def test_assign_Units_no_units(self):
self.meta = pysat.Meta(units_label='Units', name_label='Long_Name')
self.meta['new'] = {'Units': 'hey', 'Long_Name': 'boo'}
self.meta['new2'] = {'Units': 'hey2', 'Long_Name': 'boo2'}
# print ( self.meta['new'])
# print (self.meta['new2', 'units'])
self.meta['new'].units
def test_repeated_set_Units_wrong_case(self):
self.meta = pysat.Meta(units_label='Units', name_label='Long_Name')
for i in np.arange(10):
self.meta['new'] = {'units': 'hey%d' % i, 'long_name': 'boo%d' % i}
self.meta['new_%d' % i] = {
'units': 'hey%d' % i,
'long_name': 'boo%d' % i
}
for i in np.arange(10):
self.meta['new_5'] = {
'units': 'hey%d' % i,
'long_name': 'boo%d' % i
}
self.meta['new_%d' % i] = {
'units': 'heyhey%d' % i,
'long_name': 'booboo%d' % i
}
# print (self.meta['new'])
assert self.meta['new'].Units == 'hey9'
assert self.meta['new'].Long_Name == 'boo9'
assert self.meta['new_9'].Units == 'heyhey9'
assert self.meta['new_9'].Long_Name == 'booboo9'
# print (self.meta['new_500'])
assert self.meta['new_5'].Units == 'hey9'
assert self.meta['new_5'].Long_Name == 'boo9'
def setup(self):
"""Runs before every method to create a clean testing setup."""
self.meta = pysat.Meta()
self.testInst = pysat.Instrument('pysat',
'testing',
tag='',
clean_level='clean')
def test_inst_ho_data_assign_meta_different_labels(self):
self.testInst.load(2009, 1)
frame = pds.DataFrame(
{
'dummy_frame1': np.arange(10),
'dummy_frame2': np.arange(10)
},
columns=['dummy_frame1', 'dummy_frame2'])
meta = pysat.Meta(units_label='blah', desc_label='whoknew')
meta['dummy_frame1'] = {'blah': 'A'}
meta['dummy_frame2'] = {'whoknew': 'nothing'}
self.testInst['help'] = {
'data': [frame] * len(self.testInst.data.index),
'units': 'V',
'long_name': 'The Doors',
'meta': meta
}
assert self.testInst.meta['help', 'long_name'] == 'The Doors'
assert 'dummy_frame1' in self.testInst.meta.ho_data['help']
assert 'dummy_frame2' in self.testInst.meta.ho_data['help']
assert 'dummy_frame1' in self.testInst.meta['help']['children']
assert 'dummy_frame2' in self.testInst.meta['help']['children']
assert self.testInst.meta['help']['children'].has_attr('units')
assert self.testInst.meta['help']['children'].has_attr('desc')
assert self.testInst.meta['help']['children']['dummy_frame1',
'units'] == 'A'
assert self.testInst.meta['help']['children']['dummy_frame1',
'desc'] == ''
assert self.testInst.meta['help']['children']['dummy_frame2',
'desc'] == 'nothing'
def test_concat_w_name_collision_strict(self):
self.meta['new1'] = {'units': 'hey1', 'long_name': 'crew'}
self.meta['new2'] = {'units': 'hey', 'long_name': 'boo',
'description': 'boohoo'}
meta2 = pysat.Meta()
meta2['new2'] = {'units': 'hey2', 'long_name': 'crew_brew'}
meta2['new3'] = {'units': 'hey3', 'long_name': 'crew_brew'}
self.meta = self.meta.concat(meta2, strict=True)
def test_get_Units_wrong_case(self):
self.meta = pysat.Meta(units_label='Units', name_label='Long_Name')
self.meta['new'] = {'Units': 'hey', 'Long_Name': 'boo'}
self.meta['new2'] = {'Units': 'hey2', 'Long_Name': 'boo2'}
assert ((self.meta['new', 'units'] == 'hey') &
(self.meta['new', 'long_name'] == 'boo') &
(self.meta['new2', 'units'] == 'hey2') &
(self.meta['new2', 'long_name'] == 'boo2'))
def test_basic_concat(self):
self.meta['new1'] = {'units': 'hey1', 'long_name': 'crew'}
self.meta['new2'] = {'units': 'hey', 'long_name': 'boo',
'description': 'boohoo'}
meta2 = pysat.Meta()
meta2['new3'] = {'units': 'hey3', 'long_name': 'crew_brew'}
self.meta = self.meta.concat(meta2)
assert (self.meta['new3'].units == 'hey3')
def test_set_Units_wrong_case(self):
self.meta = pysat.Meta(units_label='Units', name_label='Long_Name')
self.meta['new'] = {'units': 'hey', 'long_name': 'boo'}
self.meta['new2'] = {'units': 'hey2', 'long_name': 'boo2'}
assert self.meta['new'].Units == 'hey'
assert self.meta['new'].Long_Name == 'boo'
assert self.meta['new2'].Units == 'hey2'
assert self.meta['new2'].Long_Name == 'boo2'
def test_assign_Units(self):
self.meta = pysat.Meta(units_label='Units', name_label='Long_Name')
self.meta['new'] = {'Units': 'hey', 'Long_Name': 'boo'}
self.meta['new2'] = {'Units': 'hey2', 'Long_Name': 'boo2'}
assert ((self.meta['new'].Units == 'hey') &
(self.meta['new'].Long_Name == 'boo') &
(self.meta['new2'].Units == 'hey2') &
(self.meta['new2'].Long_Name == 'boo2'))
def test_contains_case_insensitive_w_ho(self):
self.meta['new'] = {'units': 'hey', 'long_name': 'boo'}
meta2 = pysat.Meta()
meta2['new21'] = {'units': 'hey2', 'long_name': 'boo2'}
self.meta['new2'] = meta2
assert ('new2' in self.meta)
assert ('NEW2' in self.meta)
assert not ('new21' in self.meta)
assert not ('NEW21' in self.meta)
def load(fnames, tag=None, inst_id=None):
"""Load CHAIN GPS Files
Parameters
----------
fnames : (list or array-like)
series of filenames to be loaded
tag : (string or NoneType)
Denotes type of file to load.
(default=None)
inst_id : (string or NoneType)
Specifies the satellite ID for a constellation. Not used.
(default=None)
"""
if not fnames:
print('fnames are go')
warnings.warn('You no got no fname, you no get no data')
return pysat.DataFrame(None), pysat.Meta(None)
elif len(fnames) == 1:
print('fnames are greater than 1')
meta = pysat.Meta()
signal_meta = pysat.Meta()
# load the rinex
data = gr.load(fnames[0])
# get the metadata from the xarray.Dataset
xr_attrs = data.attrs
# format the metadata
# for at in xr_attrs:
# print(type(at))
# meta[at] = {'units': '', 'long_name': at}
# keys = data.data_vars.keys()
# for key in keys:
# # find the xarray equivalents of units and long_name
# print(key)
# signal_meta[key] = {meta.labels.units: var_units[key][0],
# meta.labels.name: var_units[key][1]}
# # format the data
# meta['signals'] = signal_meta
# output = data.variables
return data, meta
else:
raise ValueError('Only one filename currently supported')
print('no idea wahts goin on')
def test_assign_higher_order_meta_from_dict_w_multiple(self):
meta = pysat.Meta()
meta['dm'] = {'units': 'hey', 'long_name': 'boo'}
meta['rpa'] = {'units': 'crazy', 'long_name': 'boo_whoo'}
self.meta[['higher', 'lower']] = {'meta': [meta, None],
'units': [None, 'boo'],
'long_name': [None, 'boohoo']}
assert self.meta['lower'].units == 'boo'
assert self.meta['lower'].long_name == 'boohoo'
assert self.meta['higher'].children == meta
def test_change_Units_and_Name_case(self):
self.meta = pysat.Meta(units_label='units', name_label='long_name')
self.meta['new'] = {'units': 'hey', 'long_name': 'boo'}
self.meta['new2'] = {'units': 'hey2', 'long_name': 'boo2'}
self.meta.units_label = 'Units'
self.meta.name_label = 'Long_Name'
assert ((self.meta['new'].Units == 'hey') &
(self.meta['new'].Long_Name == 'boo') &
(self.meta['new2'].Units == 'hey2') &
(self.meta['new2'].Long_Name == 'boo2'))
def test_has_attr_name_case_preservation_w_higher_order(self):
self.meta['new'] = {'units': 'hey', 'long_name': 'boo'}
meta2 = pysat.Meta()
meta2['NEW21'] = {'units': 'hey2', 'long_name': 'boo2',
'YoYoYO': 'yolo'}
self.meta['NEW2'] = meta2
assert not (self.meta.has_attr('YoYoYo'))
assert not (self.meta.has_attr('yoyoyo'))
assert not (self.meta.has_attr('YoYoYyo'))
def setup(self):
"""Runs before every method to create a clean testing setup"""
# Load a test instrument
self.testInst = pysat.Instrument()
self.testInst.data = \
pds.DataFrame({'Kp': np.arange(0, 4, 1.0 / 3.0),
'ap_nan': np.full(shape=12, fill_value=np.nan),
'ap_inf': np.full(shape=12, fill_value=np.inf)},
index=[dt.datetime(2009, 1, 1)
+ pds.DateOffset(hours=3 * i)
for i in range(12)])
self.testInst.meta = pysat.Meta()
self.testInst.meta['Kp'] = {self.testInst.meta.labels.fill_val: np.nan}
self.testInst.meta['ap_nan'] = {self.testInst.meta.labels.fill_val:
np.nan}
self.testInst.meta['ap_inf'] = {self.testInst.meta.labels.fill_val:
np.inf}
# Load a test Metadata
self.testMeta = pysat.Meta()
def test_multiple_meta_ho_data_retrieval(self):
meta = pysat.Meta()
meta['dm'] = {'units': 'hey', 'long_name': 'boo'}
meta['rpa'] = {'units': 'crazy', 'long_name': 'boo_whoo'}
self.meta[['higher', 'lower']] = {'meta': [meta, None],
'units': [None, 'boo'],
'long_name': [None, 'boohoo']}
assert self.meta['lower'].units == 'boo'
assert self.meta['lower'].long_name == 'boohoo'
assert self.meta['higher'].children == meta
self.meta['higher',('axis','scale')]
def test_merge_meta(self):
self.meta['new'] = {'units': 'hey', 'long_name': 'boo'}
meta2 = pysat.Meta()
meta2['NEW21'] = {'units': 'hey2', 'long_name': 'boo2',
'YoYoYO': 'yolo'}
self.meta.merge(meta2)
assert (self.meta['new'].units == 'hey')
assert (self.meta['new'].long_name == 'boo')
assert (self.meta['NEW21'].units == 'hey2')
assert (self.meta['NEW21'].long_name == 'boo2')
assert (self.meta['NEW21'].YoYoYO == 'yolo')
