def load(fnames,
tag=None,
sat_id=None,
sim_multi_file_right=False,
sim_multi_file_left=False,
root_date=None,
file_date_range=None,
malformed_index=False,
mangle_file_dates=False):
""" Loads the test files
Parameters
----------
fnames : list
List of filenames
tag : str or NoneType
Instrument tag (accepts '')
sat_id : str or NoneType
Instrument satellite ID (accepts '' or a number (i.e., '10'), which
specifies the number of data points to include in the test instrument)
sim_multi_file_right : boolean
Adjusts date range to be 12 hours in the future or twelve hours beyond
root_date (default=False)
sim_multi_file_left : boolean
Adjusts date range to be 12 hours in the past or twelve hours before
root_date (default=False)
root_date : NoneType
Optional central date, uses _test_dates if not specified.
(default=None)
file_date_range : pds.date_range or NoneType
Range of dates for files or None, if this optional arguement is not
used. Shift actually performed by the init function.
(default=None)
malformed_index : boolean
If True, time index will be non-unique and non-monotonic.
mangle_file_dates : bool
If True, the loaded file list time index is shifted by 5-minutes.
This shift is actually performed by the init function.
Returns
-------
data : (pds.DataFrame)
Testing data
meta : (pysat.Meta)
Metadataxs
"""
# create an artifical satellite data set
iperiod = mm_test.define_period()
drange = mm_test.define_range()
uts, index, date = mm_test.generate_times(fnames, sat_id, freq='1S')
# Specify the date tag locally and determine the desired date range
pds_offset = pds.DateOffset(hours=12)
if sim_multi_file_right:
root_date = root_date or _test_dates[''][''] + pds_offset
elif sim_multi_file_left:
root_date = root_date or _test_dates[''][''] - pds_offset
else:
root_date = root_date or _test_dates['']['']
data = pysat.DataFrame(uts, columns=['uts'])
# need to create simple orbits here. Have start of first orbit default
# to 1 Jan 2009, 00:00 UT. 14.84 orbits per day
time_delta = date - root_date
data['mlt'] = mm_test.generate_fake_data(time_delta.total_seconds(),
uts,
period=iperiod['lt'],
data_range=drange['lt'])
# do slt, 20 second offset from mlt
data['slt'] = mm_test.generate_fake_data(time_delta.total_seconds() + 20,
uts,
period=iperiod['lt'],
data_range=drange['lt'])
# create a fake longitude, resets every 6240 seconds
# sat moves at 360/5820 deg/s, Earth rotates at 360/86400, takes extra time
# to go around full longitude
data['longitude'] = mm_test.generate_fake_data(time_delta.total_seconds(),
uts,
period=iperiod['lon'],
data_range=drange['lon'])
# create latitude area for testing polar orbits
angle = mm_test.generate_fake_data(time_delta.total_seconds(),
uts,
period=iperiod['angle'],
data_range=drange['angle'])
data['latitude'] = 90.0 * np.cos(angle)
# fake orbit number
fake_delta = date - (_test_dates[''][''] - pds.DateOffset(years=1))
data['orbit_num'] = mm_test.generate_fake_data(fake_delta.total_seconds(),
uts,
period=iperiod['lt'],
cyclic=False)
# create some fake data to support testing of averaging routines
mlt_int = data['mlt'].astype(int)
long_int = (data['longitude'] / 15.0).astype(int)
if tag == 'ascend':
data['dummy1'] = [i for i in range(len(data['mlt']))]
elif tag == 'descend':
data['dummy1'] = [-i for i in range(len(data['mlt']))]
elif tag == 'plus10':
data['dummy1'] = [i + 10 for i in range(len(data['mlt']))]
elif tag == 'fives':
data['dummy1'] = [5 for i in range(len(data['mlt']))]
elif tag == 'mlt_offset':
data['dummy1'] = mlt_int + 5
else:
data['dummy1'] = mlt_int
data['dummy2'] = long_int
data['dummy3'] = mlt_int + long_int * 1000.0
data['dummy4'] = uts
data['string_dummy'] = ['test'] * len(data)
data['unicode_dummy'] = [u'test'] * len(data)
data['int8_dummy'] = np.ones(len(data), dtype=np.int8)
data['int16_dummy'] = np.ones(len(data), dtype=np.int16)
data['int32_dummy'] = np.ones(len(data), dtype=np.int32)
data['int64_dummy'] = np.ones(len(data), dtype=np.int64)
if malformed_index:
index = index.tolist()
# nonmonotonic
index[0:3], index[3:6] = index[3:6], index[0:3]
# non unique
index[6:9] = [index[6]] * 3
data.index = index
data.index.name = 'Epoch'
return data, meta.copy()
def load(fnames,
tag=None,
inst_id=None,
malformed_index=False,
num_samples=None,
test_load_kwarg=None):
""" Loads the test files
Parameters
----------
fnames : list
List of filenames
tag : str or NoneType
Instrument tag (accepts '')
inst_id : str or NoneType
Instrument satellite ID (accepts '')
malformed_index : bool
If True, the time index will be non-unique and non-monotonic.
(default=False)
num_samples : int
Number of samples
test_load_kwarg : any or NoneType
Testing keyword (default=None)
Returns
-------
data : pds.DataFrame
Testing data
meta : pysat.Meta
Testing metadata
"""
# Support keyword testing
logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg))))
# create an artifical satellite data set
iperiod = mm_test.define_period()
drange = mm_test.define_range()
if num_samples is None:
# Default to 1 day at a frequency of 100S
num_samples = 864
# Using 100s frequency for compatibility with seasonal analysis unit tests
uts, index, dates = mm_test.generate_times(fnames,
num_samples,
freq='100S')
# seed DataFrame with UT array
data = pds.DataFrame(np.mod(uts, 86400.), columns=['uts'])
# need to create simple orbits here. Have start of first orbit
# at 2009,1, 0 UT. 14.84 orbits per day
# figure out how far in time from the root start
# use that info to create a signal that is continuous from that start
# going to presume there are 5820 seconds per orbit (97 minute period)
time_delta = dates[0] - dt.datetime(2009, 1, 1)
# mlt runs 0-24 each orbit.
data['mlt'] = mm_test.generate_fake_data(time_delta.total_seconds(),
uts,
period=iperiod['lt'],
data_range=drange['lt'])
# do slt, 20 second offset from mlt
data['slt'] = mm_test.generate_fake_data(time_delta.total_seconds() + 20,
uts,
period=iperiod['lt'],
data_range=drange['lt'])
# create a fake longitude, resets every 6240 seconds
# sat moves at 360/5820 deg/s, Earth rotates at 360/86400, takes extra time
# to go around full longitude
data['longitude'] = mm_test.generate_fake_data(time_delta.total_seconds(),
uts,
period=iperiod['lon'],
data_range=drange['lon'])
# create latitude signal for testing polar orbits
angle = mm_test.generate_fake_data(time_delta.total_seconds(),
uts,
period=iperiod['angle'],
data_range=drange['angle'])
data['latitude'] = 90.0 * np.cos(angle)
# create constant altitude at 400 km
alt0 = 400.0
data['altitude'] = alt0 * np.ones(data['latitude'].shape)
if malformed_index:
index = index.tolist()
# nonmonotonic
index[0:3], index[3:6] = index[3:6], index[0:3]
# non unique
index[6:9] = [index[6]] * 3
data.index = index
data.index.name = 'epoch'
# higher rate time signal (for scalar >= 2)
# this time signal used for 2D profiles associated with each time in main
# DataFrame
num_profiles = 50 if num_samples >= 50 else num_samples
end_date = dates[0] + dt.timedelta(seconds=2 * num_profiles - 1)
high_rate_template = pds.date_range(dates[0], end_date, freq='2S')
# create a few simulated profiles
# DataFrame at each time with mixed variables
profiles = []
# DataFrame at each time with numeric variables only
alt_profiles = []
# Serie at each time, numeric data only
series_profiles = []
# frame indexed by date times
frame = pds.DataFrame(
{
'density': data.loc[data.index[0:num_profiles],
'mlt'].values.copy(),
'dummy_str': ['test'] * num_profiles,
'dummy_ustr': [u'test'] * num_profiles
},
index=data.index[0:num_profiles],
columns=['density', 'dummy_str', 'dummy_ustr'])
# frame indexed by float
dd = np.arange(num_profiles) * 1.2
ff = np.arange(num_profiles) / num_profiles
ii = np.arange(num_profiles) * 0.5
frame_alt = pds.DataFrame({
'density': dd,
'fraction': ff
},
index=ii,
columns=['density', 'fraction'])
# series version of storage
series_alt = pds.Series(dd, index=ii, name='series_profiles')
for time in data.index:
frame.index = high_rate_template + (time - data.index[0])
profiles.append(frame)
alt_profiles.append(frame_alt)
series_profiles.append(series_alt)
# store multiple data types into main frame
data['profiles'] = pds.Series(profiles, index=data.index)
data['alt_profiles'] = pds.Series(alt_profiles, index=data.index)
data['series_profiles'] = pds.Series(series_profiles, index=data.index)
# create very limited metadata
meta = pysat.Meta()
meta['uts'] = {'units': 's', 'long_name': 'Universal Time'}
meta['mlt'] = {'units': 'hours', 'long_name': 'Magnetic Local Time'}
meta['slt'] = {'units': 'hours', 'long_name': 'Solar Local Time'}
meta['longitude'] = {'units': 'degrees', 'long_name': 'Longitude'}
meta['latitude'] = {'units': 'degrees', 'long_name': 'Latitude'}
meta['altitude'] = {'units': 'km', 'long_name': 'Altitude'}
series_profile_meta = pysat.Meta()
series_profile_meta['series_profiles'] = {'long_name': 'series'}
meta['series_profiles'] = {
'meta': series_profile_meta,
'long_name': 'series'
}
profile_meta = pysat.Meta()
profile_meta['density'] = {'long_name': 'profiles'}
profile_meta['dummy_str'] = {'long_name': 'profiles'}
profile_meta['dummy_ustr'] = {'long_name': 'profiles'}
meta['profiles'] = {'meta': profile_meta, 'long_name': 'profiles'}
alt_profile_meta = pysat.Meta()
alt_profile_meta['density'] = {'long_name': 'profiles'}
alt_profile_meta['fraction'] = {'long_name': 'profiles'}
meta['alt_profiles'] = {'meta': alt_profile_meta, 'long_name': 'profiles'}
return data, meta
def load(fnames,
tag=None,
inst_id=None,
malformed_index=False,
num_samples=None,
test_load_kwarg=None):
""" Loads the test files
Parameters
----------
fnames : list
List of filenames
tag : str or NoneType
Instrument tag (accepts '')
inst_id : str or NoneType
Instrument satellite ID (accepts '')
malformed_index : bool False
If True, the time index will be non-unique and non-monotonic.
num_samples : int
Number of samples
test_load_kwarg : any or NoneType
Testing keyword (default=None)
Returns
-------
data : xr.Dataset
Testing data
meta : pysat.Meta
Testing metadata
"""
# Support keyword testing
logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg))))
# create an artifical satellite data set
iperiod = mm_test.define_period()
drange = mm_test.define_range()
if num_samples is None:
# Default to 1 day at a frequency of 100S
num_samples = 864
# Using 100s frequency for compatibility with seasonal analysis unit tests
uts, index, dates = mm_test.generate_times(fnames,
num_samples,
freq='100S')
if malformed_index:
index = index.tolist()
# nonmonotonic
index[0:3], index[3:6] = index[3:6], index[0:3]
# non unique
index[6:9] = [index[6]] * 3
data = xr.Dataset({'uts': ((epoch_name), index)},
coords={epoch_name: index})
# need to create simple orbits here. Have start of first orbit
# at 2009,1, 0 UT. 14.84 orbits per day
# figure out how far in time from the root start
# use that info to create a signal that is continuous from that start
# going to presume there are 5820 seconds per orbit (97 minute period)
time_delta = dates[0] - dt.datetime(2009, 1, 1)
# mlt runs 0-24 each orbit.
mlt = mm_test.generate_fake_data(time_delta.total_seconds(),
uts,
period=iperiod['lt'],
data_range=drange['lt'])
data['mlt'] = ((epoch_name), mlt)
# do slt, 20 second offset from mlt
slt = mm_test.generate_fake_data(time_delta.total_seconds() + 20,
uts,
period=iperiod['lt'],
data_range=drange['lt'])
data['slt'] = ((epoch_name), slt)
# create a fake satellite longitude, resets every 6240 seconds
# sat moves at 360/5820 deg/s, Earth rotates at 360/86400, takes extra time
# to go around full longitude
longitude = mm_test.generate_fake_data(time_delta.total_seconds(),
uts,
period=iperiod['lon'],
data_range=drange['lon'])
data['longitude'] = ((epoch_name), longitude)
# create fake satellite latitude for testing polar orbits
angle = mm_test.generate_fake_data(time_delta.total_seconds(),
uts,
period=iperiod['angle'],
data_range=drange['angle'])
latitude = 90.0 * np.cos(angle)
data['latitude'] = ((epoch_name), latitude)
# create constant altitude at 400 km for a satellite that has yet
# to experience orbital decay
alt0 = 400.0
altitude = alt0 * np.ones(data['latitude'].shape)
data['altitude'] = ((epoch_name), altitude)
# create some fake data to support testing of averaging routines
mlt_int = data['mlt'].astype(int).data
long_int = (data['longitude'] / 15.).astype(int).data
data['dummy1'] = ((epoch_name), mlt_int)
data['dummy2'] = ((epoch_name), long_int)
data['dummy3'] = ((epoch_name), mlt_int + long_int * 1000.)
data['dummy4'] = ((epoch_name), uts)
# Add dummy coords
data.coords['x'] = (('x'), np.arange(17))
data.coords['y'] = (('y'), np.arange(17))
data.coords['z'] = (('z'), np.arange(15))
# create altitude 'profile' at each location to simulate remote data
num = len(data['uts'])
data['profiles'] = ((epoch_name, 'profile_height'),
data['dummy3'].values[:, np.newaxis] * np.ones(
(num, 15)))
data.coords['profile_height'] = ('profile_height', np.arange(15))
# profiles that could have different altitude values
data['variable_profiles'] = ((epoch_name,
'z'), data['dummy3'].values[:, np.newaxis] *
np.ones((num, 15)))
data.coords['variable_profile_height'] = ((epoch_name, 'z'),
np.arange(15)[np.newaxis, :] *
np.ones((num, 15)))
# Create fake image type data, projected to lat / lon at some location
# from satellite
data['images'] = ((epoch_name, 'x',
'y'), data['dummy3'].values[:, np.newaxis, np.newaxis] *
np.ones((num, 17, 17)))
data.coords['image_lat'] = \
((epoch_name, 'x', 'y'),
np.arange(17)[np.newaxis,
np.newaxis,
:] * np.ones((num, 17, 17)))
data.coords['image_lon'] = ((epoch_name, 'x', 'y'),
np.arange(17)[np.newaxis, np.newaxis, :] *
np.ones((num, 17, 17)))
# create very limited metadata
meta = pysat.Meta()
meta[epoch_name] = {'long_name': 'Datetime Index'}
meta['uts'] = {'units': 's', 'long_name': 'Universal Time'}
meta['mlt'] = {'units': 'hours', 'long_name': 'Magnetic Local Time'}
meta['slt'] = {'units': 'hours', 'long_name': 'Solar Local Time'}
meta['longitude'] = {'units': 'degrees', 'long_name': 'Longitude'}
meta['latitude'] = {'units': 'degrees', 'long_name': 'Latitude'}
meta['altitude'] = {'units': 'km', 'long_name': 'Altitude'}
variable_profile_meta = pysat.Meta()
variable_profile_meta['variable_profiles'] = {'long_name': 'series'}
meta['variable_profiles'] = {
'meta': variable_profile_meta,
'long_name': 'series'
}
profile_meta = pysat.Meta()
profile_meta['density'] = {'long_name': 'profiles'}
profile_meta['dummy_str'] = {'long_name': 'profiles'}
profile_meta['dummy_ustr'] = {'long_name': 'profiles'}
meta['profiles'] = {'meta': profile_meta, 'long_name': 'profiles'}
image_meta = pysat.Meta()
image_meta['density'] = {'long_name': 'profiles'}
image_meta['fraction'] = {'long_name': 'profiles'}
meta['images'] = {'meta': image_meta, 'long_name': 'profiles'}
for var in data.keys():
if var.find('dummy') >= 0:
meta[var] = {
'units': 'none',
'long_name': var,
'notes': 'Dummy variable'
}
meta['x'] = {
'long_name': 'x-value of image pixel',
'notes': 'Dummy Variable'
}
meta['y'] = {
'long_name': 'y-value of image pixel',
'notes': 'Dummy Variable'
}
meta['z'] = {
'long_name': 'z-value of profile height',
'notes': 'Dummy Variable'
}
meta['image_lat'] = {
'long_name': 'Latitude of image pixel',
'notes': 'Dummy Variable'
}
meta['image_lon'] = {
'long_name': 'Longitude of image pixel',
'notes': 'Dummy Variable'
}
meta['profile_height'] = {'long_name': 'profile height'}
meta['variable_profile_height'] = {'long_name': 'Variable Profile Height'}
return data, meta
def load(fnames, tag=None, sat_id=None, sim_multi_file_right=False,
sim_multi_file_left=False, malformed_index=False,
**kwargs):
""" Loads the test files
Parameters
----------
fnames : list
List of filenames
tag : str or NoneType
Instrument tag (accepts '')
sat_id : str or NoneType
Instrument satellite ID (accepts '' or a number (i.e., '10'), which
specifies the number of data points to include in the test instrument)
sim_multi_file_right : boolean
Adjusts date range to be 12 hours in the future or twelve hours beyond
root_date (default=False)
sim_multi_file_left : boolean
Adjusts date range to be 12 hours in the past or twelve hours before
root_date (default=False)
malformed_index : boolean
If True, time index will be non-unique and non-monotonic.
kwargs : dict
Additional unspecified keywords supplied to pysat.Instrument upon
instantiation are passed here.
Returns
-------
data : (xr.Dataset)
Testing data
meta : (pysat.Meta)
Metadata
"""
# create an artifical satellite data set
iperiod = mm_test.define_period()
drange = mm_test.define_range()
uts, index, date = mm_test.generate_times(fnames, sat_id=sat_id, freq='1S')
if sim_multi_file_right:
root_date = pysat.datetime(2009, 1, 1, 12)
elif sim_multi_file_left:
root_date = pysat.datetime(2008, 12, 31, 12)
else:
root_date = pysat.datetime(2009, 1, 1)
if malformed_index:
index = index.tolist()
# nonmonotonic
index[0:3], index[3:6] = index[3:6], index[0:3]
# non unique
index[6:9] = [index[6]]*3
data = xarray.Dataset({'uts': (('time'), index)}, coords={'time': index})
# need to create simple orbits here. Have start of first orbit
# at 2009,1, 0 UT. 14.84 orbits per day
time_delta = date - root_date
mlt = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
period=iperiod['lt'],
data_range=drange['lt'])
data['mlt'] = (('time'), mlt)
# do slt, 20 second offset from mlt
slt = mm_test.generate_fake_data(time_delta.total_seconds()+20, uts,
period=iperiod['lt'],
data_range=drange['lt'])
data['slt'] = (('time'), slt)
# create a fake longitude, resets every 6240 seconds
# sat moves at 360/5820 deg/s, Earth rotates at 360/86400, takes extra time
# to go around full longitude
longitude = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
period=iperiod['lon'],
data_range=drange['lon'])
data['longitude'] = (('time'), longitude)
# create latitude area for testing polar orbits
angle = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
period=iperiod['angle'],
data_range=drange['angle'])
latitude = 90.0 * np.cos(angle)
data['latitude'] = (('time'), latitude)
# fake orbit number
fake_delta = date - pysat.datetime(2008, 1, 1)
orbit_num = mm_test.generate_fake_data(fake_delta.total_seconds(),
uts, period=iperiod['lt'],
cyclic=False)
data['orbit_num'] = (('time'), orbit_num)
# create some fake data to support testing of averaging routines
mlt_int = data['mlt'].astype(int)
long_int = (data['longitude'] / 15.).astype(int)
data['dummy1'] = (('time'), mlt_int)
data['dummy2'] = (('time'), long_int)
data['dummy3'] = (('time'), mlt_int + long_int * 1000.)
data['dummy4'] = (('time'), uts)
data['string_dummy'] = (('time'), ['test'] * len(data.indexes['time']))
data['unicode_dummy'] = (('time'), [u'test'] * len(data.indexes['time']))
data['int8_dummy'] = (('time'), np.array([1] * len(data.indexes['time']),
dtype=np.int8))
data['int16_dummy'] = (('time'), np.array([1] * len(data.indexes['time']),
dtype=np.int16))
data['int32_dummy'] = (('time'), np.array([1] * len(data.indexes['time']),
dtype=np.int32))
data['int64_dummy'] = (('time'), np.array([1] * len(data.indexes['time']),
dtype=np.int64))
return data, meta.copy()
def load(fnames, tag=None, sat_id=None, malformed_index=False):
""" Loads the test files
Parameters
----------
fnames : list
List of filenames
tag : str or NoneType
Instrument tag (accepts '')
sat_id : str or NoneType
Instrument satellite ID (accepts '' or a number (i.e., '10'), which
specifies the number of data points to include in the test instrument)
malformed_index : bool
If True, the time index will be non-unique and non-monotonic.
(default=False)
Returns
-------
data : pds.DataFrame
Testing data
meta : pysat.Meta
Metadataxs
"""
# create an artifical satellite data set
iperiod = mm_test.define_period()
drange = mm_test.define_range()
# Using 100s frequency for compatibility with seasonal analysis unit tests
uts, index, date = mm_test.generate_times(fnames, sat_id, freq='100S')
# seed DataFrame with UT array
data = pysat.DataFrame(uts, columns=['uts'])
# need to create simple orbits here. Have start of first orbit
# at 2009,1, 0 UT. 14.84 orbits per day
# figure out how far in time from the root start
# use that info to create a signal that is continuous from that start
# going to presume there are 5820 seconds per orbit (97 minute period)
time_delta = date - pysat.datetime(2009, 1, 1)
# mlt runs 0-24 each orbit.
data['mlt'] = mm_test.generate_fake_data(time_delta.total_seconds(),
uts,
period=iperiod['lt'],
data_range=drange['lt'])
# do slt, 20 second offset from mlt
data['slt'] = mm_test.generate_fake_data(time_delta.total_seconds() + 20,
uts,
period=iperiod['lt'],
data_range=drange['lt'])
# create a fake longitude, resets every 6240 seconds
# sat moves at 360/5820 deg/s, Earth rotates at 360/86400, takes extra time
# to go around full longitude
data['longitude'] = mm_test.generate_fake_data(time_delta.total_seconds(),
uts,
period=iperiod['lon'],
data_range=drange['lon'])
# create latitude signal for testing polar orbits
angle = mm_test.generate_fake_data(time_delta.total_seconds(),
uts,
period=iperiod['angle'],
data_range=drange['angle'])
data['latitude'] = 90.0 * np.cos(angle)
if malformed_index:
index = index.tolist()
# nonmonotonic
index[0:3], index[3:6] = index[3:6], index[0:3]
# non unique
index[6:9] = [index[6]] * 3
data.index = index
data.index.name = 'epoch'
# higher rate time signal (for scalar >= 2)
# this time signal used for 2D profiles associated with each time in main
# DataFrame
high_rate_template = pds.date_range(
date, date + pds.DateOffset(hours=0, minutes=1, seconds=39), freq='2S')
# create a few simulated profiles
# DataFrame at each time with mixed variables
profiles = []
# DataFrame at each time with numeric variables only
alt_profiles = []
# Serie at each time, numeric data only
series_profiles = []
# frame indexed by date times
frame = pds.DataFrame(
{
'density': data.loc[data.index[0:50], 'mlt'].values.copy(),
'dummy_str': ['test'] * 50,
'dummy_ustr': [u'test'] * 50
},
index=data.index[0:50],
columns=['density', 'dummy_str', 'dummy_ustr'])
# frame indexed by float
dd = np.arange(50) * 1.2
ff = np.arange(50) / 50.
ii = np.arange(50) * 0.5
frame_alt = pds.DataFrame({
'density': dd,
'fraction': ff
},
index=ii,
columns=['density', 'fraction'])
# series version of storage
series_alt = pds.Series(dd, index=ii, name='series_profiles')
for time in data.index:
frame.index = high_rate_template + (time - data.index[0])
profiles.append(frame)
alt_profiles.append(frame_alt)
series_profiles.append(series_alt)
# store multiple data types into main frame
data['profiles'] = pds.Series(profiles, index=data.index)
data['alt_profiles'] = pds.Series(alt_profiles, index=data.index)
data['series_profiles'] = pds.Series(series_profiles, index=data.index)
return data, meta.copy()
def load(fnames, tag=None, inst_id=None, sim_multi_file_right=False,
sim_multi_file_left=False, malformed_index=False,
num_samples=None, test_load_kwarg=None):
""" Loads the test files
Parameters
----------
fnames : list
List of filenames
tag : str or NoneType
Instrument tag (accepts '')
inst_id : str or NoneType
Instrument satellite ID (accepts '')
sim_multi_file_right : boolean
Adjusts date range to be 12 hours in the future or twelve hours beyond
root_date (default=False)
sim_multi_file_left : boolean
Adjusts date range to be 12 hours in the past or twelve hours before
root_date (default=False)
malformed_index : boolean
If True, time index will be non-unique and non-monotonic.
num_samples : int
Number of samples
test_load_kwarg : any or NoneType
Testing keyword (default=None)
Returns
-------
data : xr.Dataset
Testing data
meta : pysat.Meta
Metadata
"""
# Support keyword testing
logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg))))
# create an artifical satellite data set
iperiod = mm_test.define_period()
drange = mm_test.define_range()
if num_samples is None:
# Default to 1 day at a frequency of 1S
num_samples = 86400
uts, index, dates = mm_test.generate_times(fnames, num_samples,
freq='1S')
if sim_multi_file_right:
root_date = dt.datetime(2009, 1, 1, 12)
elif sim_multi_file_left:
root_date = dt.datetime(2008, 12, 31, 12)
else:
root_date = dt.datetime(2009, 1, 1)
if malformed_index:
index = index.tolist()
# Create a nonmonotonic index
index[0:3], index[3:6] = index[3:6], index[0:3]
# Create a non-unique index
index[6:9] = [index[6]] * 3
data = xr.Dataset({'uts': ((epoch_name), index)},
coords={epoch_name: index})
# need to create simple orbits here. Have start of first orbit
# at 2009,1, 0 UT. 14.84 orbits per day
time_delta = dates[0] - root_date
mlt = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
period=iperiod['lt'],
data_range=drange['lt'])
data['mlt'] = ((epoch_name), mlt)
# do slt, 20 second offset from mlt
slt = mm_test.generate_fake_data(time_delta.total_seconds() + 20, uts,
period=iperiod['lt'],
data_range=drange['lt'])
data['slt'] = ((epoch_name), slt)
# create a fake longitude, resets every 6240 seconds
# sat moves at 360/5820 deg/s, Earth rotates at 360/86400, takes extra time
# to go around full longitude
longitude = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
period=iperiod['lon'],
data_range=drange['lon'])
data['longitude'] = ((epoch_name), longitude)
# create latitude area for testing polar orbits
angle = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
period=iperiod['angle'],
data_range=drange['angle'])
latitude = 90.0 * np.cos(angle)
data['latitude'] = ((epoch_name), latitude)
# create constant altitude at 400 km
alt0 = 400.0
altitude = alt0 * np.ones(data['latitude'].shape)
data['altitude'] = ((epoch_name), altitude)
# fake orbit number
fake_delta = dates[0] - dt.datetime(2008, 1, 1)
orbit_num = mm_test.generate_fake_data(fake_delta.total_seconds(),
uts, period=iperiod['lt'],
cyclic=False)
data['orbit_num'] = ((epoch_name), orbit_num)
# create some fake data to support testing of averaging routines
mlt_int = data['mlt'].astype(int).data
long_int = (data['longitude'] / 15.).astype(int).data
data['dummy1'] = ((epoch_name), mlt_int)
data['dummy2'] = ((epoch_name), long_int)
data['dummy3'] = ((epoch_name), mlt_int + long_int * 1000.)
data['dummy4'] = ((epoch_name), uts)
data['string_dummy'] = ((epoch_name),
['test'] * len(data.indexes[epoch_name]))
data['unicode_dummy'] = ((epoch_name),
[u'test'] * len(data.indexes[epoch_name]))
data['int8_dummy'] = ((epoch_name),
np.array([1] * len(data.indexes[epoch_name]),
dtype=np.int8))
data['int16_dummy'] = ((epoch_name),
np.array([1] * len(data.indexes[epoch_name]),
dtype=np.int16))
data['int32_dummy'] = ((epoch_name),
np.array([1] * len(data.indexes[epoch_name]),
dtype=np.int32))
data['int64_dummy'] = ((epoch_name),
np.array([1] * len(data.indexes[epoch_name]),
dtype=np.int64))
meta = pysat.Meta()
meta['uts'] = {'units': 's', 'long_name': 'Universal Time',
'custom': False}
meta[epoch_name] = {'units': 'Milliseconds since 1970-1-1',
'Bin_Location': 0.5,
'notes':
'UTC time at middle of geophysical measurement.',
'desc': 'UTC seconds', }
meta['mlt'] = {'units': 'hours',
'long_name': 'Magnetic Local Time',
'desc': 'Magnetic Local Time',
'value_min': 0.,
'value_max': 24.,
'notes': ''.join(['Magnetic Local Time is the solar local ',
'time of the field line at the location ',
'where the field crosses the magnetic ',
'equator. In this case we just simulate ',
'0-24 with a consistent orbital period ',
'and an offset with SLT.'])}
meta['slt'] = {'units': 'hours', 'long_name': 'Solar Local Time',
'desc': 'Solar Local Time', 'value_min': 0.,
'value_max': 24.,
'notes': ''.join(['Solar Local Time is the local time ',
'(zenith angle of thee sun) of the given',
' locaiton. Overhead noon, +/- 90 is 6,',
' 18 SLT .'])}
meta['orbit_num'] = {'long_name': 'Orbit Number', 'desc': 'Orbit Number',
'value_min': 0., 'value_max': 25000.,
'notes': ''.join(['Number of orbits since the start ',
'of the mission. For this ',
'simulation we use the ',
'number of 5820 second periods ',
'since the start, 2008-01-01.'])}
meta['longitude'] = {'units': 'degrees', 'long_name': 'Longitude'}
meta['latitude'] = {'units': 'degrees', 'long_name': 'Latitude'}
meta['altitude'] = {'units': 'km', 'long_name': 'Altitude'}
for var in data.keys():
if var.find('dummy') >= 0:
meta[var] = {'units': 'none', 'notes': 'Dummy variable'}
return data, meta
def load(fnames, tag=None, sat_id=None, malformed_index=False):
""" Loads the test files
Parameters
----------
fnames : list
List of filenames
tag : str or NoneType
Instrument tag (accepts '')
sat_id : str or NoneType
Instrument satellite ID (accepts '' or a number (i.e., '10'), which
specifies the number of data points to include in the test instrument)
malformed_index : bool False
If True, the time index will be non-unique and non-monotonic.
Returns
-------
data : xr.Dataset
Testing data
meta : pysat.Meta
Metadataxs
"""
# create an artifical satellite data set
iperiod = mm_test.define_period()
drange = mm_test.define_range()
# Using 100s frequency for compatibility with seasonal analysis unit tests
uts, index, date = mm_test.generate_times(fnames, sat_id, freq='100S')
if malformed_index:
index = index.tolist()
# nonmonotonic
index[0:3], index[3:6] = index[3:6], index[0:3]
# non unique
index[6:9] = [index[6]]*3
data = xr.Dataset({'uts': (('time'), index)}, coords={'time': index})
# need to create simple orbits here. Have start of first orbit
# at 2009,1, 0 UT. 14.84 orbits per day
# figure out how far in time from the root start
# use that info to create a signal that is continuous from that start
# going to presume there are 5820 seconds per orbit (97 minute period)
time_delta = date - pysat.datetime(2009, 1, 1)
# mlt runs 0-24 each orbit.
mlt = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
period=iperiod['lt'],
data_range=drange['lt'])
data['mlt'] = (('time'), mlt)
# do slt, 20 second offset from mlt
slt = mm_test.generate_fake_data(time_delta.total_seconds()+20, uts,
period=iperiod['lt'],
data_range=drange['lt'])
data['slt'] = (('time'), slt)
# create a fake longitude, resets every 6240 seconds
# sat moves at 360/5820 deg/s, Earth rotates at 360/86400, takes extra time
# to go around full longitude
longitude = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
period=iperiod['lon'],
data_range=drange['lon'])
data['longitude'] = (('time'), longitude)
# create latitude signal for testing polar orbits
angle = mm_test.generate_fake_data(time_delta.total_seconds(), uts,
period=iperiod['angle'],
data_range=drange['angle'])
latitude = 90.0 * np.cos(angle)
data['latitude'] = (('time'), latitude)
# create some fake data to support testing of averaging routines
mlt_int = data['mlt'].astype(int)
long_int = (data['longitude'] / 15.).astype(int)
data['dummy1'] = (('time'), mlt_int)
data['dummy2'] = (('time'), long_int)
data['dummy3'] = (('time'), mlt_int + long_int * 1000.)
data['dummy4'] = (('time'), uts)
# create altitude 'profile' at each location
num = len(data['uts'])
data['profiles'] = \
(('time', 'altitude'),
data['dummy3'].values[:, np.newaxis] * np.ones((num, 15)))
data.coords['altitude'] = ('altitude', np.arange(15))
# profiles that could have different altitude values
data['variable_profiles'] = \
(('time', 'z'),
data['dummy3'].values[:, np.newaxis] * np.ones((num, 15)))
data.coords['altitude2'] = \
(('time', 'z'),
np.arange(15)[np.newaxis, :]*np.ones((num, 15)))
# basic image simulation
data['images'] = \
(('time', 'x', 'y'),
data['dummy3'].values[:,
np.newaxis,
np.newaxis] * np.ones((num, 17, 17)))
data.coords['latitude'] = \
(('time', 'x', 'y'),
np.arange(17)[np.newaxis,
np.newaxis,
:]*np.ones((num, 17, 17)))
data.coords['longitude'] = \
(('time', 'x', 'y'),
np.arange(17)[np.newaxis,
np.newaxis,
:] * np.ones((num, 17, 17)))
return data, meta.copy()
def load(fnames,
tag=None,
inst_id=None,
sim_multi_file_right=False,
sim_multi_file_left=False,
root_date=None,
malformed_index=False,
num_samples=None,
test_load_kwarg=None):
""" Loads the test files
Parameters
----------
fnames : list
List of filenames
tag : str or NoneType
Instrument tag (accepts '' or a string to change the behaviour of
certain instrument aspects for testing)
inst_id : str or NoneType
Instrument satellite ID (accepts '')
sim_multi_file_right : boolean
Adjusts date range to be 12 hours in the future or twelve hours beyond
root_date (default=False)
sim_multi_file_left : boolean
Adjusts date range to be 12 hours in the past or twelve hours before
root_date (default=False)
root_date : NoneType
Optional central date, uses _test_dates if not specified.
(default=None)
malformed_index : boolean
If True, time index will be non-unique and non-monotonic (default=False)
num_samples : int
Number of samples per day
test_load_kwarg : any or NoneType
Testing keyword (default=None)
Returns
-------
data : pds.DataFrame
Testing data
meta : pysat.Meta
Metadata
"""
# Support keyword testing
logger.info(''.join(('test_load_kwarg = ', str(test_load_kwarg))))
# create an artificial satellite data set
iperiod = mm_test.define_period()
drange = mm_test.define_range()
if num_samples is None:
# Default to 1 day at a frequency of 1S
num_samples = 86400
uts, index, dates = mm_test.generate_times(fnames, num_samples, freq='1S')
# Specify the date tag locally and determine the desired date range
pds_offset = dt.timedelta(hours=12)
if sim_multi_file_right:
root_date = root_date or _test_dates[''][''] + pds_offset
elif sim_multi_file_left:
root_date = root_date or _test_dates[''][''] - pds_offset
else:
root_date = root_date or _test_dates['']['']
# store UTS, mod 86400
data = pds.DataFrame(np.mod(uts, 86400.), columns=['uts'])
# need to create simple orbits here. Have start of first orbit default
# to 1 Jan 2009, 00:00 UT. 14.84 orbits per day
time_delta = dates[0] - root_date
data['mlt'] = mm_test.generate_fake_data(time_delta.total_seconds(),
uts,
period=iperiod['lt'],
data_range=drange['lt'])
# do slt, 20 second offset from mlt
data['slt'] = mm_test.generate_fake_data(time_delta.total_seconds() + 20,
uts,
period=iperiod['lt'],
data_range=drange['lt'])
# create a fake longitude, resets every 6240 seconds
# sat moves at 360/5820 deg/s, Earth rotates at 360/86400, takes extra time
# to go around full longitude
data['longitude'] = mm_test.generate_fake_data(time_delta.total_seconds(),
uts,
period=iperiod['lon'],
data_range=drange['lon'])
# create latitude area for testing polar orbits
angle = mm_test.generate_fake_data(time_delta.total_seconds(),
uts,
period=iperiod['angle'],
data_range=drange['angle'])
data['latitude'] = 90.0 * np.cos(angle)
# create constant altitude at 400 km
alt0 = 400.0
data['altitude'] = alt0 * np.ones(data['latitude'].shape)
# fake orbit number
fake_delta = dates[0] - (_test_dates[''][''] - pds.DateOffset(years=1))
data['orbit_num'] = mm_test.generate_fake_data(fake_delta.total_seconds(),
uts,
period=iperiod['lt'],
cyclic=False)
# create some fake data to support testing of averaging routines
mlt_int = data['mlt'].astype(int)
long_int = (data['longitude'] / 15.0).astype(int)
data['dummy1'] = mlt_int
data['dummy2'] = long_int
data['dummy3'] = mlt_int + long_int * 1000.0
data['dummy4'] = uts
data['string_dummy'] = ['test'] * len(data)
data['unicode_dummy'] = [u'test'] * len(data)
data['int8_dummy'] = np.ones(len(data), dtype=np.int8)
data['int16_dummy'] = np.ones(len(data), dtype=np.int16)
data['int32_dummy'] = np.ones(len(data), dtype=np.int32)
data['int64_dummy'] = np.ones(len(data), dtype=np.int64)
# Activate for testing malformed_index, and for instrument_test_class
if malformed_index or tag == 'non_strict':
index = index.tolist()
# nonmonotonic
index[0:3], index[3:6] = index[3:6], index[0:3]
# non unique
index[6:9] = [index[6]] * 3
data.index = index
data.index.name = 'Epoch'
# Set the meta data
meta = pysat.Meta()
meta['uts'] = {
'units': 's',
'long_name': 'Universal Time',
'custom': False
}
meta['Epoch'] = {
'units': 'Milliseconds since 1970-1-1',
'Bin_Location': 0.5,
'notes': 'UTC time at middle of geophysical measurement.',
'desc': 'UTC seconds'
}
meta['mlt'] = {
'units':
'hours',
'long_name':
'Magnetic Local Time',
'desc':
'Magnetic Local Time',
'value_min':
0.0,
'value_max':
24.0,
'notes':
''.join([
'Magnetic Local Time is the solar local ',
'time of thefield line at the location ',
'where the field crosses the magnetic ',
'equator. In this case we just simulate ',
'0-24 with a consistent orbital period ', 'and an offset with SLT.'
]),
'scale':
'linear'
}
meta['slt'] = {
'units':
'hours',
'long_name':
'Solar Local Time',
'desc':
'Solar Local Time',
'value_min':
0.0,
'value_max':
24.0,
'notes':
''.join([
'Solar Local Time is the local time ',
'(zenith angle of sun) of the given ',
'location. Overhead noon, +/- 90 is 6, ', '18 SLT .'
])
}
meta['orbit_num'] = {
'units':
'',
'long_name':
'Orbit Number',
'desc':
'Orbit Number',
'value_min':
0.0,
'value_max':
25000.0,
'notes':
''.join([
'Number of orbits since the start ', 'of the mission. For this ',
'simulation we use the number of ',
'5820 second periods since the ', 'start, 2008-01-01.'
])
}
meta['longitude'] = {'units': 'degrees', 'long_name': 'Longitude'}
meta['latitude'] = {'units': 'degrees', 'long_name': 'Latitude'}
meta['altitude'] = {'units': 'km', 'long_name': 'Altitude'}
if tag != 'default_meta':
for var in data.keys():
if var.find('dummy') >= 0:
meta[var] = {
'units': 'none',
'notes': 'Dummy variable for testing'
}
return data, meta