def test_files_when_duplicates_forced(self):
# create new files and make sure that new files are captured
start = pysat.datetime(2008, 1, 11)
stop = pysat.datetime(2008, 1, 15)
dates = pysat.utils.time.create_date_range(start, stop, freq='1D')
# clear out old files, create new ones
remove_files(self.testInst)
create_versioned_files(self.testInst, start, stop, freq='1D',
use_doy=False,
root_fname=''.join(('pysat_testing_unique_',
'{version:02d}_',
'{revision:03d}_{year:04d}',
'_g_{day:03d}_st.pysat_',
'testing_file')))
pysat.instruments.pysat_testing.list_files = list_files
self.testInst = \
pysat.Instrument(inst_module=pysat.instruments.pysat_testing,
clean_level='clean',
file_format=''.join(('pysat_testing_unique_??_',
'???_{year:04d}_g_{day:03d}',
'_st.pysat_testing_file')),
update_files=True,
temporary_file_list=self.temporary_file_list)
assert (np.all(self.testInst.files.files.index == dates))
def test_set_bounds_by_date_season(self):
start = [pysat.datetime(2009, 1, 1), pysat.datetime(2009, 2, 1)]
stop = [pysat.datetime(2009, 1, 15), pysat.datetime(2009, 2, 15)]
self.testInst.bounds = (start, stop)
out = pds.date_range(start[0], stop[0]).tolist()
out.extend(pds.date_range(start[1], stop[1]).tolist())
assert np.all(self.testInst._iter_list == out)
def create_versioned_files(inst, start=None, stop=None, freq='1D', use_doy=True,
root_fname=None):
# create a bunch of files
if start is None:
start = pysat.datetime(2009, 1, 1)
if stop is None:
stop = pysat.datetime(2013, 12, 31)
dates = pysat.utils.season_date_range(start, stop, freq=freq)
versions = np.array([1, 2])
revisions = np.array([0, 1])
if root_fname is None:
root_fname = 'pysat_testing_junk_{year:04d}_{month:02d}_{day:03d}{hour:02d}{min:02d}{sec:02d}_stuff_{version:02d}_{revision:03d}.pysat_testing_file'
# create empty file
for date in dates:
for version in versions:
for revision in revisions:
yr, doy = pysat.utils.getyrdoy(date)
if use_doy:
doy = doy
else:
doy = date.day
fname = os.path.join(inst.files.data_path, root_fname.format(year=yr,
day=doy, month=date.month, hour=date.hour,
min=date.minute, sec=date.second,
version=version, revision=revision))
with open(fname, 'w') as f:
pass
def test_year_month_day_hour_files_direct_call_to_from_os(self):
# create a bunch of files by year and doy
start = pysat.datetime(2008, 1, 1)
stop = pysat.datetime(2009, 12, 31)
create_files(
self.testInst,
start,
stop,
freq='6h',
use_doy=False,
root_fname=
'pysat_testing_junk_{year:04d}_gold_{day:03d}_stuff_{month:02d}_{hour:02d}.pysat_testing_file'
)
# use from_os function to get pandas Series of files and dates
files = pysat.Files.from_os(
data_path=self.testInst.files.data_path,
format_str=
'pysat_testing_junk_{year:04d}_gold_{day:03d}_stuff_{month:02d}_{hour:02d}.pysat_testing_file'
)
# check overall length
check1 = len(files) == (365 + 366) * 4 - 3
# check specific dates
check2 = files.index[0].to_datetime() == pysat.datetime(2008, 1, 1)
check3 = files.index[1460].to_datetime() == pysat.datetime(
2008, 12, 31)
check4 = files.index[-1].to_datetime() == pysat.datetime(2009, 12, 31)
assert (check1 & check2 & check3 & check4)
def load(fnames, tag=None, sat_id=None):
# create an artifical satellite data set
parts = fnames[0].split('/')
yr = int('20' + parts[-1][0:2])
month = int(parts[-3])
day = int(parts[-2])
date = pysat.datetime(yr, month, day)
num = 86400 #int(tag)
uts = np.arange(num)
data = pysat.DataFrame(uts, columns=['uts'])
# need to create simple orbits here. Have start of first orbit
# at 2009,1, 0 UT. 14.84 orbits per day
time_delta = date - pysat.datetime(2009, 1, 1)
uts_root = np.mod(time_delta.total_seconds(), 5820)
mlt = np.mod(uts_root + np.arange(num), 5820) * (24. / 5820.)
data['mlt'] = mlt
# do slt, 20 second offset from mlt
uts_root = np.mod(time_delta.total_seconds() + 20, 5820)
data['slt'] = np.mod(uts_root + np.arange(num), 5820) * (24. / 5820.)
index = pds.date_range(date,
date +
pds.DateOffset(hours=23, minutes=59, seconds=59),
freq='S')
data.index = index
data.index.name = 'time'
return data, meta.copy()
def list_files(tag=None, sat_id=None, data_path=None, format_str=None):
"""Produce a fake list of files spanning a year"""
index = pds.date_range(pysat.datetime(2008, 1, 1),
pysat.datetime(2010, 12, 31))
names = [data_path + date.strftime('%D') + '.nofile' for date in index]
return pysat.Series(names, index=index)
def test_basic_orbit_mean(self):
orbit_info = {'kind':'local time', 'index':'mlt'}
self.testInst = pysat.Instrument('pysat','testing', clean_level='clean', orbit_info=orbit_info)
self.testInst.bounds = (pysat.datetime(2009,1,1), pysat.datetime(2009,1,2))
ans = pysat.ssnl.avg.mean_by_orbit(self.testInst, 'mlt')
# note last orbit is incomplete thus not expected to satisfy relation
assert np.allclose(ans[:-1], np.ones(len(ans)-1)*12., 1.E-2)
def test_set_bounds_by_date_season(self):
start = [pysat.datetime(2009,1,1), pysat.datetime(2009,2,1)]
stop = [pysat.datetime(2009,1,15), pysat.datetime(2009,2,15)]
self.testInst.bounds = (start, stop)
out = pds.date_range(start[0], stop[0]).tolist()
out.extend(pds.date_range(start[1], stop[1]).tolist())
assert np.all(self.testInst._iter_list == out)
def test_heterogenous_constellation_average(self):
for inst in self.testC:
inst.bounds = (pysat.datetime(2008,1,1), pysat.datetime(2008,2,1))
results = pysat.ssnl.avg.median2D(self.testC, [0., 360., 24.], 'longitude',
[0., 24, 24], 'mlt', ['dummy1', 'dummy2', 'dummy3'])
dummy_val = results['dummy1']['median']
dummy_dev = results['dummy1']['avg_abs_dev']
dummy2_val = results['dummy2']['median']
dummy2_dev = results['dummy2']['avg_abs_dev']
dummy3_val = results['dummy3']['median']
dummy3_dev = results['dummy3']['avg_abs_dev']
dummy_x = results['dummy1']['bin_x']
dummy_y = results['dummy1']['bin_y']
# iterate over all y rows, value should be equal to integer value of mlt
# no variation in the median, all values should be the same
check = []
for i, y in enumerate(dummy_y[:-1]):
check.append(np.all(dummy_val[i, :] == y.astype(int)))
check.append(np.all(dummy_dev[i, :] == 0))
for i, x in enumerate(dummy_x[:-1]):
check.append(np.all(dummy2_val[:, i] == x/15.) )
check.append(np.all(dummy2_dev[:, i] == 0))
for i, x in enumerate(dummy_x[:-1]):
check.append(np.all(dummy3_val[:, i] == x/15.*1000. + dummy_y[:-1]) )
check.append(np.all(dummy3_dev[:, i] == 0))
assert np.all(check)
def setup(self):
"""Runs before every method to create a clean testing setup."""
self.testInst = pysat.Instrument(platform='pysat',
name='testing',
clean_level='clean')
self.start = pysat.datetime(2009, 1, 1)
self.stop = pysat.datetime(2009, 1, 1)
def test_year_month_files_direct_call_to_from_os(self):
# create a bunch of files by year and doy
start = pysat.datetime(2008, 1, 1)
stop = pysat.datetime(2009, 12, 31)
create_files(
self.testInst,
start,
stop,
freq='1MS',
root_fname=
'pysat_testing_junk_{year:04d}_gold_stuff_{month:02d}.pysat_testing_file'
)
# use from_os function to get pandas Series of files and dates
files = pysat.Files.from_os(
data_path=self.testInst.files.data_path,
format_str=
'pysat_testing_junk_{year:04d}_gold_stuff_{month:02d}.pysat_testing_file'
)
# check overall length
check1 = len(files) == 24
# check specific dates
check2 = pds.to_datetime(files.index[0]) == pysat.datetime(2008, 1, 1)
check3 = pds.to_datetime(files.index[11]) == pysat.datetime(
2008, 12, 1)
check4 = pds.to_datetime(files.index[-1]) == pysat.datetime(
2009, 12, 1)
assert (check1 & check2 & check3 & check4)
def test_basic_file_mean(self):
index = pds.date_range(pysat.datetime(2008, 1, 1),
pysat.datetime(2008, 2, 1))
names = [date.strftime('%D') + '.nofile' for date in index]
self.testInst.bounds = (names[0], names[-1])
ans = pysat.ssnl.avg.mean_by_file(self.testInst, 'dummy4')
assert np.all(ans == 86399 / 2.)
def create_files(inst,
start=None,
stop=None,
freq='1D',
use_doy=True,
root_fname=None):
# create a bunch of files
if start is None:
start = pysat.datetime(2009, 1, 1)
if stop is None:
stop = pysat.datetime(2013, 12, 31)
dates = pysat.utils.season_date_range(start, stop, freq=freq)
if root_fname is None:
root_fname = 'pysat_testing_junk_{year:04d}_gold_{day:03d}_stuff.pysat_testing_file'
# create empty file
for date in dates:
yr, doy = pysat.utils.getyrdoy(date)
if use_doy:
doy = doy
else:
doy = date.day
fname = os.path.join(
inst.files.data_path,
root_fname.format(year=yr,
day=doy,
month=date.month,
hour=date.hour,
min=date.minute,
sec=date.second))
with open(fname, 'w') as f:
pass #f.close()
def test_get_new_files_after_deleting_files_and_adding_files(self):
# create new files and make sure that new files are captured
start = pysat.datetime(2008, 1, 11)
stop = pysat.datetime(2008, 1, 12)
dates = pysat.utils.season_date_range(start, stop, freq='100min')
# remove files, same number as will be added
to_be_removed = len(dates)
for the_file in os.listdir(self.testInst.files.data_path):
if (the_file[0:13] == 'pysat_testing') & (the_file[-19:] == '.pysat_testing_file'):
file_path = os.path.join(self.testInst.files.data_path, the_file)
if os.path.isfile(file_path) & (to_be_removed > 0):
to_be_removed -= 1
# Remove all versions of the file
# otherwise, previous versions will look like new files
pattern = '_'.join(file_path.split('_')[0:7])+'*.pysat_testing_file'
map(os.unlink, glob.glob(pattern))
#os.unlink(file_path)
# add new files
create_versioned_files(self.testInst, start, stop, freq='100min',
use_doy=False,
root_fname=self.root_fname)
# get new files
new_files = self.testInst.files.get_new()
assert (np.all(new_files.index == dates))
def test_files_non_standard_pysat_directory(self):
# create new files and make sure that new files are captured
start = pysat.datetime(2008, 1, 11)
stop = pysat.datetime(2008, 1, 15)
dates = pysat.utils.season_date_range(start, stop, freq='100min')
pysat.instruments.pysat_testing.list_files = list_versioned_files
self.testInst = pysat.Instrument(inst_module=pysat.instruments.pysat_testing,
clean_level='clean',
sat_id='hello',
directory_format='pysat_testing_{tag}_{sat_id}',
update_files=True,
temporary_file_list=self.temporary_file_list)
# add new files
create_dir(self.testInst)
remove_files(self.testInst)
create_versioned_files(self.testInst, start, stop, freq='100min',
use_doy=False,
root_fname=self.root_fname)
self.testInst = pysat.Instrument(inst_module=pysat.instruments.pysat_testing,
clean_level='clean',
sat_id='hello',
directory_format='pysat_testing_{tag}_{sat_id}',
update_files=True,
temporary_file_list=self.temporary_file_list)
# get new files
new_files = self.testInst.files.get_new()
assert (np.all(self.testInst.files.files.index == dates) &
np.all(new_files.index == dates))
def setup(self):
"""Runs before every method to create a clean testing setup."""
# store current pysat directory
self.data_path = pysat.data_dir
# create temporary directory
dir_name = tempfile.gettempdir()
pysat.utils.set_data_dir(dir_name, store=False)
# create testing directory
create_dir(temporary_file_list=self.temporary_file_list)
# create a test instrument, make sure it is getting files from filesystem
re_load(pysat.instruments.pysat_testing)
# self.stored_files_fcn = pysat.instruments.pysat_testing.list_files
pysat.instruments.pysat_testing.list_files = list_versioned_files
# create a bunch of files by year and doy
self.testInst = pysat.Instrument(
inst_module=pysat.instruments.pysat_testing,
clean_level='clean',
temporary_file_list=self.temporary_file_list)
self.root_fname = 'pysat_testing_junk_{year:04d}_{month:02d}_{day:03d}{hour:02d}{min:02d}{sec:02d}_stuff_{version:02d}_{revision:03d}.pysat_testing_file'
start = pysat.datetime(2007, 12, 31)
stop = pysat.datetime(2008, 1, 10)
create_versioned_files(self.testInst,
start,
stop,
freq='100min',
use_doy=False,
root_fname=self.root_fname)
self.testInst = pysat.Instrument(
inst_module=pysat.instruments.pysat_testing,
clean_level='clean',
update_files=True,
temporary_file_list=self.temporary_file_list)
def test_get_new_files_after_deleting_files_and_adding_files(self):
# create new files and make sure that new files are captured
start = pysat.datetime(2008, 1, 11)
stop = pysat.datetime(2008, 1, 12)
dates = pysat.utils.season_date_range(start, stop, freq='100min')
# remove files, same number as will be added
to_be_removed = len(dates)
for the_file in os.listdir(self.testInst.files.data_path):
if (the_file[0:13] == 'pysat_testing') & (
the_file[-19:] == '.pysat_testing_file'):
file_path = os.path.join(self.testInst.files.data_path,
the_file)
if os.path.isfile(file_path) & (to_be_removed > 0):
to_be_removed -= 1
# Remove all versions of the file
# otherwise, previous versions will look like new files
pattern = '_'.join(
file_path.split('_')[0:7]) + '*.pysat_testing_file'
map(os.unlink, glob.glob(pattern))
#os.unlink(file_path)
# add new files
create_versioned_files(self.testInst,
start,
stop,
freq='100min',
use_doy=False,
root_fname=self.root_fname)
# get new files
new_files = self.testInst.files.get_new()
assert (np.all(new_files.index == dates))
def test_refresh_on_unchanged_files(self):
start = pysat.datetime(2007, 12, 31)
stop = pysat.datetime(2008, 1, 10)
dates = pysat.utils.season_date_range(start, stop, freq='100min')
self.testInst.files.refresh()
assert (np.all(self.testInst.files.files.index == dates))
def test_refresh_on_unchanged_files(self):
start = pysat.datetime(2007, 12, 31)
stop = pysat.datetime(2008, 1, 10)
dates = pysat.utils.season_date_range(start, stop, freq='100min')
self.testInst.files.refresh()
assert (np.all(self.testInst.files.files.index == dates))
def test_get_new_files_after_adding_files_and_adding_file(self):
# create new files and make sure that new files are captured
start = pysat.datetime(2008, 1, 11)
stop = pysat.datetime(2008, 1, 12)
create_versioned_files(self.testInst,
start,
stop,
freq='100min',
use_doy=False,
root_fname=self.root_fname)
dates = pysat.utils.season_date_range(start, stop, freq='100min')
new_files = self.testInst.files.get_new()
start = pysat.datetime(2008, 1, 15)
stop = pysat.datetime(2008, 1, 18)
create_versioned_files(self.testInst,
start,
stop,
freq='100min',
use_doy=False,
root_fname=self.root_fname)
dates2 = pysat.utils.season_date_range(start, stop, freq='100min')
new_files2 = self.testInst.files.get_new()
assert (np.all(new_files.index == dates)
& np.all(new_files2.index == dates2))
def test_files_non_standard_file_format_template(self):
# create new files and make sure that new files are captured
start = pysat.datetime(2008, 1, 11)
stop = pysat.datetime(2008, 1, 15)
dates = pysat.utils.season_date_range(start, stop, freq='1D')
# clear out old files, create new ones
remove_files(self.testInst)
create_files(
self.testInst,
start,
stop,
freq='1D',
use_doy=False,
root_fname=
'pysat_testing_unique_junk_{year:04d}_gold_{day:03d}_stuff.pysat_testing_file'
)
pysat.instruments.pysat_testing.list_files = list_files
self.testInst = pysat.Instrument(
inst_module=pysat.instruments.pysat_testing,
clean_level='clean',
file_format=
'pysat_testing_unique_junk_{year:04d}_gold_{day:03d}_stuff.pysat_testing_file',
update_files=True,
temporary_file_list=self.temporary_file_list)
assert (np.all(self.testInst.files.files.index == dates))
def setup(self):
"""Runs before every method to create a clean testing setup."""
self.testInst = pysat.Instrument('pysat', 'testing2D',
clean_level='clean')
self.testInst.bounds = (pysat.datetime(2008, 1, 1),
pysat.datetime(2008, 2, 1))
self.dname = 'series_profiles'
def test_files_non_standard_pysat_directory(self):
# create new files and make sure that new files are captured
start = pysat.datetime(2008, 1, 11)
stop = pysat.datetime(2008, 1, 15)
dates = pysat.utils.season_date_range(start, stop, freq='100min')
self.testInst = pysat.Instrument(
inst_module=pysat.instruments.pysat_testing,
clean_level='clean',
sat_id='hello',
directory_format='pysat_testing_{tag}_{sat_id}',
update_files=True,
temporary_file_list=self.temporary_file_list)
# add new files
create_dir(self.testInst)
remove_files(self.testInst)
create_files(self.testInst,
start,
stop,
freq='100min',
use_doy=False,
root_fname=self.root_fname)
self.testInst = pysat.Instrument(
inst_module=pysat.instruments.pysat_testing,
clean_level='clean',
sat_id='hello',
directory_format='pysat_testing_{tag}_{sat_id}',
update_files=True,
temporary_file_list=self.temporary_file_list)
# get new files
new_files = self.testInst.files.get_new()
assert (np.all(self.testInst.files.files.index == dates)
& np.all(new_files.index == dates))
def setup(self):
"""Runs before every method to create a clean testing setup."""
self.testInst = pysat.Instrument('pysat',
'testing',
clean_level='clean')
self.testInst.bounds = (pysat.datetime(2008, 1,
1), pysat.datetime(2008, 1, 3))
def test_get_new_files_after_deleting_files_and_adding_files(self):
# create new files and make sure that new files are captured
start = pysat.datetime(2008, 1, 11)
stop = pysat.datetime(2008, 1, 12)
dates = pysat.utils.season_date_range(start, stop, freq='100min')
# remove files, same number as will be added
to_be_removed = len(dates)
for the_file in os.listdir(self.testInst.files.data_path):
if (the_file[0:13] == 'pysat_testing') & (
the_file[-19:] == '.pysat_testing_file'):
file_path = os.path.join(self.testInst.files.data_path,
the_file)
if os.path.isfile(file_path) & (to_be_removed > 0):
#print(file_path)
to_be_removed -= 1
os.unlink(file_path)
# add new files
create_files(self.testInst,
start,
stop,
freq='100min',
use_doy=False,
root_fname=self.root_fname)
# get new files
new_files = self.testInst.files.get_new()
assert (np.all(new_files.index == dates))
def setup(self):
"""Runs before every method to create a clean testing setup."""
# store current pysat directory
self.data_path = pysat.data_dir
# create temporary directory
dir_name = tempfile.gettempdir()
pysat.utils.set_data_dir(dir_name, store=False)
# create testing directory
create_dir(temporary_file_list=self.temporary_file_list)
# create a test instrument, make sure it is getting files from filesystem
re_load(pysat.instruments.pysat_testing)
# self.stored_files_fcn = pysat.instruments.pysat_testing.list_files
pysat.instruments.pysat_testing.list_files = list_versioned_files
# create a bunch of files by year and doy
self.testInst = pysat.Instrument(inst_module=pysat.instruments.pysat_testing,
clean_level='clean',
temporary_file_list=self.temporary_file_list)
self.root_fname = 'pysat_testing_junk_{year:04d}_{month:02d}_{day:03d}{hour:02d}{min:02d}{sec:02d}_stuff_{version:02d}_{revision:03d}.pysat_testing_file'
start = pysat.datetime(2007, 12, 31)
stop = pysat.datetime(2008, 1, 10)
create_versioned_files(self.testInst, start, stop, freq='100min',
use_doy=False,
root_fname=self.root_fname)
self.testInst = pysat.Instrument(inst_module=pysat.instruments.pysat_testing,
clean_level='clean', update_files=True,
temporary_file_list=self.temporary_file_list)
def setup(self):
"""Runs before every method to create a clean testing setup"""
self.testInst = pysat.Instrument('pysat', 'testing',
clean_level='clean',
update_files=True)
self.testInst.bounds = (pysat.datetime(2008, 1, 1),
pysat.datetime(2008, 1, 31))
self.test_bins = [0, 24, 24]
self.test_label = 'slt'
self.test_data = ['dummy1', 'dummy2']
self.out_keys = ['count', 'avg_abs_dev', 'median', 'bin_x']
self.out_data = {'dummy1':
{'count': [111780., 111320., 111780., 111320.,
111780., 111320., 111780., 111320.,
111780., 111320., 111780., 111320.,
111780., 111320., 111918., 111562.,
112023., 111562., 112023., 111412.,
111780., 111320., 111780., 111320.],
'avg_abs_dev': np.zeros(shape=24),
'median': np.linspace(0.0, 23.0, 24.0)},
'dummy2':
{'count': [111780., 111320., 111780., 111320.,
111780., 111320., 111780., 111320.,
111780., 111320., 111780., 111320.,
111780., 111320., 111918., 111562.,
112023., 111562., 112023., 111412.,
111780., 111320., 111780., 111320.],
'avg_abs_dev': np.zeros(shape=24) + 6.0,
'median': [11., 12., 11., 11., 12., 11., 12., 11.,
12., 12., 11., 12., 11., 12., 11., 11.,
12., 11., 12., 11., 11., 11., 11., 12.]}}
def test_basic_instrument_load_by_date(self):
date = pysat.datetime(2009, 1, 1)
self.testInst.load(date=date)
test_date = self.testInst.index[0]
test_date = pysat.datetime(test_date.year, test_date.month,
test_date.day)
assert (test_date == pds.datetime(
2009, 1, 1)) & (test_date == self.testInst.date)
def setup(self):
insts = []
for i in range(5):
insts.append(pysat.Instrument('pysat', 'testing',
clean_level='clean'))
self.testC = pysat.Constellation(instruments=insts)
self.testI = pysat.Instrument('pysat', 'testing', clean_level='clean')
self.bounds = (pysat.datetime(2008, 1, 1), pysat.datetime(2008, 1, 3))
def test_set_bounds_by_date_extra_time(self):
start = pysat.datetime(2009, 1, 1, 1, 10)
stop = pysat.datetime(2009, 1, 15, 1, 10)
self.testInst.bounds = (start, stop)
start = self.testInst._filter_datetime_input(start)
stop = self.testInst._filter_datetime_input(stop)
assert np.all(
self.testInst._iter_list == pds.date_range(start, stop).tolist())
def setup(self):
"""Runs before every method to create a clean testing setup."""
self.testInst = pysat.Instrument('pysat', 'testing2D',
clean_level='clean')
self.testInst.bounds = (pysat.datetime(2008, 1, 1),
pysat.datetime(2008, 1, 3))
self.dname = 'alt_profiles'
self.test_vals = np.arange(50) * 1.2
self.test_fracs = np.arange(50) / 50.0
def setup(self):
"""Runs before every method to create a clean testing setup."""
orbit_info = {'index': 'longitude', 'kind': 'longitude'}
self.testInst = pysat.Instrument('pysat',
'testing',
clean_level='clean',
orbit_info=orbit_info)
self.testInst.bounds = (pysat.datetime(2008, 1,
1), pysat.datetime(2008, 1, 31))
def test_basic_instrument_load_by_date_with_extra_time(self):
# put in a date that has more than year, month, day
date = pysat.datetime(2009, 1, 1, 1, 1, 1)
self.testInst.load(date=date)
test_date = self.testInst.index[0]
test_date = pysat.datetime(test_date.year, test_date.month,
test_date.day)
assert (test_date == pds.datetime(
2009, 1, 1)) & (test_date == self.testInst.date)
def test_iterate_over_bounds_set_by_date2(self):
start = pysat.datetime(2008, 1, 1)
stop = pysat.datetime(2010, 12, 31)
self.testInst.bounds = (start, stop)
dates = []
for inst in self.testInst:
dates.append(inst.date)
out = pds.date_range(start, stop).tolist()
assert np.all(dates == out)
def test_iterate_over_default_bounds(self):
start = pysat.datetime(2008,1,1)
stop = pysat.datetime(2010,12,31)
self.testInst.bounds = (start, stop)
dates = []
for inst in self.testInst:
dates.append(inst.date)
out = pds.date_range(start, stop).tolist()
assert np.all(dates == out)
def test_bin_data_type(self):
'''If a non array-like is given to median2D numpy.digitize should
raise an exception
'''
self.testInst.bounds = (pysat.datetime(2008, 1,
1), pysat.datetime(2008, 2, 1))
pysat.ssnl.avg.median2D(self.testInst, ['1', 'a', '23', '10'],
'longitude', ['0', 'd', '24', 'c'],
'mlt', ['dummy1', 'dummy2', 'dummy3'],
auto_bin=False)
def test_iterate_over_bounds_set_by_date_season(self):
start = [pysat.datetime(2009, 1, 1), pysat.datetime(2009, 2, 1)]
stop = [pysat.datetime(2009, 1, 15), pysat.datetime(2009, 2, 15)]
self.testInst.bounds = (start, stop)
dates = []
for inst in self.testInst:
dates.append(inst.date)
out = pds.date_range(start[0], stop[0]).tolist()
out.extend(pds.date_range(start[1], stop[1]).tolist())
assert np.all(dates == out)
def test_iterate_over_bounds_set_by_date_season(self):
start = [pysat.datetime(2009,1,1), pysat.datetime(2009,2,1)]
stop = [pysat.datetime(2009,1,15), pysat.datetime(2009,2,15)]
self.testInst.bounds = (start, stop)
dates = []
for inst in self.testInst:
dates.append(inst.date)
out = pds.date_range(start[0], stop[0]).tolist()
out.extend(pds.date_range(start[1], stop[1]).tolist())
assert np.all(dates == out)
def test_iterate_over_bounds_set_by_fname(self):
start = '2009-01-01.nofile'
stop = '2009-01-15.nofile'
start_d = pysat.datetime(2009,1,1)
stop_d = pysat.datetime(2009,1,15)
self.testInst.bounds = (start, stop)
dates = []
for inst in self.testInst:
dates.append(inst.date)
out = pds.date_range(start_d, stop_d).tolist()
assert np.all(dates == out)
def test_get_new_files_after_adding_files(self):
# create new files and make sure that new files are captured
start = pysat.datetime(2008, 1, 11)
stop = pysat.datetime(2008, 1, 12)
create_versioned_files(self.testInst, start, stop, freq='100min',
use_doy=False,
root_fname=self.root_fname)
dates = pysat.utils.season_date_range(start, stop, freq='100min')
new_files = self.testInst.files.get_new()
assert (np.all(new_files.index == dates))
def test_refresh(self):
# create new files and make sure that new files are captured
start = pysat.datetime(2008,1,10)
stop = pysat.datetime(2008,1,12)
create_files(self.testInst, start, stop, freq='100min',
use_doy=False,
root_fname = self.root_fname)
start = pysat.datetime(2007,12,31)
dates = pysat.utils.season_date_range(start, stop, freq='100min')
self.testInst.files.refresh()
assert (np.all(self.testInst.files.files.index == dates))
def test_iterate_over_bounds_set_by_fname_season(self):
start = ['2009-01-01.nofile', '2009-02-01.nofile']
stop = ['2009-01-15.nofile', '2009-02-15.nofile']
start_d = [pysat.datetime(2009,1,1), pysat.datetime(2009,2,1)]
stop_d = [pysat.datetime(2009,1,15), pysat.datetime(2009,2,15)]
self.testInst.bounds = (start, stop)
dates = []
for inst in self.testInst:
dates.append(inst.date)
out = pds.date_range(start_d[0], stop_d[0]).tolist()
out.extend(pds.date_range(start_d[1], stop_d[1]).tolist())
assert np.all(dates == out)
def setup(self):
'''Runs before every method to create a clean testing setup.'''
info = {'index':'mlt'}
self.testInst = pysat.Instrument('pysat','testing', '86400',
clean_level='clean',
orbit_info=info)
times = [ [pysat.datetime(2008,12,31,4), pysat.datetime(2008,12,31,5,37)],
[pysat.datetime(2009,1,1), pysat.datetime(2009,1,1,1,37)]
]
for seconds in np.arange(38):
day = pysat.datetime(2009,1,2)+pds.DateOffset(days=int(seconds))
times.append([day, day+pds.DateOffset(hours=1, minutes=37, seconds=int(seconds))-pds.DateOffset(seconds=20)])
self.testInst.custom.add(filter_data2, 'modify', times=times)
def test_refresh(self):
# create new files and make sure that new files are captured
# files slready exist from 2007, 12, 31 through to 10th
start = pysat.datetime(2008, 1, 10)
stop = pysat.datetime(2008, 1, 12)
create_versioned_files(self.testInst, start, stop, freq='100min',
use_doy=False,
root_fname=self.root_fname)
# create list of dates for all files that should be there
start = pysat.datetime(2007, 12, 31)
dates = pysat.utils.season_date_range(start, stop, freq='100min')
# update instrument file list
self.testInst.files.refresh()
assert (np.all(self.testInst.files.files.index == dates))
def test_iterate_over_bounds_set_by_fname_via_prev(self):
start = '2009-01-01.nofile'
stop = '2009-01-15.nofile'
start_d = pysat.datetime(2009,1,1)
stop_d = pysat.datetime(2009,1,15)
self.testInst.bounds = (start, stop)
dates = []
self.testInst.prev()
dates.append(self.testInst.date)
while self.testInst.date > start_d:
self.testInst.prev()
dates.append(self.testInst.date)
out = pds.date_range(start_d, stop_d).tolist()
assert np.all(dates == out[::-1])
def test_year_doy_files_direct_call_to_from_os(self):
# create a bunch of files by year and doy
start = pysat.datetime(2008,1,1)
stop = pysat.datetime(2009,12,31)
create_files(self.testInst, start, stop, freq='1D')
# use from_os function to get pandas Series of files and dates
files = pysat.Files.from_os(data_path=self.testInst.files.data_path,
format_str='pysat_testing_junk_{year:04d}_gold_{day:03d}_stuff.pysat_testing_file')
# check overall length
check1 = len(files) == (365+366)
# check specific dates
check2 = pds.to_datetime(files.index[0]) == pysat.datetime(2008,1,1)
check3 = pds.to_datetime(files.index[365]) == pysat.datetime(2008,12,31)
check4 = pds.to_datetime(files.index[-1]) == pysat.datetime(2009,12,31)
assert(check1 & check2 & check3 & check4)
def test_instrument_has_files(self):
import pysat.instruments.pysat_testing
root_fname='pysat_testing_junk_{year:04d}_gold_{day:03d}_stuff_{month:02d}_{hour:02d}_{min:02d}_{sec:02d}.pysat_testing_file'
# create a bunch of files by year and doy
start = pysat.datetime(2007,12,31)
stop = pysat.datetime(2008,1,10)
create_files(self.testInst, start, stop, freq='100min',
use_doy=False,
root_fname = root_fname)
# create the same range of dates
dates = pysat.utils.season_date_range(start, stop, freq='100min')
pysat.instruments.pysat_testing.list_files = list_files
inst = pysat.Instrument(platform='pysat', name='testing', update_files=True)
re_load(pysat.instruments.pysat_testing)
assert (np.all(inst.files.files.index == dates))
def test_next_filename_load_default(self):
"""Test next day is being loaded (checking object date)."""
self.testInst.load(fname='2010-12-30.nofile')
self.testInst.next()
test_date = self.testInst.data.index[0]
test_date = pysat.datetime(test_date.year, test_date.month, test_date.day)
assert (test_date == pds.datetime(2010,12,31)) & (test_date == self.testInst.date)
def test_prev_fid_load_default(self):
"""Test prev day is loaded when invoking .prev."""
self.testInst.load(fid=3)
self.testInst.prev()
test_date = self.testInst.data.index[0]
test_date = pysat.datetime(test_date.year, test_date.month, test_date.day)
assert (test_date == pds.datetime(2008,1,3)) & (test_date == self.testInst.date)
def test_next_fid_load_default(self):
"""Test next day is being loaded (checking object date)."""
self.testInst.load(fid=0)
self.testInst.next()
test_date = self.testInst.data.index[0]
test_date = pysat.datetime(test_date.year, test_date.month, test_date.day)
assert (test_date == pds.datetime(2008,1,2)) & (test_date == self.testInst.date)
def test_prev_filename_load_default(self):
"""Test prev day is loaded when invoking .prev."""
self.testInst.load(fname='2009-01-04.nofile')
# print(self.testInst.date)
self.testInst.prev()
test_date = self.testInst.data.index[0]
test_date = pysat.datetime(test_date.year, test_date.month, test_date.day)
assert (test_date == pds.datetime(2009,1,3)) & (test_date == self.testInst.date)
def test_year_month_day_hour_minute_second_files_direct_call_to_from_os(self):
root_fname='pysat_testing_junk_{year:04d}_gold_{day:03d}_stuff_{month:02d}_{hour:02d}_{min:02d}_{sec:02d}.pysat_testing_file'
# create a bunch of files by year and doy
start = pysat.datetime(2008,1,1)
stop = pysat.datetime(2008,1,3)
create_files(self.testInst, start, stop, freq='30s',
use_doy=False,
root_fname = root_fname)
# use from_os function to get pandas Series of files and dates
files = pysat.Files.from_os(data_path=self.testInst.files.data_path,
format_str=root_fname)
# check overall length
check1 = len(files) == 5761
# check specific dates
check2 = pds.to_datetime(files.index[0]) == pysat.datetime(2008,1,1)
check3 = pds.to_datetime(files.index[1]) == pysat.datetime(2008,1,1,0,0,30)
check4 = pds.to_datetime(files.index[-1]) == pysat.datetime(2008,1,3)
assert(check1 & check2 & check3 & check4)
def test_constellation_average(self):
for i in self.testC.instruments:
i.bounds = (pysat.datetime(2008,1,1), pysat.datetime(2008,2,1))
self.testI.bounds = (pysat.datetime(2008,1,1), pysat.datetime(2008,2,1))
resultsC = pysat.ssnl.avg.median2D(self.testC, [0., 360., 24.], 'longitude',
[0., 24, 24], 'mlt', ['dummy1', 'dummy2', 'dummy3'])
resultsI = pysat.ssnl.avg.median2D(self.testI, [0., 360., 24.], 'longitude',
[0., 24, 24], 'mlt', ['dummy1', 'dummy2', 'dummy3'])
medC1 = resultsC['dummy1']['median']
medI1 = resultsI['dummy1']['median']
medC2 = resultsC['dummy2']['median']
medI2 = resultsI['dummy2']['median']
medC3 = resultsC['dummy3']['median']
medI3 = resultsI['dummy3']['median']
assert np.array_equal(medC1, medI1)
assert np.array_equal(medC2, medI2)
assert np.array_equal(medC3, medI3)
def test_2D_avg(self):
for i in self.testC.instruments:
i.bounds = (pysat.datetime(2008,1,1), pysat.datetime(2008,2,1))
results = pysat.ssnl.avg.median2D(self.testC, [0., 360., 24.], 'mlt',
[0., 24, 24], 'slt', ['uts'])
dummy_val = results['uts']['median']
dummy_dev = results['uts']['avg_abs_dev']
dummy_x = results['uts']['bin_x']
dummy_y = results['uts']['bin_y']
# iterate over all y rows, value should be equal to integer value of mlt
# no variation in the median, all values should be the same
check = []
for i, y in enumerate(dummy_y[:-1]):
check.append(np.all(dummy_val[i, :] == y.astype(int)))
check.append(np.all(dummy_dev[i, :] == 0))
def test_basic_seasonal_average(self):
self.testInst.bounds = (pysat.datetime(2008, 1, 1), pysat.datetime(2008, 2, 1))
results = pysat.ssnl.avg.median2D(self.testInst, [0., 360., 24.], 'longitude',
[0., 24, 24], 'mlt', ['series_profiles'])
dummy_val = results['series_profiles']['median']
dummy_dev = results['series_profiles']['avg_abs_dev']
# iterate over all
# no variation in the median, all values should be the same
test_vals = np.arange(50)*1.2
for i, row in enumerate(dummy_val):
for j, item in enumerate(row):
assert np.all(item == test_vals)
for i, row in enumerate(dummy_dev):
for j, item in enumerate(row):
assert np.all(item == 0)
def getyrdoy(date):
"""Return a tuple of year, day of year for a supplied datetime object."""
try:
doy = date.toordinal()-datetime(date.year,1,1).toordinal()+1
except AttributeError:
raise AttributeError("Must supply a pandas datetime object or " +
"equivalent")
else:
return date.year, doy
def test_files_when_duplicates_forced(self):
# create new files and make sure that new files are captured
start = pysat.datetime(2008, 1, 11)
stop = pysat.datetime(2008, 1, 15)
dates = pysat.utils.season_date_range(start, stop, freq='1D')
# clear out old files, create new ones
remove_files(self.testInst)
create_versioned_files(self.testInst, start, stop, freq='1D',
use_doy=False,
root_fname='pysat_testing_unique_{version:02d}_{revision:03d}_{year:04d}_g_{day:03d}_st.pysat_testing_file')
pysat.instruments.pysat_testing.list_files = list_files
self.testInst = pysat.Instrument(inst_module=pysat.instruments.pysat_testing,
clean_level='clean',
file_format='pysat_testing_unique_??_???_{year:04d}_g_{day:03d}_st.pysat_testing_file',
update_files=True,
temporary_file_list=self.temporary_file_list)
assert (np.all(self.testInst.files.files.index == dates))
def test_all_single_orbit_calls_in_day(self):
self.testInst.load(2009,1)
ans = []; ans2=[];
self.testInst.bounds= (pysat.datetime(2009,1,1), None)
for i,inst in enumerate(self.testInst.orbits):
if i > 14:
break
ans.append(self.testInst.data.index[0] == (pds.datetime(2009,1,1)+i*relativedelta(hours=1, minutes=37)))
ans2.append(self.testInst.data.index[-1] == (pds.datetime(2009,1,1)+(i+1)*relativedelta(hours=1, minutes=37) -
relativedelta(seconds=1) ))
assert np.all(ans) & np.all(ans2)
