def test_files_when_duplicates_forced(self): # create new files and make sure that new files are captured start = pysat.datetime(2008, 1, 11) stop = pysat.datetime(2008, 1, 15) dates = pysat.utils.time.create_date_range(start, stop, freq='1D') # clear out old files, create new ones remove_files(self.testInst) create_versioned_files(self.testInst, start, stop, freq='1D', use_doy=False, root_fname=''.join(('pysat_testing_unique_', '{version:02d}_', '{revision:03d}_{year:04d}', '_g_{day:03d}_st.pysat_', 'testing_file'))) pysat.instruments.pysat_testing.list_files = list_files self.testInst = \ pysat.Instrument(inst_module=pysat.instruments.pysat_testing, clean_level='clean', file_format=''.join(('pysat_testing_unique_??_', '???_{year:04d}_g_{day:03d}', '_st.pysat_testing_file')), update_files=True, temporary_file_list=self.temporary_file_list) assert (np.all(self.testInst.files.files.index == dates))
def test_set_bounds_by_date_season(self): start = [pysat.datetime(2009, 1, 1), pysat.datetime(2009, 2, 1)] stop = [pysat.datetime(2009, 1, 15), pysat.datetime(2009, 2, 15)] self.testInst.bounds = (start, stop) out = pds.date_range(start[0], stop[0]).tolist() out.extend(pds.date_range(start[1], stop[1]).tolist()) assert np.all(self.testInst._iter_list == out)
def create_versioned_files(inst, start=None, stop=None, freq='1D', use_doy=True, root_fname=None): # create a bunch of files if start is None: start = pysat.datetime(2009, 1, 1) if stop is None: stop = pysat.datetime(2013, 12, 31) dates = pysat.utils.season_date_range(start, stop, freq=freq) versions = np.array([1, 2]) revisions = np.array([0, 1]) if root_fname is None: root_fname = 'pysat_testing_junk_{year:04d}_{month:02d}_{day:03d}{hour:02d}{min:02d}{sec:02d}_stuff_{version:02d}_{revision:03d}.pysat_testing_file' # create empty file for date in dates: for version in versions: for revision in revisions: yr, doy = pysat.utils.getyrdoy(date) if use_doy: doy = doy else: doy = date.day fname = os.path.join(inst.files.data_path, root_fname.format(year=yr, day=doy, month=date.month, hour=date.hour, min=date.minute, sec=date.second, version=version, revision=revision)) with open(fname, 'w') as f: pass
def test_year_month_day_hour_files_direct_call_to_from_os(self): # create a bunch of files by year and doy start = pysat.datetime(2008, 1, 1) stop = pysat.datetime(2009, 12, 31) create_files( self.testInst, start, stop, freq='6h', use_doy=False, root_fname= 'pysat_testing_junk_{year:04d}_gold_{day:03d}_stuff_{month:02d}_{hour:02d}.pysat_testing_file' ) # use from_os function to get pandas Series of files and dates files = pysat.Files.from_os( data_path=self.testInst.files.data_path, format_str= 'pysat_testing_junk_{year:04d}_gold_{day:03d}_stuff_{month:02d}_{hour:02d}.pysat_testing_file' ) # check overall length check1 = len(files) == (365 + 366) * 4 - 3 # check specific dates check2 = files.index[0].to_datetime() == pysat.datetime(2008, 1, 1) check3 = files.index[1460].to_datetime() == pysat.datetime( 2008, 12, 31) check4 = files.index[-1].to_datetime() == pysat.datetime(2009, 12, 31) assert (check1 & check2 & check3 & check4)
def load(fnames, tag=None, sat_id=None): # create an artifical satellite data set parts = fnames[0].split('/') yr = int('20' + parts[-1][0:2]) month = int(parts[-3]) day = int(parts[-2]) date = pysat.datetime(yr, month, day) num = 86400 #int(tag) uts = np.arange(num) data = pysat.DataFrame(uts, columns=['uts']) # need to create simple orbits here. Have start of first orbit # at 2009,1, 0 UT. 14.84 orbits per day time_delta = date - pysat.datetime(2009, 1, 1) uts_root = np.mod(time_delta.total_seconds(), 5820) mlt = np.mod(uts_root + np.arange(num), 5820) * (24. / 5820.) data['mlt'] = mlt # do slt, 20 second offset from mlt uts_root = np.mod(time_delta.total_seconds() + 20, 5820) data['slt'] = np.mod(uts_root + np.arange(num), 5820) * (24. / 5820.) index = pds.date_range(date, date + pds.DateOffset(hours=23, minutes=59, seconds=59), freq='S') data.index = index data.index.name = 'time' return data, meta.copy()
def list_files(tag=None, sat_id=None, data_path=None, format_str=None): """Produce a fake list of files spanning a year""" index = pds.date_range(pysat.datetime(2008, 1, 1), pysat.datetime(2010, 12, 31)) names = [data_path + date.strftime('%D') + '.nofile' for date in index] return pysat.Series(names, index=index)
def test_basic_orbit_mean(self): orbit_info = {'kind':'local time', 'index':'mlt'} self.testInst = pysat.Instrument('pysat','testing', clean_level='clean', orbit_info=orbit_info) self.testInst.bounds = (pysat.datetime(2009,1,1), pysat.datetime(2009,1,2)) ans = pysat.ssnl.avg.mean_by_orbit(self.testInst, 'mlt') # note last orbit is incomplete thus not expected to satisfy relation assert np.allclose(ans[:-1], np.ones(len(ans)-1)*12., 1.E-2)
def test_set_bounds_by_date_season(self): start = [pysat.datetime(2009,1,1), pysat.datetime(2009,2,1)] stop = [pysat.datetime(2009,1,15), pysat.datetime(2009,2,15)] self.testInst.bounds = (start, stop) out = pds.date_range(start[0], stop[0]).tolist() out.extend(pds.date_range(start[1], stop[1]).tolist()) assert np.all(self.testInst._iter_list == out)
def test_heterogenous_constellation_average(self): for inst in self.testC: inst.bounds = (pysat.datetime(2008,1,1), pysat.datetime(2008,2,1)) results = pysat.ssnl.avg.median2D(self.testC, [0., 360., 24.], 'longitude', [0., 24, 24], 'mlt', ['dummy1', 'dummy2', 'dummy3']) dummy_val = results['dummy1']['median'] dummy_dev = results['dummy1']['avg_abs_dev'] dummy2_val = results['dummy2']['median'] dummy2_dev = results['dummy2']['avg_abs_dev'] dummy3_val = results['dummy3']['median'] dummy3_dev = results['dummy3']['avg_abs_dev'] dummy_x = results['dummy1']['bin_x'] dummy_y = results['dummy1']['bin_y'] # iterate over all y rows, value should be equal to integer value of mlt # no variation in the median, all values should be the same check = [] for i, y in enumerate(dummy_y[:-1]): check.append(np.all(dummy_val[i, :] == y.astype(int))) check.append(np.all(dummy_dev[i, :] == 0)) for i, x in enumerate(dummy_x[:-1]): check.append(np.all(dummy2_val[:, i] == x/15.) ) check.append(np.all(dummy2_dev[:, i] == 0)) for i, x in enumerate(dummy_x[:-1]): check.append(np.all(dummy3_val[:, i] == x/15.*1000. + dummy_y[:-1]) ) check.append(np.all(dummy3_dev[:, i] == 0)) assert np.all(check)
def setup(self): """Runs before every method to create a clean testing setup.""" self.testInst = pysat.Instrument(platform='pysat', name='testing', clean_level='clean') self.start = pysat.datetime(2009, 1, 1) self.stop = pysat.datetime(2009, 1, 1)
def test_year_month_files_direct_call_to_from_os(self): # create a bunch of files by year and doy start = pysat.datetime(2008, 1, 1) stop = pysat.datetime(2009, 12, 31) create_files( self.testInst, start, stop, freq='1MS', root_fname= 'pysat_testing_junk_{year:04d}_gold_stuff_{month:02d}.pysat_testing_file' ) # use from_os function to get pandas Series of files and dates files = pysat.Files.from_os( data_path=self.testInst.files.data_path, format_str= 'pysat_testing_junk_{year:04d}_gold_stuff_{month:02d}.pysat_testing_file' ) # check overall length check1 = len(files) == 24 # check specific dates check2 = pds.to_datetime(files.index[0]) == pysat.datetime(2008, 1, 1) check3 = pds.to_datetime(files.index[11]) == pysat.datetime( 2008, 12, 1) check4 = pds.to_datetime(files.index[-1]) == pysat.datetime( 2009, 12, 1) assert (check1 & check2 & check3 & check4)
def test_basic_file_mean(self): index = pds.date_range(pysat.datetime(2008, 1, 1), pysat.datetime(2008, 2, 1)) names = [date.strftime('%D') + '.nofile' for date in index] self.testInst.bounds = (names[0], names[-1]) ans = pysat.ssnl.avg.mean_by_file(self.testInst, 'dummy4') assert np.all(ans == 86399 / 2.)
def create_files(inst, start=None, stop=None, freq='1D', use_doy=True, root_fname=None): # create a bunch of files if start is None: start = pysat.datetime(2009, 1, 1) if stop is None: stop = pysat.datetime(2013, 12, 31) dates = pysat.utils.season_date_range(start, stop, freq=freq) if root_fname is None: root_fname = 'pysat_testing_junk_{year:04d}_gold_{day:03d}_stuff.pysat_testing_file' # create empty file for date in dates: yr, doy = pysat.utils.getyrdoy(date) if use_doy: doy = doy else: doy = date.day fname = os.path.join( inst.files.data_path, root_fname.format(year=yr, day=doy, month=date.month, hour=date.hour, min=date.minute, sec=date.second)) with open(fname, 'w') as f: pass #f.close()
def test_get_new_files_after_deleting_files_and_adding_files(self): # create new files and make sure that new files are captured start = pysat.datetime(2008, 1, 11) stop = pysat.datetime(2008, 1, 12) dates = pysat.utils.season_date_range(start, stop, freq='100min') # remove files, same number as will be added to_be_removed = len(dates) for the_file in os.listdir(self.testInst.files.data_path): if (the_file[0:13] == 'pysat_testing') & (the_file[-19:] == '.pysat_testing_file'): file_path = os.path.join(self.testInst.files.data_path, the_file) if os.path.isfile(file_path) & (to_be_removed > 0): to_be_removed -= 1 # Remove all versions of the file # otherwise, previous versions will look like new files pattern = '_'.join(file_path.split('_')[0:7])+'*.pysat_testing_file' map(os.unlink, glob.glob(pattern)) #os.unlink(file_path) # add new files create_versioned_files(self.testInst, start, stop, freq='100min', use_doy=False, root_fname=self.root_fname) # get new files new_files = self.testInst.files.get_new() assert (np.all(new_files.index == dates))
def test_files_non_standard_pysat_directory(self): # create new files and make sure that new files are captured start = pysat.datetime(2008, 1, 11) stop = pysat.datetime(2008, 1, 15) dates = pysat.utils.season_date_range(start, stop, freq='100min') pysat.instruments.pysat_testing.list_files = list_versioned_files self.testInst = pysat.Instrument(inst_module=pysat.instruments.pysat_testing, clean_level='clean', sat_id='hello', directory_format='pysat_testing_{tag}_{sat_id}', update_files=True, temporary_file_list=self.temporary_file_list) # add new files create_dir(self.testInst) remove_files(self.testInst) create_versioned_files(self.testInst, start, stop, freq='100min', use_doy=False, root_fname=self.root_fname) self.testInst = pysat.Instrument(inst_module=pysat.instruments.pysat_testing, clean_level='clean', sat_id='hello', directory_format='pysat_testing_{tag}_{sat_id}', update_files=True, temporary_file_list=self.temporary_file_list) # get new files new_files = self.testInst.files.get_new() assert (np.all(self.testInst.files.files.index == dates) & np.all(new_files.index == dates))
def setup(self): """Runs before every method to create a clean testing setup.""" # store current pysat directory self.data_path = pysat.data_dir # create temporary directory dir_name = tempfile.gettempdir() pysat.utils.set_data_dir(dir_name, store=False) # create testing directory create_dir(temporary_file_list=self.temporary_file_list) # create a test instrument, make sure it is getting files from filesystem re_load(pysat.instruments.pysat_testing) # self.stored_files_fcn = pysat.instruments.pysat_testing.list_files pysat.instruments.pysat_testing.list_files = list_versioned_files # create a bunch of files by year and doy self.testInst = pysat.Instrument( inst_module=pysat.instruments.pysat_testing, clean_level='clean', temporary_file_list=self.temporary_file_list) self.root_fname = 'pysat_testing_junk_{year:04d}_{month:02d}_{day:03d}{hour:02d}{min:02d}{sec:02d}_stuff_{version:02d}_{revision:03d}.pysat_testing_file' start = pysat.datetime(2007, 12, 31) stop = pysat.datetime(2008, 1, 10) create_versioned_files(self.testInst, start, stop, freq='100min', use_doy=False, root_fname=self.root_fname) self.testInst = pysat.Instrument( inst_module=pysat.instruments.pysat_testing, clean_level='clean', update_files=True, temporary_file_list=self.temporary_file_list)
def test_get_new_files_after_deleting_files_and_adding_files(self): # create new files and make sure that new files are captured start = pysat.datetime(2008, 1, 11) stop = pysat.datetime(2008, 1, 12) dates = pysat.utils.season_date_range(start, stop, freq='100min') # remove files, same number as will be added to_be_removed = len(dates) for the_file in os.listdir(self.testInst.files.data_path): if (the_file[0:13] == 'pysat_testing') & ( the_file[-19:] == '.pysat_testing_file'): file_path = os.path.join(self.testInst.files.data_path, the_file) if os.path.isfile(file_path) & (to_be_removed > 0): to_be_removed -= 1 # Remove all versions of the file # otherwise, previous versions will look like new files pattern = '_'.join( file_path.split('_')[0:7]) + '*.pysat_testing_file' map(os.unlink, glob.glob(pattern)) #os.unlink(file_path) # add new files create_versioned_files(self.testInst, start, stop, freq='100min', use_doy=False, root_fname=self.root_fname) # get new files new_files = self.testInst.files.get_new() assert (np.all(new_files.index == dates))
def test_refresh_on_unchanged_files(self): start = pysat.datetime(2007, 12, 31) stop = pysat.datetime(2008, 1, 10) dates = pysat.utils.season_date_range(start, stop, freq='100min') self.testInst.files.refresh() assert (np.all(self.testInst.files.files.index == dates))
def test_get_new_files_after_adding_files_and_adding_file(self): # create new files and make sure that new files are captured start = pysat.datetime(2008, 1, 11) stop = pysat.datetime(2008, 1, 12) create_versioned_files(self.testInst, start, stop, freq='100min', use_doy=False, root_fname=self.root_fname) dates = pysat.utils.season_date_range(start, stop, freq='100min') new_files = self.testInst.files.get_new() start = pysat.datetime(2008, 1, 15) stop = pysat.datetime(2008, 1, 18) create_versioned_files(self.testInst, start, stop, freq='100min', use_doy=False, root_fname=self.root_fname) dates2 = pysat.utils.season_date_range(start, stop, freq='100min') new_files2 = self.testInst.files.get_new() assert (np.all(new_files.index == dates) & np.all(new_files2.index == dates2))
def test_files_non_standard_file_format_template(self): # create new files and make sure that new files are captured start = pysat.datetime(2008, 1, 11) stop = pysat.datetime(2008, 1, 15) dates = pysat.utils.season_date_range(start, stop, freq='1D') # clear out old files, create new ones remove_files(self.testInst) create_files( self.testInst, start, stop, freq='1D', use_doy=False, root_fname= 'pysat_testing_unique_junk_{year:04d}_gold_{day:03d}_stuff.pysat_testing_file' ) pysat.instruments.pysat_testing.list_files = list_files self.testInst = pysat.Instrument( inst_module=pysat.instruments.pysat_testing, clean_level='clean', file_format= 'pysat_testing_unique_junk_{year:04d}_gold_{day:03d}_stuff.pysat_testing_file', update_files=True, temporary_file_list=self.temporary_file_list) assert (np.all(self.testInst.files.files.index == dates))
def setup(self): """Runs before every method to create a clean testing setup.""" self.testInst = pysat.Instrument('pysat', 'testing2D', clean_level='clean') self.testInst.bounds = (pysat.datetime(2008, 1, 1), pysat.datetime(2008, 2, 1)) self.dname = 'series_profiles'
def test_files_non_standard_pysat_directory(self): # create new files and make sure that new files are captured start = pysat.datetime(2008, 1, 11) stop = pysat.datetime(2008, 1, 15) dates = pysat.utils.season_date_range(start, stop, freq='100min') self.testInst = pysat.Instrument( inst_module=pysat.instruments.pysat_testing, clean_level='clean', sat_id='hello', directory_format='pysat_testing_{tag}_{sat_id}', update_files=True, temporary_file_list=self.temporary_file_list) # add new files create_dir(self.testInst) remove_files(self.testInst) create_files(self.testInst, start, stop, freq='100min', use_doy=False, root_fname=self.root_fname) self.testInst = pysat.Instrument( inst_module=pysat.instruments.pysat_testing, clean_level='clean', sat_id='hello', directory_format='pysat_testing_{tag}_{sat_id}', update_files=True, temporary_file_list=self.temporary_file_list) # get new files new_files = self.testInst.files.get_new() assert (np.all(self.testInst.files.files.index == dates) & np.all(new_files.index == dates))
def setup(self): """Runs before every method to create a clean testing setup.""" self.testInst = pysat.Instrument('pysat', 'testing', clean_level='clean') self.testInst.bounds = (pysat.datetime(2008, 1, 1), pysat.datetime(2008, 1, 3))
def test_get_new_files_after_deleting_files_and_adding_files(self): # create new files and make sure that new files are captured start = pysat.datetime(2008, 1, 11) stop = pysat.datetime(2008, 1, 12) dates = pysat.utils.season_date_range(start, stop, freq='100min') # remove files, same number as will be added to_be_removed = len(dates) for the_file in os.listdir(self.testInst.files.data_path): if (the_file[0:13] == 'pysat_testing') & ( the_file[-19:] == '.pysat_testing_file'): file_path = os.path.join(self.testInst.files.data_path, the_file) if os.path.isfile(file_path) & (to_be_removed > 0): #print(file_path) to_be_removed -= 1 os.unlink(file_path) # add new files create_files(self.testInst, start, stop, freq='100min', use_doy=False, root_fname=self.root_fname) # get new files new_files = self.testInst.files.get_new() assert (np.all(new_files.index == dates))
def setup(self): """Runs before every method to create a clean testing setup.""" # store current pysat directory self.data_path = pysat.data_dir # create temporary directory dir_name = tempfile.gettempdir() pysat.utils.set_data_dir(dir_name, store=False) # create testing directory create_dir(temporary_file_list=self.temporary_file_list) # create a test instrument, make sure it is getting files from filesystem re_load(pysat.instruments.pysat_testing) # self.stored_files_fcn = pysat.instruments.pysat_testing.list_files pysat.instruments.pysat_testing.list_files = list_versioned_files # create a bunch of files by year and doy self.testInst = pysat.Instrument(inst_module=pysat.instruments.pysat_testing, clean_level='clean', temporary_file_list=self.temporary_file_list) self.root_fname = 'pysat_testing_junk_{year:04d}_{month:02d}_{day:03d}{hour:02d}{min:02d}{sec:02d}_stuff_{version:02d}_{revision:03d}.pysat_testing_file' start = pysat.datetime(2007, 12, 31) stop = pysat.datetime(2008, 1, 10) create_versioned_files(self.testInst, start, stop, freq='100min', use_doy=False, root_fname=self.root_fname) self.testInst = pysat.Instrument(inst_module=pysat.instruments.pysat_testing, clean_level='clean', update_files=True, temporary_file_list=self.temporary_file_list)
def setup(self): """Runs before every method to create a clean testing setup""" self.testInst = pysat.Instrument('pysat', 'testing', clean_level='clean', update_files=True) self.testInst.bounds = (pysat.datetime(2008, 1, 1), pysat.datetime(2008, 1, 31)) self.test_bins = [0, 24, 24] self.test_label = 'slt' self.test_data = ['dummy1', 'dummy2'] self.out_keys = ['count', 'avg_abs_dev', 'median', 'bin_x'] self.out_data = {'dummy1': {'count': [111780., 111320., 111780., 111320., 111780., 111320., 111780., 111320., 111780., 111320., 111780., 111320., 111780., 111320., 111918., 111562., 112023., 111562., 112023., 111412., 111780., 111320., 111780., 111320.], 'avg_abs_dev': np.zeros(shape=24), 'median': np.linspace(0.0, 23.0, 24.0)}, 'dummy2': {'count': [111780., 111320., 111780., 111320., 111780., 111320., 111780., 111320., 111780., 111320., 111780., 111320., 111780., 111320., 111918., 111562., 112023., 111562., 112023., 111412., 111780., 111320., 111780., 111320.], 'avg_abs_dev': np.zeros(shape=24) + 6.0, 'median': [11., 12., 11., 11., 12., 11., 12., 11., 12., 12., 11., 12., 11., 12., 11., 11., 12., 11., 12., 11., 11., 11., 11., 12.]}}
def test_basic_instrument_load_by_date(self): date = pysat.datetime(2009, 1, 1) self.testInst.load(date=date) test_date = self.testInst.index[0] test_date = pysat.datetime(test_date.year, test_date.month, test_date.day) assert (test_date == pds.datetime( 2009, 1, 1)) & (test_date == self.testInst.date)
def setup(self): insts = [] for i in range(5): insts.append(pysat.Instrument('pysat', 'testing', clean_level='clean')) self.testC = pysat.Constellation(instruments=insts) self.testI = pysat.Instrument('pysat', 'testing', clean_level='clean') self.bounds = (pysat.datetime(2008, 1, 1), pysat.datetime(2008, 1, 3))
def test_set_bounds_by_date_extra_time(self): start = pysat.datetime(2009, 1, 1, 1, 10) stop = pysat.datetime(2009, 1, 15, 1, 10) self.testInst.bounds = (start, stop) start = self.testInst._filter_datetime_input(start) stop = self.testInst._filter_datetime_input(stop) assert np.all( self.testInst._iter_list == pds.date_range(start, stop).tolist())
def setup(self): """Runs before every method to create a clean testing setup.""" self.testInst = pysat.Instrument('pysat', 'testing2D', clean_level='clean') self.testInst.bounds = (pysat.datetime(2008, 1, 1), pysat.datetime(2008, 1, 3)) self.dname = 'alt_profiles' self.test_vals = np.arange(50) * 1.2 self.test_fracs = np.arange(50) / 50.0
def setup(self): """Runs before every method to create a clean testing setup.""" orbit_info = {'index': 'longitude', 'kind': 'longitude'} self.testInst = pysat.Instrument('pysat', 'testing', clean_level='clean', orbit_info=orbit_info) self.testInst.bounds = (pysat.datetime(2008, 1, 1), pysat.datetime(2008, 1, 31))
def test_basic_instrument_load_by_date_with_extra_time(self): # put in a date that has more than year, month, day date = pysat.datetime(2009, 1, 1, 1, 1, 1) self.testInst.load(date=date) test_date = self.testInst.index[0] test_date = pysat.datetime(test_date.year, test_date.month, test_date.day) assert (test_date == pds.datetime( 2009, 1, 1)) & (test_date == self.testInst.date)
def test_iterate_over_bounds_set_by_date2(self): start = pysat.datetime(2008, 1, 1) stop = pysat.datetime(2010, 12, 31) self.testInst.bounds = (start, stop) dates = [] for inst in self.testInst: dates.append(inst.date) out = pds.date_range(start, stop).tolist() assert np.all(dates == out)
def test_iterate_over_default_bounds(self): start = pysat.datetime(2008,1,1) stop = pysat.datetime(2010,12,31) self.testInst.bounds = (start, stop) dates = [] for inst in self.testInst: dates.append(inst.date) out = pds.date_range(start, stop).tolist() assert np.all(dates == out)
def test_bin_data_type(self): '''If a non array-like is given to median2D numpy.digitize should raise an exception ''' self.testInst.bounds = (pysat.datetime(2008, 1, 1), pysat.datetime(2008, 2, 1)) pysat.ssnl.avg.median2D(self.testInst, ['1', 'a', '23', '10'], 'longitude', ['0', 'd', '24', 'c'], 'mlt', ['dummy1', 'dummy2', 'dummy3'], auto_bin=False)
def test_iterate_over_bounds_set_by_date_season(self): start = [pysat.datetime(2009, 1, 1), pysat.datetime(2009, 2, 1)] stop = [pysat.datetime(2009, 1, 15), pysat.datetime(2009, 2, 15)] self.testInst.bounds = (start, stop) dates = [] for inst in self.testInst: dates.append(inst.date) out = pds.date_range(start[0], stop[0]).tolist() out.extend(pds.date_range(start[1], stop[1]).tolist()) assert np.all(dates == out)
def test_iterate_over_bounds_set_by_date_season(self): start = [pysat.datetime(2009,1,1), pysat.datetime(2009,2,1)] stop = [pysat.datetime(2009,1,15), pysat.datetime(2009,2,15)] self.testInst.bounds = (start, stop) dates = [] for inst in self.testInst: dates.append(inst.date) out = pds.date_range(start[0], stop[0]).tolist() out.extend(pds.date_range(start[1], stop[1]).tolist()) assert np.all(dates == out)
def test_iterate_over_bounds_set_by_fname(self): start = '2009-01-01.nofile' stop = '2009-01-15.nofile' start_d = pysat.datetime(2009,1,1) stop_d = pysat.datetime(2009,1,15) self.testInst.bounds = (start, stop) dates = [] for inst in self.testInst: dates.append(inst.date) out = pds.date_range(start_d, stop_d).tolist() assert np.all(dates == out)
def test_get_new_files_after_adding_files(self): # create new files and make sure that new files are captured start = pysat.datetime(2008, 1, 11) stop = pysat.datetime(2008, 1, 12) create_versioned_files(self.testInst, start, stop, freq='100min', use_doy=False, root_fname=self.root_fname) dates = pysat.utils.season_date_range(start, stop, freq='100min') new_files = self.testInst.files.get_new() assert (np.all(new_files.index == dates))
def test_refresh(self): # create new files and make sure that new files are captured start = pysat.datetime(2008,1,10) stop = pysat.datetime(2008,1,12) create_files(self.testInst, start, stop, freq='100min', use_doy=False, root_fname = self.root_fname) start = pysat.datetime(2007,12,31) dates = pysat.utils.season_date_range(start, stop, freq='100min') self.testInst.files.refresh() assert (np.all(self.testInst.files.files.index == dates))
def test_iterate_over_bounds_set_by_fname_season(self): start = ['2009-01-01.nofile', '2009-02-01.nofile'] stop = ['2009-01-15.nofile', '2009-02-15.nofile'] start_d = [pysat.datetime(2009,1,1), pysat.datetime(2009,2,1)] stop_d = [pysat.datetime(2009,1,15), pysat.datetime(2009,2,15)] self.testInst.bounds = (start, stop) dates = [] for inst in self.testInst: dates.append(inst.date) out = pds.date_range(start_d[0], stop_d[0]).tolist() out.extend(pds.date_range(start_d[1], stop_d[1]).tolist()) assert np.all(dates == out)
def setup(self): '''Runs before every method to create a clean testing setup.''' info = {'index':'mlt'} self.testInst = pysat.Instrument('pysat','testing', '86400', clean_level='clean', orbit_info=info) times = [ [pysat.datetime(2008,12,31,4), pysat.datetime(2008,12,31,5,37)], [pysat.datetime(2009,1,1), pysat.datetime(2009,1,1,1,37)] ] for seconds in np.arange(38): day = pysat.datetime(2009,1,2)+pds.DateOffset(days=int(seconds)) times.append([day, day+pds.DateOffset(hours=1, minutes=37, seconds=int(seconds))-pds.DateOffset(seconds=20)]) self.testInst.custom.add(filter_data2, 'modify', times=times)
def test_refresh(self): # create new files and make sure that new files are captured # files slready exist from 2007, 12, 31 through to 10th start = pysat.datetime(2008, 1, 10) stop = pysat.datetime(2008, 1, 12) create_versioned_files(self.testInst, start, stop, freq='100min', use_doy=False, root_fname=self.root_fname) # create list of dates for all files that should be there start = pysat.datetime(2007, 12, 31) dates = pysat.utils.season_date_range(start, stop, freq='100min') # update instrument file list self.testInst.files.refresh() assert (np.all(self.testInst.files.files.index == dates))
def test_iterate_over_bounds_set_by_fname_via_prev(self): start = '2009-01-01.nofile' stop = '2009-01-15.nofile' start_d = pysat.datetime(2009,1,1) stop_d = pysat.datetime(2009,1,15) self.testInst.bounds = (start, stop) dates = [] self.testInst.prev() dates.append(self.testInst.date) while self.testInst.date > start_d: self.testInst.prev() dates.append(self.testInst.date) out = pds.date_range(start_d, stop_d).tolist() assert np.all(dates == out[::-1])
def test_year_doy_files_direct_call_to_from_os(self): # create a bunch of files by year and doy start = pysat.datetime(2008,1,1) stop = pysat.datetime(2009,12,31) create_files(self.testInst, start, stop, freq='1D') # use from_os function to get pandas Series of files and dates files = pysat.Files.from_os(data_path=self.testInst.files.data_path, format_str='pysat_testing_junk_{year:04d}_gold_{day:03d}_stuff.pysat_testing_file') # check overall length check1 = len(files) == (365+366) # check specific dates check2 = pds.to_datetime(files.index[0]) == pysat.datetime(2008,1,1) check3 = pds.to_datetime(files.index[365]) == pysat.datetime(2008,12,31) check4 = pds.to_datetime(files.index[-1]) == pysat.datetime(2009,12,31) assert(check1 & check2 & check3 & check4)
def test_instrument_has_files(self): import pysat.instruments.pysat_testing root_fname='pysat_testing_junk_{year:04d}_gold_{day:03d}_stuff_{month:02d}_{hour:02d}_{min:02d}_{sec:02d}.pysat_testing_file' # create a bunch of files by year and doy start = pysat.datetime(2007,12,31) stop = pysat.datetime(2008,1,10) create_files(self.testInst, start, stop, freq='100min', use_doy=False, root_fname = root_fname) # create the same range of dates dates = pysat.utils.season_date_range(start, stop, freq='100min') pysat.instruments.pysat_testing.list_files = list_files inst = pysat.Instrument(platform='pysat', name='testing', update_files=True) re_load(pysat.instruments.pysat_testing) assert (np.all(inst.files.files.index == dates))
def test_next_filename_load_default(self): """Test next day is being loaded (checking object date).""" self.testInst.load(fname='2010-12-30.nofile') self.testInst.next() test_date = self.testInst.data.index[0] test_date = pysat.datetime(test_date.year, test_date.month, test_date.day) assert (test_date == pds.datetime(2010,12,31)) & (test_date == self.testInst.date)
def test_prev_fid_load_default(self): """Test prev day is loaded when invoking .prev.""" self.testInst.load(fid=3) self.testInst.prev() test_date = self.testInst.data.index[0] test_date = pysat.datetime(test_date.year, test_date.month, test_date.day) assert (test_date == pds.datetime(2008,1,3)) & (test_date == self.testInst.date)
def test_next_fid_load_default(self): """Test next day is being loaded (checking object date).""" self.testInst.load(fid=0) self.testInst.next() test_date = self.testInst.data.index[0] test_date = pysat.datetime(test_date.year, test_date.month, test_date.day) assert (test_date == pds.datetime(2008,1,2)) & (test_date == self.testInst.date)
def test_prev_filename_load_default(self): """Test prev day is loaded when invoking .prev.""" self.testInst.load(fname='2009-01-04.nofile') # print(self.testInst.date) self.testInst.prev() test_date = self.testInst.data.index[0] test_date = pysat.datetime(test_date.year, test_date.month, test_date.day) assert (test_date == pds.datetime(2009,1,3)) & (test_date == self.testInst.date)
def test_year_month_day_hour_minute_second_files_direct_call_to_from_os(self): root_fname='pysat_testing_junk_{year:04d}_gold_{day:03d}_stuff_{month:02d}_{hour:02d}_{min:02d}_{sec:02d}.pysat_testing_file' # create a bunch of files by year and doy start = pysat.datetime(2008,1,1) stop = pysat.datetime(2008,1,3) create_files(self.testInst, start, stop, freq='30s', use_doy=False, root_fname = root_fname) # use from_os function to get pandas Series of files and dates files = pysat.Files.from_os(data_path=self.testInst.files.data_path, format_str=root_fname) # check overall length check1 = len(files) == 5761 # check specific dates check2 = pds.to_datetime(files.index[0]) == pysat.datetime(2008,1,1) check3 = pds.to_datetime(files.index[1]) == pysat.datetime(2008,1,1,0,0,30) check4 = pds.to_datetime(files.index[-1]) == pysat.datetime(2008,1,3) assert(check1 & check2 & check3 & check4)
def test_constellation_average(self): for i in self.testC.instruments: i.bounds = (pysat.datetime(2008,1,1), pysat.datetime(2008,2,1)) self.testI.bounds = (pysat.datetime(2008,1,1), pysat.datetime(2008,2,1)) resultsC = pysat.ssnl.avg.median2D(self.testC, [0., 360., 24.], 'longitude', [0., 24, 24], 'mlt', ['dummy1', 'dummy2', 'dummy3']) resultsI = pysat.ssnl.avg.median2D(self.testI, [0., 360., 24.], 'longitude', [0., 24, 24], 'mlt', ['dummy1', 'dummy2', 'dummy3']) medC1 = resultsC['dummy1']['median'] medI1 = resultsI['dummy1']['median'] medC2 = resultsC['dummy2']['median'] medI2 = resultsI['dummy2']['median'] medC3 = resultsC['dummy3']['median'] medI3 = resultsI['dummy3']['median'] assert np.array_equal(medC1, medI1) assert np.array_equal(medC2, medI2) assert np.array_equal(medC3, medI3)
def test_2D_avg(self): for i in self.testC.instruments: i.bounds = (pysat.datetime(2008,1,1), pysat.datetime(2008,2,1)) results = pysat.ssnl.avg.median2D(self.testC, [0., 360., 24.], 'mlt', [0., 24, 24], 'slt', ['uts']) dummy_val = results['uts']['median'] dummy_dev = results['uts']['avg_abs_dev'] dummy_x = results['uts']['bin_x'] dummy_y = results['uts']['bin_y'] # iterate over all y rows, value should be equal to integer value of mlt # no variation in the median, all values should be the same check = [] for i, y in enumerate(dummy_y[:-1]): check.append(np.all(dummy_val[i, :] == y.astype(int))) check.append(np.all(dummy_dev[i, :] == 0))
def test_basic_seasonal_average(self): self.testInst.bounds = (pysat.datetime(2008, 1, 1), pysat.datetime(2008, 2, 1)) results = pysat.ssnl.avg.median2D(self.testInst, [0., 360., 24.], 'longitude', [0., 24, 24], 'mlt', ['series_profiles']) dummy_val = results['series_profiles']['median'] dummy_dev = results['series_profiles']['avg_abs_dev'] # iterate over all # no variation in the median, all values should be the same test_vals = np.arange(50)*1.2 for i, row in enumerate(dummy_val): for j, item in enumerate(row): assert np.all(item == test_vals) for i, row in enumerate(dummy_dev): for j, item in enumerate(row): assert np.all(item == 0)
def getyrdoy(date): """Return a tuple of year, day of year for a supplied datetime object.""" try: doy = date.toordinal()-datetime(date.year,1,1).toordinal()+1 except AttributeError: raise AttributeError("Must supply a pandas datetime object or " + "equivalent") else: return date.year, doy
def test_files_when_duplicates_forced(self): # create new files and make sure that new files are captured start = pysat.datetime(2008, 1, 11) stop = pysat.datetime(2008, 1, 15) dates = pysat.utils.season_date_range(start, stop, freq='1D') # clear out old files, create new ones remove_files(self.testInst) create_versioned_files(self.testInst, start, stop, freq='1D', use_doy=False, root_fname='pysat_testing_unique_{version:02d}_{revision:03d}_{year:04d}_g_{day:03d}_st.pysat_testing_file') pysat.instruments.pysat_testing.list_files = list_files self.testInst = pysat.Instrument(inst_module=pysat.instruments.pysat_testing, clean_level='clean', file_format='pysat_testing_unique_??_???_{year:04d}_g_{day:03d}_st.pysat_testing_file', update_files=True, temporary_file_list=self.temporary_file_list) assert (np.all(self.testInst.files.files.index == dates))
def test_all_single_orbit_calls_in_day(self): self.testInst.load(2009,1) ans = []; ans2=[]; self.testInst.bounds= (pysat.datetime(2009,1,1), None) for i,inst in enumerate(self.testInst.orbits): if i > 14: break ans.append(self.testInst.data.index[0] == (pds.datetime(2009,1,1)+i*relativedelta(hours=1, minutes=37))) ans2.append(self.testInst.data.index[-1] == (pds.datetime(2009,1,1)+(i+1)*relativedelta(hours=1, minutes=37) - relativedelta(seconds=1) )) assert np.all(ans) & np.all(ans2)