def test_create_datetime_index_wo_month_day_uts(self):
"""Tests ability to generate missing parameters"""
dates = pytime.create_datetime_index(year=self.year)
assert dates[0] == dt.datetime(2012, 1, 1)
assert dates[-1] == dt.datetime(2012, 1, 1)
assert len(dates) == 4
def test_calc_freq_ns():
"""Test index frequency calculation with nanosecond output"""
tind = pytime.create_datetime_index(year=np.ones(shape=(4, )) * 2001,
month=np.ones(shape=(4, )),
uts=np.arange(0.0, 0.04, .01))
freq = pytime.calc_freq(tind)
assert freq.find("10000000N") == 0
def test_calc_freq():
"""Test index frequency calculation"""
tind = pytime.create_datetime_index(year=np.ones(shape=(4, )) * 2001,
month=np.ones(shape=(4, )),
uts=np.arange(0.0, 4.0, 1.0))
freq = pytime.calc_freq(tind)
assert freq.find("1S") == 0
def test_calc_freq_ns(self):
"""Test index frequency calculation with nanosecond output"""
tind = pytime.create_datetime_index(year=self.year,
month=self.month,
uts=np.arange(0.0, 0.04, .01))
freq = pytime.calc_freq(tind)
assert freq.find("10000000N") == 0
def test_calc_freq(self):
"""Test index frequency calculation"""
tind = pytime.create_datetime_index(year=self.year,
month=self.month,
uts=np.arange(0.0, 4.0, 1.0))
freq = pytime.calc_freq(tind)
assert freq.find("1S") == 0
def test_create_datetime_index_wo_month_day_uts():
"""Tests ability to generate missing paramters"""
arr = np.ones(4)
dates = pytime.create_datetime_index(year=2012 * arr)
assert dates[0] == pds.datetime(2012, 1, 1)
assert dates[-1] == pds.datetime(2012, 1, 1)
assert len(dates) == 4
def test_create_datetime_index(self):
"""Tests ability to create an array of datetime objects from distinct
arrays of input parameters"""
dates = pytime.create_datetime_index(year=self.year,
month=self.month,
day=self.day,
uts=self.uts)
assert dates[0] == dt.datetime(2012, 2, 28)
assert dates[-1] == dt.datetime(2012, 2, 28, 0, 0, 3)
assert len(dates) == 4
def test_create_datetime_index():
"""Tests ability to create an array of datetime objects from distinct
arrays of input paramters"""
arr = np.ones(4)
dates = pytime.create_datetime_index(year=2012 * arr,
month=2 * arr,
day=28 * arr,
uts=np.arange(0, 4))
assert dates[0] == pds.datetime(2012, 2, 28)
assert dates[-1] == pds.datetime(2012, 2, 28, 0, 0, 3)
assert len(dates) == 4
def setup(self):
"""Runs before every method to create a clean testing setup."""
self.test_angles = np.array([340.0, 348.0, 358.9, 0.5, 5.0, 9.87])
self.testInst = pysat.Instrument(platform='pysat',
name='testing',
clean_level='clean')
# Add longitude to the test instrument
ones = np.ones(shape=len(self.test_angles))
tind = time.create_datetime_index(year=ones*2001,
month=ones,
uts=np.arange(0.0, len(ones), 1.0))
self.testInst.data = \
pds.DataFrame(np.array([tind, self.test_angles]).transpose(),
index=tind, columns=["time", "longitude"])
self.deg_units = ["deg", "degree", "degrees", "rad", "radian",
"radians", "h", "hr", "hrs", "hours"]
self.dist_units = ["m", "km", "cm"]
self.vel_units = ["m/s", "cm/s", "km/s", 'm s$^{-1}$', 'cm s$^{-1}$',
'km s$^{-1}$', 'm s-1', 'cm s-1', 'km s-1']
def process_parsed_filenames(stored, two_digit_year_break=None):
"""Accepts dict with data parsed from filenames and creates
a pandas Series object formatted for the Files class.
Parameters
----------
stored : orderedDict
Dict produced by parse_fixed_width_filenames or
parse_delimited_filenames
two_digit_year_break : int
If filenames only store two digits for the year, then
'1900' will be added for years >= two_digit_year_break
and '2000' will be added for years < two_digit_year_break.
Returns
-------
pandas.Series
Series, indexed by datetime, with file strings
Note
----
If two files have the same date and time information in the
filename then the file with the higher version/revision is used.
Series returned only has one file der datetime. Version is required
for this filtering, revision is optional.
"""
from pysat.utils.time import create_datetime_index
search_dict = construct_searchstring_from_format(stored['format_str'])
keys = search_dict['keys']
if len(stored['files']) > 0:
# deal with the possibility of two digit years
# years above or equal to break are considered to be 1900+
# years below break are considered to be 2000+
if two_digit_year_break is not None:
idx, = np.where(np.array(stored['year']) >= two_digit_year_break)
stored['year'][idx] = stored['year'][idx] + 1900
idx, = np.where(np.array(stored['year']) < two_digit_year_break)
stored['year'][idx] = stored['year'][idx] + 2000
# need to sort the information for things to work
rec_arr = [stored[key] for key in keys]
rec_arr.append(stored['files'])
# sort all arrays
# create a sortable records array
# keys with files
val_keys = keys + ['files']
rec_arr = np.rec.fromarrays(rec_arr, names=val_keys)
rec_arr.sort(order=val_keys, axis=0)
# pull out sorted info
for key in keys:
stored[key] = rec_arr[key]
files = rec_arr['files']
# add hour and minute information to 'second'
if stored['second'] is None:
stored['second'] = np.zeros(len(files))
if stored['hour'] is not None:
stored['second'] += 3600 * stored['hour']
if stored['minute'] is not None:
stored['second'] += 60 * stored['minute']
# version shouldn't be set to zero
# version is required to remove duplicate datetimes
if stored['revision'] is None:
stored['revision'] = np.zeros(len(files))
index = create_datetime_index(year=stored['year'],
month=stored['month'],
day=stored['day'],
uts=stored['second'])
# if version and revision are supplied
# use these parameters to weed out files that have been replaced
# with updated versions
# first, check for duplicate index times
dups = index[index.duplicated()].unique()
if (len(dups) > 0) and (stored['version'] is not None):
# we have duplicates
# keep the highest version/revision combo
version = pds.Series(stored['version'], index=index)
revision = pds.Series(stored['revision'], index=index)
revive = version * 100000. + revision
frame = pds.DataFrame(
{
'files': files,
'revive': revive,
'time': index
}, index=index)
frame = frame.sort_values(by=['time', 'revive'],
ascending=[True, False])
frame = frame.drop_duplicates(subset='time', keep='first')
return frame['files']
else:
return pds.Series(files, index=index)
else:
return pds.Series(None)
def test_create_datetime_index_wo_year():
"""Must include a year"""
dates = pytime.create_datetime_index()
def test_create_datetime_index_wo_year(self):
"""Must include a year"""
with pytest.raises(ValueError):
_ = pytime.create_datetime_index()