def test_irregular_append(self):
initial_data = reader.read(self.tsdb_existing_file)
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(
index = [
datetime(2014,1,5),
datetime(2014,1,7),
datetime(2014,1,8)
],
data = [
5.0,
7.0,
8.0
]
),
'IRR'
)
created = log_entries['C']
modified = log_entries['U']
self.assertEqual(0, len(modified))
self.assertEqual(3, len(created))
self.assertEqual((1388880000, 5.0, 0), created[0])
data = reader.read(self.tsdb_existing_file)
self.assertEqual(len(initial_data) + 3, len(data))
self.assertEqual(3.0, data.values[2])
self.assertEqual(5.0, data.values[3])
self.assertEqual(7.0, data.values[4])
self.assertEqual(8.0, data.values[5])
def test_update_over_nan(self):
# Append a nan value
log_entries = writer.write(self.tsdb_existing_file, pd.Series(index = [date(2014,1,4)], data = [np.nan]), 'D')
# Test the nan value was written
data = reader.read(self.tsdb_existing_file)
self.assertTrue(np.isnan(data.values[3]))
# Replace the nan value
log_entries = writer.write(self.tsdb_existing_file, pd.Series(index = [date(2014,1,4)], data = [4.0]), 'D')
updated_data = reader.read(self.tsdb_existing_file)
self.assertEqual(4.0, updated_data.values[3])
modified = log_entries['U']
self.assertEqual(1, len(modified))
def test_append_single(self):
log_entries = writer.write(self.tsdb_existing_file, pd.Series(index = [datetime(2014,1,4)], data = [4.0]), 'D')
modified = log_entries['U']
self.assertEqual([], modified)
data = reader.read(self.tsdb_existing_file)
self.assertEqual(4.0, data.values[-1])
def test_irregular_update_of_nan(self):
writer.write(
self.tsdb_existing_file,
pd.Series(index=[
datetime(2014, 1, 2),
datetime(2014, 1, 3),
datetime(2014, 1, 5),
datetime(2014, 1, 7),
datetime(2014, 1, 8)
],
data=[2, np.nan, 5.0, 7.0, 8.0]), 'IRR')
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(index=[
datetime(2014, 1, 2),
datetime(2014, 1, 3),
datetime(2014, 1, 5),
datetime(2014, 1, 7),
datetime(2014, 1, 8)
],
data=[2, 3.5, 5.0, 7.0, 8.0]), 'IRR')
modified = log_entries['U']
self.assertEqual(1, len(modified))
self.assertEqual(1388707200, modified[0][0])
self.assertTrue(np.isnan(modified[0][1]))
self.assertEqual(9999, modified[0][2])
data = reader.read(self.tsdb_existing_file)
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.0, data.values[1])
self.assertEqual(3.5, data.values[2])
self.assertEqual(datetime(2014, 1, 3), data.index[2].to_pydatetime())
def test_irregular_update_and_append(self):
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(index=[
datetime(2014, 1, 2),
datetime(2014, 1, 3),
datetime(2014, 1, 5),
datetime(2014, 1, 7),
datetime(2014, 1, 8)
],
data=[2.5, 3.0, 5.0, 7.0, 8.0]), 'IRR')
created = log_entries['C']
modified = log_entries['U']
self.assertEqual(1, len(modified))
self.assertEqual(4, len(created))
self.assertEqual((1388620800, 2.0, 0), modified[0])
self.assertEqual((1388620800, 2.5, 0), created[0])
data = reader.read(self.tsdb_existing_file)
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.5, data.values[1])
self.assertEqual(3.0, data.values[2])
self.assertEqual(5.0, data.values[3])
self.assertEqual(7.0, data.values[4])
self.assertEqual(8.0, data.values[5])
self.assertEqual(datetime(2014, 1, 1), data.index[0].to_pydatetime())
self.assertEqual(datetime(2014, 1, 2), data.index[1].to_pydatetime())
self.assertEqual(datetime(2014, 1, 3), data.index[2].to_pydatetime())
self.assertEqual(datetime(2014, 1, 5), data.index[3].to_pydatetime())
self.assertEqual(datetime(2014, 1, 7), data.index[4].to_pydatetime())
self.assertEqual(datetime(2014, 1, 8), data.index[5].to_pydatetime())
def test_new_write_with_missing(self):
writer.write(self.tsdb_file, pd.Series(index = [datetime(2014,1,1), datetime(2014,1,2), datetime(2014,1,3)], data = [1.0, np.nan, 3.0]), 'D')
data = reader.read(self.tsdb_file)
self.assertEqual(1.0, data.values[0])
self.assertTrue(np.isnan(data.values[1]))
self.assertEqual(3.0, data.values[2])
def test_write_overlapping_hourly(self):
input_a = pd.Series(index=[
datetime(2014, 1, 1, 0, 0, 0),
datetime(2014, 1, 1, 1, 0, 0),
datetime(2014, 1, 1, 2, 0, 0)
],
data=[1.0, 2.0, 3.0])
input_b = pd.Series(index=[
datetime(2014, 1, 1, 2, 0, 0),
datetime(2014, 1, 1, 3, 0, 0),
datetime(2014, 1, 1, 4, 0, 0),
datetime(2014, 1, 1, 5, 0, 0)
],
data=[4.0, 5.0, 6.0, 7.0])
writer.write(self.tsdb_file, input_a, 'H')
log_entries = writer.write(self.tsdb_file, input_b, 'H')
modified = log_entries['U']
self.assertEqual(1, len(modified))
self.assertEqual([(1388541600, 3.0, 0)], modified)
data = reader.read(self.tsdb_file)
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.0, data.values[1])
self.assertEqual(4.0, data.values[2])
self.assertEqual(5.0, data.values[3])
self.assertEqual(6.0, data.values[4])
self.assertEqual(7.0, data.values[5])
def test_update_and_append(self):
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(index=[
datetime(2014, 1, 2),
datetime(2014, 1, 3),
datetime(2014, 1, 4),
datetime(2014, 1, 5),
datetime(2014, 1, 6)
],
data=[2.5, 3.0, 4.0, 5.0, 6.0]), 'D')
modified = log_entries['U']
self.assertEqual(1, len(modified))
self.assertEqual((1388620800, 2.0, 0), modified[0])
data = reader.read(self.tsdb_existing_file)
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.5, data.values[1])
self.assertEqual(3.0, data.values[2])
self.assertEqual(4.0, data.values[3])
self.assertEqual(5.0, data.values[4])
self.assertEqual(6.0, data.values[5])
self.assertEqual(datetime(2014, 1, 1), data.index[0].to_pydatetime())
self.assertEqual(datetime(2014, 1, 2), data.index[1].to_pydatetime())
self.assertEqual(datetime(2014, 1, 3), data.index[2].to_pydatetime())
self.assertEqual(datetime(2014, 1, 4), data.index[3].to_pydatetime())
self.assertEqual(datetime(2014, 1, 5), data.index[4].to_pydatetime())
self.assertEqual(datetime(2014, 1, 6), data.index[5].to_pydatetime())
def test_prepend_with_overlap(self):
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(
index = [
datetime(2013,12,30),
datetime(2013,12,31),
datetime(2014,1,1)
],
data = [
-2.0,
-1.0,
0.0
]
),
'D'
)
created = log_entries['C']
self.assertEqual(
[
(1388361600, -2.0, 0),
(1388448000, -1.0, 0),
(1388534400, 0.0, 0)
],
created
)
modified = log_entries['U']
self.assertEqual([(1388534400, 1.0, 0)], modified)
data = reader.read(self.tsdb_existing_file)
self.assertEqual(-2.0, data.values[0])
self.assertEqual(-1.0, data.values[1])
self.assertEqual(0.0, data.values[2])
def test_write_monthly_start_data(self):
new_data = pd.Series(
index = [
datetime(2014,6,1),
datetime(2014,7,1),
datetime(2014,8,1),
datetime(2014,9,1)
],
data = [
6.0,
7.3,
8.0,
9.1
]
)
log_entries = writer.write(self.tsdb_file, new_data, 'MS')
modified = log_entries['U']
self.assertEqual(0, len(modified))
data = reader.read(self.tsdb_file)
self.assertEqual(4, len(data))
self.assertEqual(6.0, data.values[0])
self.assertEqual(7.3, data.values[1])
self.assertEqual(8.0, data.values[2])
self.assertEqual(9.1, data.values[3])
def test_write_irregular_data(self):
new_data = pd.Series(
index = [
datetime(1900,1,1),
datetime(1900,3,1),
datetime(1900,4,1),
datetime(1900,6,1),
],
data = [
1.0,
2.0,
3.0,
4.0
]
)
log_entries = writer.write(self.tsdb_file, new_data, 'IRR')
modified = log_entries['U']
self.assertEqual(0, len(modified))
data = reader.read(self.tsdb_file)
self.assertEqual(datetime(1900,1,1), data.index[0].to_pydatetime())
self.assertEqual(datetime(1900,3,1), data.index[1].to_pydatetime())
self.assertEqual(datetime(1900,4,1), data.index[2].to_pydatetime())
self.assertEqual(datetime(1900,6,1), data.index[3].to_pydatetime())
self.assertEqual(4, len(data))
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.0, data.values[1])
self.assertEqual(3.0, data.values[2])
self.assertEqual(4.0, data.values[3])
def test_write_irregular_data(self):
new_data = pd.Series(index=[
datetime(1900, 1, 1),
datetime(1900, 3, 1),
datetime(1900, 4, 1),
datetime(1900, 6, 1),
],
data=[1.0, 2.0, 3.0, 4.0])
log_entries = writer.write(self.tsdb_file, new_data, 'IRR')
modified = log_entries['U']
self.assertEqual(0, len(modified))
data = reader.read(self.tsdb_file)
self.assertEqual(datetime(1900, 1, 1), data.index[0].to_pydatetime())
self.assertEqual(datetime(1900, 3, 1), data.index[1].to_pydatetime())
self.assertEqual(datetime(1900, 4, 1), data.index[2].to_pydatetime())
self.assertEqual(datetime(1900, 6, 1), data.index[3].to_pydatetime())
self.assertEqual(4, len(data))
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.0, data.values[1])
self.assertEqual(3.0, data.values[2])
self.assertEqual(4.0, data.values[3])
def test_append_with_large_gap(self):
writer.write(self.tsdb_file,
pd.Series(index=[datetime(2005, 1, 1)], data=[1.0]), 'H')
start_time = time.time()
writer.write(self.tsdb_file,
pd.Series(index=[datetime(2014, 12, 31)], data=[2.0]),
'H')
end_time = time.time()
# Prior to the fix committed with this test, on my Macbook Air,
# the run time here was around 1.3 seconds. After the fix
# it is down to almost half a second (around 0.56 seconds for
# the total test run time).
# The performance improvement is larger with bigger gaps.
self.assertLessEqual((end_time - start_time), 1)
# A few spot checks to make sure the test actually ran as expected.
data = reader.read(self.tsdb_file)
self.assertEqual(1.0, data.values[0])
self.assertTrue(np.isnan(data.values[1]))
self.assertEqual(2.0, data.values[-1])
self.assertEqual(datetime(2005, 1, 1, 0, 0),
data.index[0].to_pydatetime())
self.assertEqual(datetime(2014, 12, 31),
data.index[-1].to_pydatetime())
def test_update_single(self):
log_entries = writer.write(self.tsdb_existing_file, pd.Series(index = [datetime(2014,1,2)], data = [2.5]), 'D')
modified = log_entries['U']
self.assertEqual(1, len(modified))
self.assertEqual([(1388620800, 2.0, 0)], modified)
data = reader.read(self.tsdb_existing_file)
self.assertEqual(2.5, data.values[1])
def test_append_single(self):
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(index=[datetime(2014, 1, 4)], data=[4.0]), 'D')
modified = log_entries['U']
self.assertEqual([], modified)
data = reader.read(self.tsdb_existing_file)
self.assertEqual(4.0, data.values[-1])
def test_prepend_single(self):
log_entries = writer.write(self.tsdb_existing_file, pd.Series(index = [datetime(2013,12,31)], data = [-1.0]), 'D')
created = log_entries['C']
self.assertEqual([(1388448000, -1.0, 0)], created)
modified = log_entries['U']
self.assertEqual([], modified)
data = reader.read(self.tsdb_existing_file)
self.assertEqual(-1.0, data.values[0])
def test_update_single(self):
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(index=[datetime(2014, 1, 2)], data=[2.5]), 'D')
modified = log_entries['U']
self.assertEqual(1, len(modified))
self.assertEqual([(1388620800, 2.0, 0)], modified)
data = reader.read(self.tsdb_existing_file)
self.assertEqual(2.5, data.values[1])
def test_prepend_with_gap(self):
log_entries = writer.write(self.tsdb_existing_file, pd.Series(index = [datetime(2013,12,30)], data = [-2.0]), 'D')
created = log_entries['C']
self.assertEqual([(1388361600, -2.0, 0), (1388448000, -9999, 9999)], created)
modified = log_entries['U']
self.assertEqual([], modified)
data = reader.read(self.tsdb_existing_file)
self.assertEqual(-2.0, data.values[0])
self.assertTrue(np.isnan(data.values[1]))
def test_read(self):
data = reader.read(self.tsdb_file)
self.assertEqual(datetime(2014, 1, 1), data.index[0].to_pydatetime())
self.assertEqual(datetime(2014, 1, 2), data.index[1].to_pydatetime())
self.assertEqual(datetime(2014, 1, 3), data.index[2].to_pydatetime())
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.0, data.values[1])
self.assertEqual(3.0, data.values[2])
def test_update_multiple(self):
log_entries = writer.write(self.tsdb_existing_file, pd.Series(index = [datetime(2014,1,2),datetime(2014,1,3)], data = [2.5, 3.5]), 'D')
modified = log_entries['U']
self.assertEqual(2, len(modified))
self.assertEqual((1388620800, 2.0, 0), modified[0])
self.assertEqual((1388707200, 3.0, 0), modified[1])
data = reader.read(self.tsdb_existing_file)
self.assertEqual(2.5, data.values[1])
self.assertEqual(3.5, data.values[2])
def test_read(self):
data = reader.read(self.tsdb_file)
self.assertEqual(datetime(2014,1,1), data.index[0].to_pydatetime())
self.assertEqual(datetime(2014,1,2), data.index[1].to_pydatetime())
self.assertEqual(datetime(2014,1,3), data.index[2].to_pydatetime())
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.0, data.values[1])
self.assertEqual(3.0, data.values[2])
def test_new_write_and_update_minute_data(self):
writer.write(
self.tsdb_file,
pd.Series(index=[
datetime(2014, 1, 1, 18, 1, 0),
datetime(2014, 1, 1, 18, 2, 0),
datetime(2014, 1, 1, 18, 3, 0)
],
data=[1.0, np.nan, 3.0]), '1T')
data = reader.read(self.tsdb_file)
self.assertEqual(1.0, data.values[0])
self.assertTrue(np.isnan(data.values[1]))
self.assertEqual(3.0, data.values[2])
self.assertEqual(datetime(2014, 1, 1, 18, 1, 0),
data.index[0].to_pydatetime())
self.assertEqual(datetime(2014, 1, 1, 18, 2, 0),
data.index[1].to_pydatetime())
self.assertEqual(datetime(2014, 1, 1, 18, 3, 0),
data.index[2].to_pydatetime())
writer.write(
self.tsdb_file,
pd.Series(index=[
datetime(2014, 1, 1, 18, 3, 0),
datetime(2014, 1, 1, 18, 4, 0),
datetime(2014, 1, 1, 18, 5, 0)
],
data=[3.5, 4.0, 5.0]), '1T')
data = reader.read(self.tsdb_file)
self.assertEqual(1.0, data.values[0])
self.assertTrue(np.isnan(data.values[1]))
self.assertEqual(3.5, data.values[2])
self.assertEqual(4.0, data.values[3])
self.assertEqual(5.0, data.values[4])
self.assertEqual(datetime(2014, 1, 1, 18, 1, 0),
data.index[0].to_pydatetime())
self.assertEqual(datetime(2014, 1, 1, 18, 5, 0),
data.index[-1].to_pydatetime())
def test_update_over_nan(self):
# Append a nan value
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(index=[date(2014, 1, 4)], data=[np.nan]), 'D')
# Test the nan value was written
data = reader.read(self.tsdb_existing_file)
self.assertTrue(np.isnan(data.values[3]))
# Replace the nan value
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(index=[date(2014, 1, 4)], data=[4.0]), 'D')
updated_data = reader.read(self.tsdb_existing_file)
self.assertEqual(4.0, updated_data.values[3])
modified = log_entries['U']
self.assertEqual(1, len(modified))
def test_prepend_single(self):
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(index=[datetime(2013, 12, 31)], data=[-1.0]), 'D')
created = log_entries['C']
self.assertEqual([(1388448000, -1.0, 0)], created)
modified = log_entries['U']
self.assertEqual([], modified)
data = reader.read(self.tsdb_existing_file)
self.assertEqual(-1.0, data.values[0])
def test_append_multiple(self):
log_entries = writer.write(self.tsdb_existing_file, pd.Series(index = [datetime(2014,1,4), datetime(2014,1,5), datetime(2014,1,6)], data = [4.0, 5.0, 6.0]), 'D')
modified = log_entries['U']
self.assertEqual([], modified)
data = reader.read(self.tsdb_existing_file)
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.0, data.values[1])
self.assertEqual(3.0, data.values[2])
self.assertEqual(4.0, data.values[3])
self.assertEqual(5.0, data.values[4])
self.assertEqual(6.0, data.values[5])
def test_prepend_with_gap(self):
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(index=[datetime(2013, 12, 30)], data=[-2.0]), 'D')
created = log_entries['C']
self.assertEqual([(1388361600, -2.0, 0), (1388448000, -9999, 9999)],
created)
modified = log_entries['U']
self.assertEqual([], modified)
data = reader.read(self.tsdb_existing_file)
self.assertEqual(-2.0, data.values[0])
self.assertTrue(np.isnan(data.values[1]))
def test_write_missing_date(self):
log_entries = writer.write(self.tsdb_existing_file, pd.Series(index = [date(2014,1,3),date(2014,1,5),date(2014,1,6)], data = [3.0, np.nan, 6.5]), 'D')
modified = log_entries['U']
self.assertEqual(0, len(modified))
data = reader.read(self.tsdb_existing_file)
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.0, data.values[1])
self.assertEqual(3.0, data.values[2])
self.assertTrue(np.isnan(data.values[3]))
self.assertTrue(np.isnan(data.values[4]))
self.assertEqual(6.5, data.values[5])
def read(self, identifier, freq, **kwargs):
"""
Read the entire timeseries record for the requested timeseries instance.
:param identifier: Identifier of the timeseries.
:type identifier: string
:param freq: Timeseries data frequency.
:type freq: string
:param kwargs: Attributes to match against timeseries instances (e.g. source, measurand).
:type kwargs: kwargs
:returns: pandas.DataFrame -- Timeseries data.
"""
return reader.read(self.get_file_path(identifier, freq, **kwargs))
def read(self, identifier, freq, **kwargs):
"""
Read the entire timeseries record for the requested timeseries instance.
:param identifier: Identifier of the timeseries.
:type identifier: string
:param freq: Timeseries data frequency.
:type freq: string
:param kwargs: Attributes to match against timeseries instances (e.g. source, measurand).
:type kwargs: kwargs
:returns: pandas.DataFrame -- Timeseries data.
"""
return reader.read(self.get_file_path(identifier, freq, **kwargs))
def test_irregular_append(self):
initial_data = reader.read(self.tsdb_existing_file)
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(index=[
datetime(2014, 1, 5),
datetime(2014, 1, 7),
datetime(2014, 1, 8)
],
data=[5.0, 7.0, 8.0]), 'IRR')
created = log_entries['C']
modified = log_entries['U']
self.assertEqual(0, len(modified))
self.assertEqual(3, len(created))
self.assertEqual((1388880000, 5.0, 0), created[0])
data = reader.read(self.tsdb_existing_file)
self.assertEqual(len(initial_data) + 3, len(data))
self.assertEqual(3.0, data.values[2])
self.assertEqual(5.0, data.values[3])
self.assertEqual(7.0, data.values[4])
self.assertEqual(8.0, data.values[5])
def test_update_multiple(self):
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(index=[datetime(2014, 1, 2),
datetime(2014, 1, 3)],
data=[2.5, 3.5]), 'D')
modified = log_entries['U']
self.assertEqual(2, len(modified))
self.assertEqual((1388620800, 2.0, 0), modified[0])
self.assertEqual((1388707200, 3.0, 0), modified[1])
data = reader.read(self.tsdb_existing_file)
self.assertEqual(2.5, data.values[1])
self.assertEqual(3.5, data.values[2])
def test_new_write_and_update_minute_data(self):
writer.write(self.tsdb_file, pd.Series(index = [datetime(2014,1,1,18,1,0), datetime(2014,1,1,18,2,0), datetime(2014,1,1,18,3,0)], data = [1.0, np.nan, 3.0]), '1T')
data = reader.read(self.tsdb_file)
self.assertEqual(1.0, data.values[0])
self.assertTrue(np.isnan(data.values[1]))
self.assertEqual(3.0, data.values[2])
self.assertEqual(datetime(2014,1,1,18,1,0), data.index[0].to_pydatetime())
self.assertEqual(datetime(2014,1,1,18,2,0), data.index[1].to_pydatetime())
self.assertEqual(datetime(2014,1,1,18,3,0), data.index[2].to_pydatetime())
writer.write(self.tsdb_file, pd.Series(index = [datetime(2014,1,1,18,3,0), datetime(2014,1,1,18,4,0), datetime(2014,1,1,18,5,0)], data = [3.5, 4.0, 5.0]), '1T')
data = reader.read(self.tsdb_file)
self.assertEqual(1.0, data.values[0])
self.assertTrue(np.isnan(data.values[1]))
self.assertEqual(3.5, data.values[2])
self.assertEqual(4.0, data.values[3])
self.assertEqual(5.0, data.values[4])
self.assertEqual(datetime(2014,1,1,18,1,0), data.index[0].to_pydatetime())
self.assertEqual(datetime(2014,1,1,18,5,0), data.index[-1].to_pydatetime())
def test_new_write_with_missing(self):
writer.write(
self.tsdb_file,
pd.Series(index=[
datetime(2014, 1, 1),
datetime(2014, 1, 2),
datetime(2014, 1, 3)
],
data=[1.0, np.nan, 3.0]), 'D')
data = reader.read(self.tsdb_file)
self.assertEqual(1.0, data.values[0])
self.assertTrue(np.isnan(data.values[1]))
self.assertEqual(3.0, data.values[2])
def test_read_missing(self):
data = reader.read(self.tsdb_file_with_missing)
self.assertEqual(datetime(2014, 1, 1), data.index[0].to_pydatetime())
self.assertEqual(datetime(2014, 1, 2), data.index[1].to_pydatetime())
self.assertEqual(datetime(2014, 1, 3), data.index[2].to_pydatetime())
self.assertEqual(datetime(2014, 1, 4), data.index[3].to_pydatetime())
self.assertEqual(datetime(2014, 1, 5), data.index[4].to_pydatetime())
self.assertEqual(datetime(2014, 1, 6), data.index[5].to_pydatetime())
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.0, data.values[1])
self.assertEqual(3.0, data.values[2])
self.assertTrue(np.isnan(data.values[3]))
self.assertEqual(5.0, data.values[4])
self.assertEqual(6.0, data.values[5])
def test_irregular_update_of_nan(self):
writer.write(self.tsdb_existing_file, pd.Series(index = [datetime(2014,1,2), datetime(2014,1,3), datetime(2014,1,5), datetime(2014,1,7), datetime(2014,1,8)], data = [2, np.nan, 5.0, 7.0, 8.0]), 'IRR')
log_entries = writer.write(self.tsdb_existing_file, pd.Series(index = [datetime(2014,1,2), datetime(2014,1,3), datetime(2014,1,5), datetime(2014,1,7), datetime(2014,1,8)], data = [2, 3.5, 5.0, 7.0, 8.0]), 'IRR')
modified = log_entries['U']
self.assertEqual(1, len(modified))
self.assertEqual(1388707200, modified[0][0])
self.assertTrue(np.isnan(modified[0][1]))
self.assertEqual(9999, modified[0][2])
data = reader.read(self.tsdb_existing_file)
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.0, data.values[1])
self.assertEqual(3.5, data.values[2])
self.assertEqual(datetime(2014,1,3), data.index[2].to_pydatetime())
def test_read_missing(self):
data = reader.read(self.tsdb_file_with_missing)
self.assertEqual(datetime(2014,1,1), data.index[0].to_pydatetime())
self.assertEqual(datetime(2014,1,2), data.index[1].to_pydatetime())
self.assertEqual(datetime(2014,1,3), data.index[2].to_pydatetime())
self.assertEqual(datetime(2014,1,4), data.index[3].to_pydatetime())
self.assertEqual(datetime(2014,1,5), data.index[4].to_pydatetime())
self.assertEqual(datetime(2014,1,6), data.index[5].to_pydatetime())
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.0, data.values[1])
self.assertEqual(3.0, data.values[2])
self.assertTrue(np.isnan(data.values[3]))
self.assertEqual(5.0, data.values[4])
self.assertEqual(6.0, data.values[5])
def test_append_monthly_end_data(self):
new_data = pd.Series(index=[
datetime(1901, 1, 31),
datetime(1901, 2, 28),
],
data=[31.1, 28.2])
log_entries = writer.write(self.tsdb_monthly_existing_file, new_data,
'M')
modified = log_entries['U']
self.assertEqual(0, len(modified))
data = reader.read(self.tsdb_monthly_existing_file)
self.assertEqual(14, len(data))
self.assertEqual(31.1, data.values[-2])
self.assertEqual(28.2, data.values[-1])
def read_dataframe(self, identifiers, freq, **kwargs):
"""
Read the entire timeseries record for the requested timeseries instances.
:param identifiers: Identifiers of the timeseries to read into a DataFrame.
:type identifiers: array[string]
:param freq: Timeseries data frequency.
:type freq: string
:param kwargs: Attributes to match against timeseries instances (e.g. source, measurand).
:type kwargs: kwargs
:returns: pandas.DataFrame -- Timeseries data.
"""
data = {}
for ts_id in identifiers:
data[ts_id] = reader.read(self.get_file_path(ts_id, freq, **kwargs))
return pd.DataFrame(data)
def test_update_and_append_with_gap(self):
log_entries = writer.write(self.tsdb_existing_file, pd.Series(index = [datetime(2014,1,5), datetime(2014,1,6)], data = [5.0, 6.0]), 'D')
modified = log_entries['U']
data = reader.read(self.tsdb_existing_file)
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.0, data.values[1])
self.assertEqual(3.0, data.values[2])
self.assertTrue(np.isnan(data.values[3]))
self.assertEqual(5.0, data.values[4])
self.assertEqual(6.0, data.values[5])
self.assertEqual(datetime(2014,1,1), data.index[0].to_pydatetime())
self.assertEqual(datetime(2014,1,2), data.index[1].to_pydatetime())
self.assertEqual(datetime(2014,1,3), data.index[2].to_pydatetime())
self.assertEqual(datetime(2014,1,4), data.index[3].to_pydatetime())
self.assertEqual(datetime(2014,1,5), data.index[4].to_pydatetime())
self.assertEqual(datetime(2014,1,6), data.index[5].to_pydatetime())
def test_append_monthly_start_data(self):
new_data = pd.Series(index=[
datetime(1901, 1, 1),
datetime(1901, 2, 1),
],
data=[31.1, 28.2])
log_entries = writer.write(self.tsdb_monthly_start_existing_file,
new_data, 'MS')
modified = log_entries['U']
self.assertEqual(0, len(modified))
data = reader.read(self.tsdb_monthly_start_existing_file)
self.assertEqual(datetime(1900, 1, 1), data.index[0].to_pydatetime())
self.assertEqual(datetime(1901, 2, 1), data.index[-1].to_pydatetime())
self.assertEqual(14, len(data))
def read_dataframe(self, identifiers, freq, **kwargs):
"""
Read the entire timeseries record for the requested timeseries instances.
:param identifiers: Identifiers of the timeseries to read into a DataFrame.
:type identifiers: array[string]
:param freq: Timeseries data frequency.
:type freq: string
:param kwargs: Attributes to match against timeseries instances (e.g. source, measurand).
:type kwargs: kwargs
:returns: pandas.DataFrame -- Timeseries data.
"""
data = {}
for ts_id in identifiers:
data[ts_id] = reader.read(self.get_file_path(
ts_id, freq, **kwargs))
return pd.DataFrame(data)
def test_append_multiple(self):
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(index=[
datetime(2014, 1, 4),
datetime(2014, 1, 5),
datetime(2014, 1, 6)
],
data=[4.0, 5.0, 6.0]), 'D')
modified = log_entries['U']
self.assertEqual([], modified)
data = reader.read(self.tsdb_existing_file)
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.0, data.values[1])
self.assertEqual(3.0, data.values[2])
self.assertEqual(4.0, data.values[3])
self.assertEqual(5.0, data.values[4])
self.assertEqual(6.0, data.values[5])
def test_write_missing_date(self):
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(
index=[date(2014, 1, 3),
date(2014, 1, 5),
date(2014, 1, 6)],
data=[3.0, np.nan, 6.5]), 'D')
modified = log_entries['U']
self.assertEqual(0, len(modified))
data = reader.read(self.tsdb_existing_file)
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.0, data.values[1])
self.assertEqual(3.0, data.values[2])
self.assertTrue(np.isnan(data.values[3]))
self.assertTrue(np.isnan(data.values[4]))
self.assertEqual(6.5, data.values[5])
def test_append_30min(self):
new_data = pd.Series(index=[
datetime(2014, 8, 30, 6, 0, 0),
datetime(2014, 8, 30, 6, 30, 0)
],
data=[6.0, 6.30])
log_entries = writer.write(self.tsdb_30min_existing_file, new_data,
'30min')
modified = log_entries['U']
self.assertEqual(0, len(modified))
data = reader.read(self.tsdb_30min_existing_file)
self.assertEqual(148, len(data))
self.assertEqual(17.2, data.values[0])
self.assertTrue(np.isnan(data.values[-3]))
self.assertEqual(6.0, data.values[-2])
self.assertEqual(6.3, data.values[-1])
def test_prepend_with_overlap(self):
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(index=[
datetime(2013, 12, 30),
datetime(2013, 12, 31),
datetime(2014, 1, 1)
],
data=[-2.0, -1.0, 0.0]), 'D')
created = log_entries['C']
self.assertEqual([(1388361600, -2.0, 0), (1388448000, -1.0, 0),
(1388534400, 0.0, 0)], created)
modified = log_entries['U']
self.assertEqual([(1388534400, 1.0, 0)], modified)
data = reader.read(self.tsdb_existing_file)
self.assertEqual(-2.0, data.values[0])
self.assertEqual(-1.0, data.values[1])
self.assertEqual(0.0, data.values[2])
def test_float32_irregular_write(self):
"""
Test irregular write of float32 data.
See: https://github.com/amacd31/phildb/issues/16
"""
sample = pd.Series(pd.np.array([x + 0.1 for x in range(10)],
dtype=pd.np.float32),
index=pd.date_range('2017-08-06 06:50:00',
periods=10,
freq='1T'))
log_entries = writer.write(self.tsdb_existing_file, sample, 'IRR')
data = reader.read(self.tsdb_existing_file)
self.assertEqual(datetime(2017, 8, 6, 6, 50, 0, 0),
data.index[3].to_pydatetime())
self.assertEqual(datetime(2017, 8, 6, 6, 51, 0, 0),
data.index[4].to_pydatetime())
def test_update_and_append(self):
log_entries = writer.write(self.tsdb_existing_file, pd.Series(index = [datetime(2014,1,2), datetime(2014,1,3), datetime(2014,1,4), datetime(2014,1,5), datetime(2014,1,6)], data = [2.5, 3.0, 4.0, 5.0, 6.0]), 'D')
modified = log_entries['U']
self.assertEqual(1, len(modified))
self.assertEqual((1388620800, 2.0, 0), modified[0])
data = reader.read(self.tsdb_existing_file)
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.5, data.values[1])
self.assertEqual(3.0, data.values[2])
self.assertEqual(4.0, data.values[3])
self.assertEqual(5.0, data.values[4])
self.assertEqual(6.0, data.values[5])
self.assertEqual(datetime(2014,1,1), data.index[0].to_pydatetime())
self.assertEqual(datetime(2014,1,2), data.index[1].to_pydatetime())
self.assertEqual(datetime(2014,1,3), data.index[2].to_pydatetime())
self.assertEqual(datetime(2014,1,4), data.index[3].to_pydatetime())
self.assertEqual(datetime(2014,1,5), data.index[4].to_pydatetime())
self.assertEqual(datetime(2014,1,6), data.index[5].to_pydatetime())
def test_write_monthly_start_data(self):
new_data = pd.Series(index=[
datetime(2014, 6, 1),
datetime(2014, 7, 1),
datetime(2014, 8, 1),
datetime(2014, 9, 1)
],
data=[6.0, 7.3, 8.0, 9.1])
log_entries = writer.write(self.tsdb_file, new_data, 'MS')
modified = log_entries['U']
self.assertEqual(0, len(modified))
data = reader.read(self.tsdb_file)
self.assertEqual(4, len(data))
self.assertEqual(6.0, data.values[0])
self.assertEqual(7.3, data.values[1])
self.assertEqual(8.0, data.values[2])
self.assertEqual(9.1, data.values[3])
def test_append_with_large_gap(self):
writer.write(self.tsdb_file, pd.Series(index = [datetime(2005,1,1)], data = [1.0]), 'H')
start_time = time.time()
writer.write(self.tsdb_file, pd.Series(index = [datetime(2014,12,31)], data = [2.0]), 'H')
end_time = time.time()
# Prior to the fix committed with this test, on my Macbook Air,
# the run time here was around 1.3 seconds. After the fix
# it is down to almost half a second (around 0.56 seconds for
# the total test run time).
# The performance improvement is larger with bigger gaps.
self.assertLessEqual((end_time - start_time), 1)
# A few spot checks to make sure the test actually ran as expected.
data = reader.read(self.tsdb_file)
self.assertEqual(1.0, data.values[0])
self.assertTrue(np.isnan(data.values[1]))
self.assertEqual(2.0, data.values[-1])
self.assertEqual(datetime(2005,1,1,0,0), data.index[0].to_pydatetime())
self.assertEqual(datetime(2014,12,31), data.index[-1].to_pydatetime())
def test_write_overlapping_hourly(self):
input_a = pd.Series(
index = [
datetime(2014,1,1,0,0,0),
datetime(2014,1,1,1,0,0),
datetime(2014,1,1,2,0,0)
],
data = [
1.0,
2.0,
3.0
]
)
input_b = pd.Series(
index = [
datetime(2014,1,1,2,0,0),
datetime(2014,1,1,3,0,0),
datetime(2014,1,1,4,0,0),
datetime(2014,1,1,5,0,0)
],
data = [
4.0,
5.0,
6.0,
7.0
]
)
writer.write(self.tsdb_file, input_a, 'H')
log_entries = writer.write(self.tsdb_file, input_b, 'H')
modified = log_entries['U']
self.assertEqual(1, len(modified))
self.assertEqual([(1388541600, 3.0, 0)], modified)
data = reader.read(self.tsdb_file)
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.0, data.values[1])
self.assertEqual(4.0, data.values[2])
self.assertEqual(5.0, data.values[3])
self.assertEqual(6.0, data.values[4])
self.assertEqual(7.0, data.values[5])
def test_update_and_append_with_gap(self):
log_entries = writer.write(
self.tsdb_existing_file,
pd.Series(index=[datetime(2014, 1, 5),
datetime(2014, 1, 6)],
data=[5.0, 6.0]), 'D')
modified = log_entries['U']
data = reader.read(self.tsdb_existing_file)
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.0, data.values[1])
self.assertEqual(3.0, data.values[2])
self.assertTrue(np.isnan(data.values[3]))
self.assertEqual(5.0, data.values[4])
self.assertEqual(6.0, data.values[5])
self.assertEqual(datetime(2014, 1, 1), data.index[0].to_pydatetime())
self.assertEqual(datetime(2014, 1, 2), data.index[1].to_pydatetime())
self.assertEqual(datetime(2014, 1, 3), data.index[2].to_pydatetime())
self.assertEqual(datetime(2014, 1, 4), data.index[3].to_pydatetime())
self.assertEqual(datetime(2014, 1, 5), data.index[4].to_pydatetime())
self.assertEqual(datetime(2014, 1, 6), data.index[5].to_pydatetime())
def test_append_monthly_end_data(self):
new_data = pd.Series(
index = [
datetime(1901,1,31),
datetime(1901,2,28),
],
data = [
31.1,
28.2
]
)
log_entries = writer.write(self.tsdb_monthly_existing_file, new_data, 'M')
modified = log_entries['U']
self.assertEqual(0, len(modified))
data = reader.read(self.tsdb_monthly_existing_file)
self.assertEqual(14, len(data))
self.assertEqual(31.1, data.values[-2])
self.assertEqual(28.2, data.values[-1])
def test_append_monthly_start_data(self):
new_data = pd.Series(
index = [
datetime(1901,1,1),
datetime(1901,2,1),
],
data = [
31.1,
28.2
]
)
log_entries = writer.write(self.tsdb_monthly_start_existing_file, new_data, 'MS')
modified = log_entries['U']
self.assertEqual(0, len(modified))
data = reader.read(self.tsdb_monthly_start_existing_file)
self.assertEqual(datetime(1900,1,1), data.index[0].to_pydatetime())
self.assertEqual(datetime(1901,2,1), data.index[-1].to_pydatetime())
self.assertEqual(14, len(data))
def test_append_30min(self):
new_data = pd.Series(
index = [
datetime(2014,8,30,6,0,0),
datetime(2014,8,30,6,30,0)
],
data = [
6.0,
6.30
]
)
log_entries = writer.write(self.tsdb_30min_existing_file, new_data, '30min')
modified = log_entries['U']
self.assertEqual(0, len(modified))
data = reader.read(self.tsdb_30min_existing_file)
self.assertEqual(148, len(data))
self.assertEqual(17.2, data.values[0])
self.assertTrue(np.isnan(data.values[-3]))
self.assertEqual(6.0, data.values[-2])
self.assertEqual(6.3, data.values[-1])
def test_irregular_update_and_append(self):
log_entries = writer.write(self.tsdb_existing_file, pd.Series(index = [datetime(2014,1,2), datetime(2014,1,3), datetime(2014,1,5), datetime(2014,1,7), datetime(2014,1,8)], data = [2.5, 3.0, 5.0, 7.0, 8.0]), 'IRR')
created = log_entries['C']
modified = log_entries['U']
self.assertEqual(1, len(modified))
self.assertEqual(4, len(created))
self.assertEqual((1388620800, 2.0, 0), modified[0])
self.assertEqual((1388620800, 2.5, 0), created[0])
data = reader.read(self.tsdb_existing_file)
self.assertEqual(1.0, data.values[0])
self.assertEqual(2.5, data.values[1])
self.assertEqual(3.0, data.values[2])
self.assertEqual(5.0, data.values[3])
self.assertEqual(7.0, data.values[4])
self.assertEqual(8.0, data.values[5])
self.assertEqual(datetime(2014,1,1), data.index[0].to_pydatetime())
self.assertEqual(datetime(2014,1,2), data.index[1].to_pydatetime())
self.assertEqual(datetime(2014,1,3), data.index[2].to_pydatetime())
self.assertEqual(datetime(2014,1,5), data.index[3].to_pydatetime())
self.assertEqual(datetime(2014,1,7), data.index[4].to_pydatetime())
self.assertEqual(datetime(2014,1,8), data.index[5].to_pydatetime())
def test_float32_irregular_write(self):
"""
Test irregular write of float32 data.
See: https://github.com/amacd31/phildb/issues/16
"""
sample = pd.Series(
pd.np.array(
[x + 0.1 for x in range(10)],
dtype=pd.np.float32
),
index=pd.date_range(
'2017-08-06 06:50:00',
periods=10,
freq='1T'
)
)
log_entries = writer.write(self.tsdb_existing_file, sample, 'IRR')
data = reader.read(self.tsdb_existing_file)
self.assertEqual(datetime(2017,8,6,6,50,0,0), data.index[3].to_pydatetime())
self.assertEqual(datetime(2017,8,6,6,51,0,0), data.index[4].to_pydatetime())
def test_read_empty(self):
data = reader.read(self.empty_tsdb_file)
self.assertEqual(0, len(data))
def test_read(self):
data = reader.read('/tmp/not_an_actual_existing_file')
self.assertEqual(0, len(data))
def test_read_empty(self):
data = reader.read(self.empty_tsdb_file)
self.assertEqual(0, len(data))
def test_read(self):
data = reader.read('/tmp/not_an_actual_existing_file')
self.assertEqual(0, len(data))
