def test_tickstore_to_bucket_with_image():
symbol = 'SYM'
tz = 'UTC'
initial_image = {'index': dt(2014, 1, 1, 0, 0, tzinfo=mktz(tz)), 'A': 123, 'B': 54.4, 'C': 'DESC'}
data = [{'index': dt(2014, 1, 1, 0, 1, tzinfo=mktz(tz)), 'A': 124, 'D': 0},
{'index': dt(2014, 1, 1, 0, 2, tzinfo=mktz(tz)), 'A': 125, 'B': 27.2}]
bucket, final_image = TickStore._to_bucket(data, symbol, initial_image)
assert bucket[COUNT] == 2
assert bucket[END] == dt(2014, 1, 1, 0, 2, tzinfo=mktz(tz))
assert set(bucket[COLUMNS]) == set(('A', 'B', 'D'))
assert set(bucket[COLUMNS]['A']) == set((ROWMASK, DTYPE, DATA))
assert get_coldata(bucket[COLUMNS]['A']) == ([124, 125], [1, 1, 0, 0, 0, 0, 0, 0])
assert get_coldata(bucket[COLUMNS]['B']) == ([27.2], [0, 1, 0, 0, 0, 0, 0, 0])
assert get_coldata(bucket[COLUMNS]['D']) == ([0], [1, 0, 0, 0, 0, 0, 0, 0])
index = [dt.fromtimestamp(int(i/1000)).replace(tzinfo=mktz(tz)) for i in
list(np.cumsum(np.frombuffer(decompress(bucket[INDEX]), dtype='uint64')))]
assert index == [i['index'] for i in data]
assert bucket[COLUMNS]['A'][DTYPE] == 'int64'
assert bucket[COLUMNS]['B'][DTYPE] == 'float64'
assert bucket[SYMBOL] == symbol
assert bucket[START] == initial_image['index']
assert bucket[IMAGE_DOC][IMAGE] == initial_image
assert bucket[IMAGE_DOC] == {IMAGE: initial_image,
IMAGE_TIME: initial_image['index']}
assert final_image == {'index': data[-1]['index'], 'A': 125, 'B': 27.2, 'C': 'DESC', 'D': 0}
def test_tickstore_to_bucket_always_forwards_image():
symbol = 'SYM'
tz = 'UTC'
initial_image = {'index': dt(2014, 2, 1, 0, 0, tzinfo=mktz(tz)), 'A': 123, 'B': 54.4, 'C': 'DESC'}
data = [{'index': dt(2014, 1, 1, 0, 1, tzinfo=mktz(tz)), 'A': 124, 'D': 0}]
with pytest.raises(UnorderedDataException) as e:
TickStore._to_bucket(data, symbol, initial_image)
def test_mongo_date_range_query():
self = create_autospec(TickStore)
self._collection = create_autospec(Collection)
self._collection.find_one.return_value = {'s': sentinel.start}
self._symbol_query = partial(TickStore._symbol_query, self)
query = TickStore._mongo_date_range_query(
self, 'sym',
DateRange(dt(2014, 1, 1, 0, 0, tzinfo=mktz()),
dt(2014, 1, 2, 0, 0, tzinfo=mktz())))
assert self._collection.find_one.call_args_list == [
call({
'sy': 'sym',
's': {
'$lte': dt(2014, 1, 1, 0, 0, tzinfo=mktz())
}
},
sort=[('s', -1)],
projection={
's': 1,
'_id': 0
}),
call({
'sy': 'sym',
's': {
'$gt': dt(2014, 1, 2, 0, 0, tzinfo=mktz())
}
},
sort=[('s', 1)],
projection={
's': 1,
'_id': 0
})
]
assert query == {'s': {'$gte': sentinel.start, '$lt': sentinel.start}}
def test_tickstore_to_bucket_always_forwards_image():
symbol = 'SYM'
tz = 'UTC'
initial_image = {'index': dt(2014, 2, 1, 0, 0, tzinfo=mktz(tz)), 'A': 123, 'B': 54.4, 'C': 'DESC'}
data = [{'index': dt(2014, 1, 1, 0, 1, tzinfo=mktz(tz)), 'A': 124, 'D': 0}]
with pytest.raises(UnorderedDataException) as e:
TickStore._to_bucket(data, symbol, initial_image)
def test_bitemporal_store_read_as_of_timezone(bitemporal_library):
bitemporal_library.update('spam', ts1, as_of=dt(2015, 5, 1, tzinfo=mktz('Europe/London')))
bitemporal_library.update('spam', read_str_as_pandas(""" sample_dt | near
2012-12-01 17:06:11.040 | 25"""),
as_of=dt(2015, 5, 2, tzinfo=mktz('Europe/London')))
df = bitemporal_library.read('spam', as_of=dt(2015, 5, 2, tzinfo=mktz('Asia/Hong_Kong'))).data
assert_frame_equal(df, ts1)
def test_bitemporal_store_read_as_of_timezone(bitemporal_library):
bitemporal_library.update('spam', ts1, as_of=dt(2015, 5, 1, tzinfo=mktz('Europe/London')))
bitemporal_library.update('spam', read_str_as_pandas(""" sample_dt | near
2012-12-01 17:06:11.040 | 25"""),
as_of=dt(2015, 5, 2, tzinfo=mktz('Europe/London')))
df = bitemporal_library.read('spam', as_of=dt(2015, 5, 2, tzinfo=mktz('Asia/Hong_Kong'))).data
assert_frame_equal(df, ts1)
def test_read_ts_raw_all_version_ok(bitemporal_library):
bitemporal_library.update('spam',
ts1,
as_of=dt(2015, 5, 1, tzinfo=mktz('UTC')))
bitemporal_library.update('spam',
read_str_as_pandas(""" sample_dt | near
2012-12-01 17:06:11.040 | 25"""
),
as_of=dt(2015, 5, 5, tzinfo=mktz('UTC')))
bitemporal_library.update('spam',
read_str_as_pandas(""" sample_dt | near
2012-11-08 17:06:11.040 | 42"""
),
as_of=dt(2015, 5, 3, tzinfo=mktz('UTC')))
bitemporal_library.update('spam',
read_str_as_pandas(""" sample_dt | near
2012-10-08 17:06:11.040 | 42
2013-01-01 17:06:11.040 | 100"""
),
as_of=dt(2015, 5, 10, tzinfo=mktz('UTC')))
assert_frame_equal(
bitemporal_library.read('spam', raw=True).data,
read_str_as_pandas(
""" sample_dt | observed_dt | near
2012-09-08 17:06:11.040 | 2015-05-01 | 1.0
2012-10-08 17:06:11.040 | 2015-05-01 | 2.0
2012-10-08 17:06:11.040 | 2015-05-10 | 42
2012-10-09 17:06:11.040 | 2015-05-01 | 2.5
2012-11-08 17:06:11.040 | 2015-05-01 | 3.0
2012-11-08 17:06:11.040 | 2015-05-03 | 42
2012-12-01 17:06:11.040 | 2015-05-05 | 25
2013-01-01 17:06:11.040 | 2015-05-10 | 100""",
num_index=2))
def test_tickstore_to_bucket_with_image():
symbol = 'SYM'
tz = 'UTC'
initial_image = {'index': dt(2014, 1, 1, 0, 0, tzinfo=mktz(tz)), 'A': 123, 'B': 54.4, 'C': 'DESC'}
data = [{'index': dt(2014, 1, 1, 0, 1, tzinfo=mktz(tz)), 'A': 124, 'D': 0},
{'index': dt(2014, 1, 1, 0, 2, tzinfo=mktz(tz)), 'A': 125, 'B': 27.2}]
bucket, final_image = TickStore._to_bucket(data, symbol, initial_image)
assert bucket[COUNT] == 2
assert bucket[END] == dt(2014, 1, 1, 0, 2, tzinfo=mktz(tz))
assert set(bucket[COLUMNS]) == set(('A', 'B', 'D'))
assert set(bucket[COLUMNS]['A']) == set((ROWMASK, DTYPE, DATA))
assert get_coldata(bucket[COLUMNS]['A']) == ([124, 125], [1, 1, 0, 0, 0, 0, 0, 0])
assert get_coldata(bucket[COLUMNS]['B']) == ([27.2], [0, 1, 0, 0, 0, 0, 0, 0])
assert get_coldata(bucket[COLUMNS]['D']) == ([0], [1, 0, 0, 0, 0, 0, 0, 0])
index = [dt.fromtimestamp(int(i/1000)).replace(tzinfo=mktz(tz)) for i in
list(np.cumsum(np.fromstring(lz4.decompress(bucket[INDEX]), dtype='uint64')))]
assert index == [i['index'] for i in data]
assert bucket[COLUMNS]['A'][DTYPE] == 'int64'
assert bucket[COLUMNS]['B'][DTYPE] == 'float64'
assert bucket[SYMBOL] == symbol
assert bucket[START] == initial_image['index']
assert bucket[IMAGE_DOC][IMAGE] == initial_image
assert bucket[IMAGE_DOC] == {IMAGE: initial_image,
IMAGE_TIME: initial_image['index']}
assert final_image == {'index': data[-1]['index'], 'A': 125, 'B': 27.2, 'C': 'DESC', 'D': 0}
def test_read_ts_raw(bitemporal_library):
bitemporal_library.update('spam', ts1, as_of=dt(2015, 5, 1, tzinfo=mktz('UTC')))
assert_frame_equal(bitemporal_library.read('spam', raw=True).data.tz_convert(tz=mktz('UTC'), level=1), read_str_as_pandas(
""" sample_dt | observed_dt | near
2012-09-08 17:06:11.040 | 2015-05-01 | 1.0
2012-10-08 17:06:11.040 | 2015-05-01 | 2.0
2012-10-09 17:06:11.040 | 2015-05-01 | 2.5
2012-11-08 17:06:11.040 | 2015-05-01 | 3.0""", num_index=2).tz_localize(tz=mktz('UTC'), level=1))
def test_bitemporal_store_saves_as_of_with_timezone(bitemporal_library):
local_tz = mktz()
bitemporal_library.update('spam', ts1, as_of=dt(2015, 5, 1))
df = bitemporal_library.read('spam', raw=True).data
assert all([
x[1].replace(tzinfo=mktz('UTC')) == dt(2015, 5, 1, tzinfo=local_tz)
for x in df.index
])
def test_read_ts_raw(bitemporal_library):
bitemporal_library.update('spam', ts1, as_of=dt(2015, 5, 1, tzinfo=mktz('UTC')))
assert_frame_equal(bitemporal_library.read('spam', raw=True).data.tz_convert(tz=mktz('UTC'), level=1), read_str_as_pandas(
""" sample_dt | observed_dt | near
2012-09-08 17:06:11.040 | 2015-05-01 | 1.0
2012-10-08 17:06:11.040 | 2015-05-01 | 2.0
2012-10-09 17:06:11.040 | 2015-05-01 | 2.5
2012-11-08 17:06:11.040 | 2015-05-01 | 3.0""", num_index=2).tz_localize(tz=mktz('UTC'), level=1))
def test_tickstore_to_bucket_no_image():
symbol = 'SYM'
data = [{'index': dt(2014, 1, 1, 0, 1, tzinfo=mktz()), 'A': 124, 'D': 0},
{'index': dt(2014, 1, 1, 0, 2, tzinfo=mktz()), 'A': 125, 'B': 27.2}]
bucket, final_image = TickStore._to_bucket(data, symbol, None)
assert bucket[COUNT] == 2
assert bucket[END] == dt(2014, 1, 1, 0, 2, tzinfo=mktz())
assert bucket[SYMBOL] == symbol
assert bucket[START] == dt(2014, 1, 1, 0, 1, tzinfo=mktz())
assert 'A' in bucket[COLUMNS]
assert IMAGE_DOC not in bucket
assert not final_image
def test_tickstore_to_bucket_no_image():
symbol = 'SYM'
data = [{'index': dt(2014, 1, 1, 0, 1, tzinfo=mktz()), 'A': 124, 'D': 0},
{'index': dt(2014, 1, 1, 0, 2, tzinfo=mktz()), 'A': 125, 'B': 27.2}]
bucket, final_image = TickStore._to_bucket(data, symbol, None)
assert bucket[COUNT] == 2
assert bucket[END] == dt(2014, 1, 1, 0, 2, tzinfo=mktz())
assert bucket[SYMBOL] == symbol
assert bucket[START] == dt(2014, 1, 1, 0, 1, tzinfo=mktz())
assert 'A' in bucket[COLUMNS]
assert IMAGE_DOC not in bucket
assert not final_image
def test_tickstore_pandas_to_bucket_image():
symbol = 'SYM'
tz = 'UTC'
initial_image = {'index': dt(2014, 1, 1, 0, 0, tzinfo=mktz(tz)), 'A': 123, 'B': 54.4, 'C': 'DESC'}
data = [{'A': 120, 'D': 1}, {'A': 122, 'B': 2.0}, {'A': 3, 'B': 3.0, 'D': 1}]
tick_index = [dt(2014, 1, 2, 0, 0, tzinfo=mktz(tz)),
dt(2014, 1, 3, 0, 0, tzinfo=mktz(tz)),
dt(2014, 1, 4, 0, 0, tzinfo=mktz(tz))]
data = pd.DataFrame(data, index=tick_index)
bucket, final_image = TickStore._pandas_to_bucket(data, symbol, initial_image)
assert final_image == {'index': dt(2014, 1, 4, 0, 0, tzinfo=mktz(tz)), 'A': 3, 'B': 3.0, 'C': 'DESC', 'D': 1}
assert IMAGE_DOC in bucket
assert bucket[COUNT] == 3
assert bucket[START] == dt(2014, 1, 1, 0, 0, tzinfo=mktz(tz))
assert bucket[END] == dt(2014, 1, 4, 0, 0, tzinfo=mktz(tz))
assert set(bucket[COLUMNS]) == set(('A', 'B', 'D'))
assert set(bucket[COLUMNS]['A']) == set((ROWMASK, DTYPE, DATA))
assert get_coldata(bucket[COLUMNS]['A']) == ([120, 122, 3], [1, 1, 1, 0, 0, 0, 0, 0])
values, rowmask = get_coldata(bucket[COLUMNS]['B'])
assert np.isnan(values[0]) and values[1:] == [2.0, 3.0]
assert rowmask == [1, 1, 1, 0, 0, 0, 0, 0]
values, rowmask = get_coldata(bucket[COLUMNS]['D'])
assert np.isnan(values[1])
assert values[0] == 1 and values[2] == 1
assert rowmask == [1, 1, 1, 0, 0, 0, 0, 0]
index = [dt.fromtimestamp(int(i/1000)).replace(tzinfo=mktz(tz)) for i in
list(np.cumsum(np.frombuffer(decompress(bucket[INDEX]), dtype='uint64')))]
assert index == tick_index
assert bucket[COLUMNS]['A'][DTYPE] == 'int64'
assert bucket[COLUMNS]['B'][DTYPE] == 'float64'
assert bucket[SYMBOL] == symbol
assert bucket[IMAGE_DOC] == {IMAGE: initial_image,
IMAGE_TIME: initial_image['index']}
def test_tickstore_pandas_to_bucket_image():
symbol = 'SYM'
tz = 'UTC'
initial_image = {'index': dt(2014, 1, 1, 0, 0, tzinfo=mktz(tz)), 'A': 123, 'B': 54.4, 'C': 'DESC'}
data = [{'A': 120, 'D': 1}, {'A': 122, 'B': 2.0}, {'A': 3, 'B': 3.0, 'D': 1}]
tick_index = [dt(2014, 1, 2, 0, 0, tzinfo=mktz(tz)),
dt(2014, 1, 3, 0, 0, tzinfo=mktz(tz)),
dt(2014, 1, 4, 0, 0, tzinfo=mktz(tz))]
data = pd.DataFrame(data, index=tick_index)
bucket, final_image = TickStore._pandas_to_bucket(data, symbol, initial_image)
assert final_image == {'index': dt(2014, 1, 4, 0, 0, tzinfo=mktz(tz)), 'A': 3, 'B': 3.0, 'C': 'DESC', 'D': 1}
assert IMAGE_DOC in bucket
assert bucket[COUNT] == 3
assert bucket[START] == dt(2014, 1, 1, 0, 0, tzinfo=mktz(tz))
assert bucket[END] == dt(2014, 1, 4, 0, 0, tzinfo=mktz(tz))
assert set(bucket[COLUMNS]) == set(('A', 'B', 'D'))
assert set(bucket[COLUMNS]['A']) == set((ROWMASK, DTYPE, DATA))
assert get_coldata(bucket[COLUMNS]['A']) == ([120, 122, 3], [1, 1, 1, 0, 0, 0, 0, 0])
values, rowmask = get_coldata(bucket[COLUMNS]['B'])
assert np.isnan(values[0]) and values[1:] == [2.0, 3.0]
assert rowmask == [1, 1, 1, 0, 0, 0, 0, 0]
values, rowmask = get_coldata(bucket[COLUMNS]['D'])
assert np.isnan(values[1])
assert values[0] == 1 and values[2] == 1
assert rowmask == [1, 1, 1, 0, 0, 0, 0, 0]
index = [dt.fromtimestamp(int(i/1000)).replace(tzinfo=mktz(tz)) for i in
list(np.cumsum(np.fromstring(lz4.decompress(bucket[INDEX]), dtype='uint64')))]
assert index == tick_index
assert bucket[COLUMNS]['A'][DTYPE] == 'int64'
assert bucket[COLUMNS]['B'][DTYPE] == 'float64'
assert bucket[SYMBOL] == symbol
assert bucket[IMAGE_DOC] == {IMAGE: initial_image,
IMAGE_TIME: initial_image['index']}
def update(self, symbol, data, metadata=None, upsert=True, as_of=None, **kwargs):
""" Append 'data' under the specified 'symbol' name to this library.
Parameters
----------
symbol : `str`
symbol name for the item
data : `pd.DataFrame`
to be persisted
metadata : `dict`
An optional dictionary of metadata to persist along with the symbol. If None and there are existing
metadata, current metadata will be maintained
upsert : `bool`
Write 'data' if no previous version exists.
as_of : `datetime.datetime`
The "insert time". Default to datetime.now()
"""
local_tz = mktz()
if not as_of:
as_of = dt.now()
if as_of.tzinfo is None:
as_of = as_of.replace(tzinfo=local_tz)
data = self._add_observe_dt_index(data, as_of)
if upsert and not self._store.has_symbol(symbol):
df = data
else:
existing_item = self._store.read(symbol, **kwargs)
if metadata is None:
metadata = existing_item.metadata
df = existing_item.data.append(data).sort_index()
self._store.write(symbol, df, metadata=metadata, prune_previous_version=True)
def test_mongo_date_range_query():
self = create_autospec(TickStore)
self._collection = create_autospec(Collection)
self._symbol_query.return_value = {"sy": { "$in" : [ "s1" , "s2"]}}
self._collection.aggregate.return_value = iter([{"_id": "s1", "start": dt(2014, 1, 1, 0, 0, tzinfo=mktz())}])
query = TickStore._mongo_date_range_query(self, 'sym', DateRange(dt(2014, 1, 2, 0, 0, tzinfo=mktz()),
dt(2014, 1, 3, 0, 0, tzinfo=mktz())))
assert self._collection.aggregate.call_args_list == [call([
{"$match": {"s": {"$lte": dt(2014, 1, 2, 0, 0, tzinfo=mktz())}, "sy": { "$in" : [ "s1" , "s2"]}}},
{"$project": {"_id": 0, "s": 1, "sy": 1}},
{"$group": {"_id": "$sy", "start": {"$max": "$s"}}},
{"$sort": {"start": 1}},
{"$limit": 1}])]
assert query == {'s': {'$gte': dt(2014, 1, 1, 0, 0, tzinfo=mktz()), '$lte': dt(2014, 1, 3, 0, 0, tzinfo=mktz())}}
def test_read_ts_raw_all_version_ok(bitemporal_library):
bitemporal_library.update("spam", ts1, as_of=dt(2015, 5, 1, tzinfo=mktz("UTC")))
bitemporal_library.update(
"spam",
read_str_as_pandas(
""" sample_dt | near
2012-12-01 17:06:11.040 | 25"""
),
as_of=dt(2015, 5, 5, tzinfo=mktz("UTC")),
)
bitemporal_library.update(
"spam",
read_str_as_pandas(
""" sample_dt | near
2012-11-08 17:06:11.040 | 42"""
),
as_of=dt(2015, 5, 3, tzinfo=mktz("UTC")),
)
bitemporal_library.update(
"spam",
read_str_as_pandas(
""" sample_dt | near
2012-10-08 17:06:11.040 | 42
2013-01-01 17:06:11.040 | 100"""
),
as_of=dt(2015, 5, 10, tzinfo=mktz("UTC")),
)
assert_frame_equal(
bitemporal_library.read("spam", raw=True).data.tz_localize(tz=None, level=1),
read_str_as_pandas(
""" sample_dt | observed_dt | near
2012-09-08 17:06:11.040 | 2015-05-01 | 1.0
2012-10-08 17:06:11.040 | 2015-05-01 | 2.0
2012-10-08 17:06:11.040 | 2015-05-10 | 42
2012-10-09 17:06:11.040 | 2015-05-01 | 2.5
2012-11-08 17:06:11.040 | 2015-05-01 | 3.0
2012-11-08 17:06:11.040 | 2015-05-03 | 42
2012-12-01 17:06:11.040 | 2015-05-05 | 25
2013-01-01 17:06:11.040 | 2015-05-10 | 100""",
num_index=2,
),
)
def test_mongo_date_range_query():
self = create_autospec(TickStore)
self._collection = create_autospec(Collection)
self._collection.find_one.return_value = {"s": sentinel.start}
self._symbol_query = partial(TickStore._symbol_query, self)
query = TickStore._mongo_date_range_query(
self, "sym", DateRange(dt(2014, 1, 1, 0, 0, tzinfo=mktz()), dt(2014, 1, 2, 0, 0, tzinfo=mktz()))
)
assert self._collection.find_one.call_args_list == [
call(
{"sy": "sym", "s": {"$lte": dt(2014, 1, 1, 0, 0, tzinfo=mktz())}},
sort=[("s", -1)],
projection={"s": 1, "_id": 0},
),
call(
{"sy": "sym", "s": {"$gt": dt(2014, 1, 2, 0, 0, tzinfo=mktz())}},
sort=[("s", 1)],
projection={"s": 1, "_id": 0},
),
]
assert query == {"s": {"$gte": sentinel.start, "$lt": sentinel.start}}
def test_read_ts_raw_all_version_ok(bitemporal_library):
bitemporal_library.update('spam', ts1, as_of=dt(2015, 5, 1, tzinfo=mktz('UTC')))
bitemporal_library.update('spam', read_str_as_pandas(""" sample_dt | near
2012-12-01 17:06:11.040 | 25"""),
as_of=dt(2015, 5, 5, tzinfo=mktz('UTC')))
bitemporal_library.update('spam', read_str_as_pandas(""" sample_dt | near
2012-11-08 17:06:11.040 | 42"""),
as_of=dt(2015, 5, 3, tzinfo=mktz('UTC')))
bitemporal_library.update('spam', read_str_as_pandas(""" sample_dt | near
2012-10-08 17:06:11.040 | 42
2013-01-01 17:06:11.040 | 100"""),
as_of=dt(2015, 5, 10, tzinfo=mktz('UTC')))
assert_frame_equal(bitemporal_library.read('spam', raw=True).data, read_str_as_pandas(
""" sample_dt | observed_dt | near
2012-09-08 17:06:11.040 | 2015-05-01 | 1.0
2012-10-08 17:06:11.040 | 2015-05-01 | 2.0
2012-10-08 17:06:11.040 | 2015-05-10 | 42
2012-10-09 17:06:11.040 | 2015-05-01 | 2.5
2012-11-08 17:06:11.040 | 2015-05-01 | 3.0
2012-11-08 17:06:11.040 | 2015-05-03 | 42
2012-12-01 17:06:11.040 | 2015-05-05 | 25
2013-01-01 17:06:11.040 | 2015-05-10 | 100""", num_index=2))
def test_list_version_latest_only(library):
assert len(list(library.list_versions(symbol))) == 0
dates = [None, None, None]
now = dt.utcnow().replace(tzinfo=mktz('UTC'))
for x in six.moves.xrange(len(dates)):
dates[x] = now - dtd(minutes=20 - x)
with patch("bson.ObjectId", return_value=bson.ObjectId.from_datetime(dates[x])):
library.write(symbol, ts1, prune_previous_version=False)
assert len(list(library.list_versions(symbol))) == 3
library.write(symbol, ts1, prune_previous_version=True)
assert len(list(library.list_versions(symbol, latest_only=True))) == 1
versions = list(library.list_versions(symbol))
for i, x in enumerate([4, ]):
assert versions[i]['symbol'] == symbol
assert versions[i]['date'] >= dates[i]
assert versions[i]['version'] == x
def test_list_version_latest_only(library):
assert len(list(library.list_versions(symbol))) == 0
dates = [None, None, None]
now = dt.utcnow().replace(tzinfo=mktz('UTC'))
for x in six.moves.xrange(len(dates)):
dates[x] = now - dtd(minutes=20 - x)
with patch("bson.ObjectId", return_value=bson.ObjectId.from_datetime(dates[x])):
library.write(symbol, ts1, prune_previous_version=False)
assert len(list(library.list_versions(symbol))) == 3
library.write(symbol, ts1, prune_previous_version=True)
assert len(list(library.list_versions(symbol, latest_only=True))) == 1
versions = list(library.list_versions(symbol))
for i, x in enumerate([4, ]):
assert versions[i]['symbol'] == symbol
assert versions[i]['date'] >= dates[i]
assert versions[i]['version'] == x
def test_mongo_date_range_query():
self = create_autospec(TickStore)
self._collection = create_autospec(Collection)
self._collection.find_one.return_value = {'s': sentinel.start}
self._symbol_query = partial(TickStore._symbol_query, self)
query = TickStore._mongo_date_range_query(self, 'sym', DateRange(dt(2014, 1, 1, 0, 0, tzinfo=mktz()),
dt(2014, 1, 2, 0, 0, tzinfo=mktz())))
assert self._collection.find_one.call_args_list == [call({'sy': 'sym', 's': {'$lte': dt(2014, 1, 1, 0, 0, tzinfo=mktz())}},
sort=[('s', -1)], projection={'s': 1, '_id': 0}),
call({'sy': 'sym', 's': {'$gt': dt(2014, 1, 2, 0, 0, tzinfo=mktz())}},
sort=[('s', 1)], projection={'s': 1, '_id': 0})]
assert query == {'s': {'$gte': sentinel.start, '$lt': sentinel.start}}
pytest_plugins = ['arctic.fixtures.arctic']
ts1 = read_str_as_pandas(""" sample_dt | near
2012-09-08 17:06:11.040 | 1.0
2012-10-08 17:06:11.040 | 2.0
2012-10-09 17:06:11.040 | 2.5
2012-11-08 17:06:11.040 | 3.0""")
ts1_update = read_str_as_pandas(""" sample_dt | near
2012-09-08 17:06:11.040 | 1.0
2012-10-08 17:06:11.040 | 2.0
2012-10-09 17:06:11.040 | 2.5
2012-11-08 17:06:11.040 | 3.0
2012-11-09 17:06:11.040 | 3.5""")
LOCAL_TZ = mktz()
def test_new_ts_read_write(bitemporal_library):
bitemporal_library.update('spam', ts1)
assert_frame_equal(ts1, bitemporal_library.read('spam').data)
def test_read_ts_raw(bitemporal_library):
bitemporal_library.update('spam',
ts1,
as_of=dt(2015, 5, 1, tzinfo=mktz('UTC')))
assert_frame_equal(
bitemporal_library.read('spam', raw=True).data,
read_str_as_pandas(
""" sample_dt | observed_dt | near
ts1 = read_str_as_pandas(""" sample_dt | near
2012-09-08 17:06:11.040 | 1.0
2012-10-08 17:06:11.040 | 2.0
2012-10-09 17:06:11.040 | 2.5
2012-11-08 17:06:11.040 | 3.0""")
ts1_update = read_str_as_pandas(""" sample_dt | near
2012-09-08 17:06:11.040 | 1.0
2012-10-08 17:06:11.040 | 2.0
2012-10-09 17:06:11.040 | 2.5
2012-11-08 17:06:11.040 | 3.0
2012-11-09 17:06:11.040 | 3.5""")
LOCAL_TZ = mktz()
def test_new_ts_read_write(bitemporal_library):
bitemporal_library.update('spam', ts1)
assert_frame_equal(ts1, bitemporal_library.read('spam').data)
def test_read_ts_raw(bitemporal_library):
bitemporal_library.update('spam', ts1, as_of=dt(2015, 5, 1, tzinfo=mktz('UTC')))
assert_frame_equal(bitemporal_library.read('spam', raw=True).data.tz_convert(tz=mktz('UTC'), level=1), read_str_as_pandas(
""" sample_dt | observed_dt | near
2012-09-08 17:06:11.040 | 2015-05-01 | 1.0
2012-10-08 17:06:11.040 | 2015-05-01 | 2.0
2012-10-09 17:06:11.040 | 2015-05-01 | 2.5
2012-11-08 17:06:11.040 | 2015-05-01 | 3.0""", num_index=2).tz_localize(tz=mktz('UTC'), level=1))
def test_bitemporal_store_saves_as_of_with_timezone(bitemporal_library):
local_tz = mktz()
bitemporal_library.update('spam', ts1, as_of=dt(2015, 5, 1))
df = bitemporal_library.read('spam', raw=True).data
assert all([x[1].replace(tzinfo=mktz('UTC')) == dt(2015, 5, 1, tzinfo=local_tz) for x in df.index])
