def test_corrupted_read_writes_new(library):
with ArcticTransaction(library, symbol, 'u1', 'l1') as mt:
mt.write(symbol, ts1)
res = library.read(symbol)
assert res.version == 1
with ArcticTransaction(library, symbol, 'u1', 'l2') as mt:
mt.write(symbol, ts2)
res = library.read(symbol)
assert res.version == 2
with patch.object(library, 'read') as l:
l.side_effect = OperationFailure('some failure')
with ArcticTransaction(library, symbol, 'u1', 'l2') as mt:
mt.write(symbol, ts3, metadata={'a': 1, 'b': 2})
res = library.read(symbol)
# Corrupted data still increments on write to next version correctly with new data
assert res.version == 3
assert_frame_equal(ts3, library.read(symbol, 3).data)
assert res.metadata == {'a': 1, 'b': 2}
with patch.object(library, 'read') as l:
l.side_effect = OperationFailure('some failure')
with ArcticTransaction(library, symbol, 'u1', 'l2') as mt:
mt.write(symbol, ts3, metadata={'a': 1, 'b': 2})
res = library.read(symbol)
# Corrupted data still increments to next version correctly with ts & metadata unchanged
assert res.version == 4
assert_frame_equal(ts3, library.read(symbol, 4).data)
assert res.metadata == {'a': 1, 'b': 2}
def test_metadata_changes_writes(library):
with ArcticTransaction(library, symbol, 'u1', 'l1') as mt:
mt.write(symbol, ts1, metadata={'original': 'data'})
with ArcticTransaction(library, symbol, 'u2', 'l2') as mt:
mt.write(symbol, ts1, metadata={'some': 'data', 'original': 'data'})
audit_log = library.read_audit_log(symbol)
assert audit_log == [{
u'new_v': 2,
u'symbol': u'TS1',
u'message': u'l2',
u'user': u'u2',
u'orig_v': 1
}, {
u'new_v': 1,
u'symbol': u'TS1',
u'message': u'l1',
u'user': u'u1',
u'orig_v': 0
}]
assert_frame_equal(ts1, library.read(symbol, audit_log[0]['orig_v']).data)
assert_frame_equal(ts1, library.read(symbol, audit_log[0]['new_v']).data)
assert library.read(symbol, audit_log[0]['orig_v']).metadata == {
'original': 'data'
}
assert library.read(symbol, audit_log[0]['new_v']).metadata == {
'some': 'data',
'original': 'data'
}
def test_write_after_delete(library):
with ArcticTransaction(library, symbol, 'u1', 'l') as mt:
mt.write(symbol, ts1)
library.delete(symbol)
with ArcticTransaction(library, symbol, 'u1', 'l') as mt:
mt.write(symbol, ts1_append)
assert_frame_equal(library.read(symbol).data, ts1_append)
def test_ConcurrentWriteBlock_simple():
vs = create_autospec(VersionStore, _collection=Mock())
ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]})
vs.read.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=1,
metadata=None,
data=ts1)
vs.write.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=2,
metadata=None,
data=None)
vs.list_versions.return_value = [{'version': 2}, {'version': 1}]
with ArcticTransaction(vs, sentinel.symbol, sentinel.user,
sentinel.log) as cwb:
cwb.write(sentinel.symbol,
pd.DataFrame(index=[3, 4], data={'a': [1.0, 2.0]}),
metadata=sentinel.meta)
assert not vs._delete_version.called
vs.write.assert_called_once_with(sentinel.symbol,
ANY,
prune_previous_version=True,
metadata=sentinel.meta)
vs.list_versions.assert_called_once_with(sentinel.symbol)
def _copy_symbol(symbols):
for symbol in symbols:
with ArcticTransaction(dest, symbol, USER, log) as mt:
existing_data = dest.has_symbol(symbol)
if existing_data:
if force:
logger.warn("Symbol: %s already exists in destination, OVERWRITING" % symbol)
elif splice:
logger.warn("Symbol: %s already exists in destination, splicing in new data" % symbol)
else:
logger.warn("Symbol: {} already exists in {}@{}, use --force to overwrite or --splice to join "
"with existing data".format(symbol, _get_host(dest).get('l'),
_get_host(dest).get('mhost')))
continue
version = src.read(symbol)
new_data = version.data
if existing_data and splice:
original_data = dest.read(symbol).data
preserve_start = to_pandas_closed_closed(DateRange(None, new_data.index[0].to_pydatetime(),
interval=CLOSED_OPEN)).end
preserve_end = to_pandas_closed_closed(DateRange(new_data.index[-1].to_pydatetime(),
None,
interval=OPEN_CLOSED)).start
if not original_data.index.tz:
# No timezone on the original, should we even allow this?
preserve_start = preserve_start.replace(tzinfo=None)
preserve_end = preserve_end.replace(tzinfo=None)
before = original_data.loc[:preserve_start]
after = original_data[preserve_end:]
new_data = before.append(new_data).append(after)
mt.write(symbol, new_data, metadata=version.metadata)
def losing_writer():
#will attempt to write version 2, should find that version 2 is there and it ends up writing version 3
with pytest.raises(ArcticTransaction):
with ArcticTransaction(vs, sentinel.symbol, sentinel.user,
sentinel.log) as cwb:
cwb.write(sentinel.symbol, pd.DataFrame([1.0, 2.0], [3, 4]))
e1.wait()
def test_ArcticTransaction_guards_against_inconsistent_ts():
vs = Mock(spec=VersionStore)
ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]})
vs.read.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=1,
metadata=None,
data=ts1,
host=sentinel.host)
vs.write.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=2,
metadata=None,
data=None,
host=sentinel.host)
vs.list_versions.side_effect = [{'version': 2}, {'version': 1}]
ts1 = pd.DataFrame(index=[1, 2], data={'a': [2.0, 3.0]})
with pytest.raises(ConcurrentModificationException):
with ArcticTransaction(vs,
sentinel.symbol,
sentinel.user,
sentinel.log,
modify_timeseries=ts1) as cwb:
pass
def test_ArticTransaction_no_audit():
vs = Mock(spec=VersionStore)
ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]})
vs.read.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=1,
metadata=None,
data=ts1)
vs.write.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=2,
metadata=None,
data=None)
vs.list_versions.return_value = [{'version': 2}, {'version': 1}]
with ArcticTransaction(vs,
sentinel.symbol,
sentinel.user,
sentinel.log,
audit=False) as cwb:
cwb.write(sentinel.symbol,
pd.DataFrame(index=[3, 4], data={'a': [1.0, 2.0]}),
metadata=sentinel.meta)
assert vs.write.call_count == 1
assert vs._write_audit.call_count == 0
def test_ArcticTransaction_simple():
vs = Mock(spec=VersionStore)
ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]})
vs.read.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=1,
metadata=None,
data=ts1)
vs.write.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=2,
metadata=None,
data=None)
vs.list_versions.return_value = [{'version': 2}, {'version': 1}]
with ArcticTransaction(vs, sentinel.symbol, sentinel.user,
sentinel.log) as cwb:
cwb.write(sentinel.symbol,
pd.DataFrame(index=[3, 4], data={'a': [1.0, 2.0]}),
metadata=sentinel.meta)
assert not vs._delete_version.called
assert vs.write.call_args_list == [
call(sentinel.symbol,
ANY,
prune_previous_version=True,
metadata=sentinel.meta)
]
assert vs.list_versions.call_args_list == [call(sentinel.symbol)]
assert vs._write_audit.call_args_list == [
call(sentinel.user, sentinel.log, ANY)
]
def test_ArcticTransaction_writes_no_data_found():
vs = Mock(spec=VersionStore)
ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]})
vs.read.side_effect = NoDataFoundException('no data')
vs.write.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=1,
metadata=None,
data=None)
vs.list_versions.side_effect = [
[],
[{
'version': 1
}],
]
with ArcticTransaction(vs, sentinel.symbol, sentinel.user,
sentinel.log) as cwb:
cwb.write(sentinel.symbol, ts1, metadata={1: 2})
assert vs.write.call_args_list == [
call(sentinel.symbol,
ANY,
prune_previous_version=True,
metadata={1: 2})
]
assert vs.list_versions.call_args_list == [
call(sentinel.symbol, latest_only=True),
call(sentinel.symbol)
]
def test_ArcticTransaction_writes_if_base_data_corrupted():
vs = Mock(spec=VersionStore)
ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]})
vs.read.side_effect = OperationFailure('some failure')
vs.write.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=2,
metadata=None,
data=None)
vs.read_metadata.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=1,
metadata=None,
data=None)
vs.list_versions.return_value = [{'version': 2}, {'version': 1}]
with ArcticTransaction(vs, sentinel.symbol, sentinel.user,
sentinel.log) as cwb:
cwb.write(sentinel.symbol, ts1, metadata={1: 2})
vs.write.assert_called_once_with(sentinel.symbol,
ANY,
prune_previous_version=True,
metadata={1: 2})
assert vs.list_versions.call_args_list == [call(sentinel.symbol)]
def test_ArcticTransaction_write_skips_for_exact_match(library):
ts = read_str_as_pandas("""times | PX_LAST
2014-10-31 21:30:00.000 | 204324.674
2014-11-13 21:30:00.000 | 193964.45
2014-11-14 21:30:00.000 | 193650.403""")
with ArcticTransaction(library, symbol, 'u1', 'l1') as mt:
mt.write(symbol, ts)
version = library.read(symbol).version
# try and store same TimeSeries again
with ArcticTransaction(library, symbol, 'u1', 'l2') as mt:
mt.write(symbol, ts)
assert library.read(symbol).version == version
def test_cleanup_orphaned_versions_integration(library):
_id = ObjectId.from_datetime(dt(2013, 1, 1))
with patch('bson.ObjectId', return_value=_id):
with ArcticTransaction(library, symbol, 'u1', 'l1') as mt:
mt.write(symbol, ts1)
assert mongo_count(library._versions, filter={'parent': {'$size': 1}}) == 1
library._cleanup_orphaned_versions(False)
assert mongo_count(library._versions, filter={'parent': {'$size': 1}}) == 1
def test_ArcticTransaction_writes_if_metadata_changed():
vs = Mock(spec=VersionStore)
ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]})
vs.read.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=1,
metadata=None,
data=ts1,
host=sentinel.host)
vs.write.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=2,
metadata=None,
data=None,
host=sentinel.host)
vs.list_versions.return_value = [{'version': 2}, {'version': 1}]
with ArcticTransaction(vs, sentinel.symbol, sentinel.user,
sentinel.log) as cwb:
assert cwb._do_write is False
cwb.write(sentinel.symbol, ts1, metadata={1: 2})
assert cwb._do_write is True
assert not vs._delete_version.called
vs.write.assert_called_once_with(sentinel.symbol,
ANY,
prune_previous_version=True,
metadata={1: 2})
vs.list_versions.assert_called_once_with(sentinel.symbol)
# Won't write on exit with same data and metadata
vs.read.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=2,
metadata={1: 2},
data=ts1,
host=sentinel.host)
with ArcticTransaction(vs, sentinel.symbol, sentinel.user,
sentinel.log) as cwb:
assert cwb._do_write is False
cwb.write(sentinel.symbol, ts1, metadata={1: 2})
assert cwb._do_write is False
def test_ArcticTransaction_write_doesnt_skip_for_close_ts(library):
orig_ts = read_str_as_pandas("""times | PX_LAST
2014-10-31 21:30:00.000 | 204324.674
2014-11-13 21:30:00.000 | 193964.45
2014-11-14 21:30:00.000 | 193650.403""")
with ArcticTransaction(library, symbol, 'u1', 'l1') as mt:
mt.write(symbol, orig_ts)
assert_frame_equal(library.read(symbol).data, orig_ts)
# try and store slighty different TimeSeries
new_ts = read_str_as_pandas("""times | PX_LAST
2014-10-31 21:30:00.000 | 204324.672
2014-11-13 21:30:00.000 | 193964.453
2014-11-14 21:30:00.000 | 193650.406""")
with ArcticTransaction(library, symbol, 'u1', 'l2') as mt:
mt.write(symbol, new_ts)
assert_frame_equal(library.read(symbol).data, new_ts)
def test_audit_writes(library):
with ArcticTransaction(library, symbol, 'u1', 'l1') as mt:
mt.write(symbol, ts1)
with ArcticTransaction(library, symbol, 'u2', 'l2') as mt:
mt.write(symbol, ts2)
audit_log = library.read_audit_log(symbol)
assert audit_log == [{
u'new_v': 2,
u'symbol': u'TS1',
u'message': u'l2',
u'user': u'u2',
u'orig_v': 1
}, {
u'new_v': 1,
u'symbol': u'TS1',
u'message': u'l1',
u'user': u'u1',
u'orig_v': 0
}]
assert_frame_equal(ts1, library.read(symbol, audit_log[0]['orig_v']).data)
assert_frame_equal(ts2, library.read(symbol, audit_log[0]['new_v']).data)
def test_ArcticTransaction_does_nothing_when_data_is_None():
vs = Mock(spec=VersionStore)
ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]})
vs.read.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=1,
metadata=None,
data=ts1)
vs.write.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=2,
metadata=None,
data=None)
vs.list_versions.return_value = [{'version': 1}, {'version': 2}]
with ArcticTransaction(vs, sentinel.symbol, sentinel.user,
sentinel.log) as cwb:
pass
assert not vs._delete_version.called
assert not vs.write.called
def test_ArcticTransaction_does_nothing_when_data_not_modified():
vs = create_autospec(VersionStore, _collection=Mock())
ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]})
vs.read.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=1,
metadata=None,
data=ts1)
vs.write.return_value = VersionedItem(symbol=sentinel.symbol,
library=sentinel.library,
version=2,
metadata=None,
data=None)
vs.list_versions.side_effect = [{'version': 2}, {'version': 1}]
with ArcticTransaction(vs, sentinel.symbol, sentinel.user,
sentinel.log) as cwb:
cwb.write(sentinel.symbol,
pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]}))
assert not vs._delete_version.called
assert not vs.write.called
def _copy_symbol(symbols):
for symbol in symbols:
with ArcticTransaction(dest, symbol, USER, log) as mt:
existing_data = dest.has_symbol(symbol)
if existing_data:
if force:
logger.warn(
"Symbol: %s already exists in destination, OVERWRITING"
% symbol)
elif splice:
logger.warn(
"Symbol: %s already exists in destination, splicing in new data"
% symbol)
else:
logger.warn(
"Symbol: {} already exists in {}@{}, use --force to overwrite or --splice to join with existing data"
.format(symbol,
_get_host(dest).get('l'),
_get_host(dest).get('mhost')))
continue
version = src.read(symbol)
new_data = version.data
if existing_data and splice:
original_data = dest.read(symbol).data
before = original_data.ix[:to_pandas_closed_closed(
DateRange(None,
new_data.index[0].to_pydatetime(),
interval=CLOSED_OPEN)).end]
after = original_data.ix[to_pandas_closed_closed(
DateRange(new_data.index[-1].to_pydatetime(),
None,
interval=OPEN_CLOSED)).start:]
new_data = before.append(new_data).append(after)
mt.write(symbol, new_data, metadata=version.metadata)
def test_audit_read(library):
with ArcticTransaction(library, symbol3, 'u3', 'foo') as mt:
mt.write(symbol3, ts1)
with ArcticTransaction(library, symbol, 'u1', 'l1') as mt:
mt.write(symbol, ts1)
with ArcticTransaction(library, symbol, 'u2', 'l2') as mt:
mt.write(symbol, ts2)
with ArcticTransaction(library, symbol2, 'u2', 'l2') as mt:
mt.write(symbol2, ts2)
audit_log = library.read_audit_log()
assert audit_log == [
{
u'new_v': 1,
u'symbol': u'TS2',
u'message': u'l2',
u'user': u'u2',
u'orig_v': 0
},
{
u'new_v': 2,
u'symbol': u'TS1',
u'message': u'l2',
u'user': u'u2',
u'orig_v': 1
},
{
u'new_v': 1,
u'symbol': u'TS1',
u'message': u'l1',
u'user': u'u1',
u'orig_v': 0
},
{
u'new_v': 1,
u'symbol': u'TS3',
u'message': u'foo',
u'user': u'u3',
u'orig_v': 0
},
]
l2_audit_log = library.read_audit_log(message='l2')
assert l2_audit_log == [
{
u'new_v': 1,
u'symbol': u'TS2',
u'message': u'l2',
u'user': u'u2',
u'orig_v': 0
},
{
u'new_v': 2,
u'symbol': u'TS1',
u'message': u'l2',
u'user': u'u2',
u'orig_v': 1
},
]
symbol_audit_log = library.read_audit_log(symbol=symbol)
assert symbol_audit_log == [{
u'new_v': 2,
u'symbol': u'TS1',
u'message': u'l2',
u'user': u'u2',
u'orig_v': 1
}, {
u'new_v': 1,
u'symbol': u'TS1',
u'message': u'l1',
u'user': u'u1',
u'orig_v': 0
}]
symbols_audit_log = library.read_audit_log(symbol=[symbol, symbol2])
assert symbols_audit_log == [{
u'new_v': 1,
u'symbol': u'TS2',
u'message': u'l2',
u'user': u'u2',
u'orig_v': 0
}, {
u'new_v': 2,
u'symbol': u'TS1',
u'message': u'l2',
u'user': u'u2',
u'orig_v': 1
}, {
u'new_v': 1,
u'symbol': u'TS1',
u'message': u'l1',
u'user': u'u1',
u'orig_v': 0
}]
symbol_message_audit_log = library.read_audit_log(symbol=symbol,
message='l2')
assert symbol_message_audit_log == [
{
u'new_v': 2,
u'symbol': u'TS1',
u'message': u'l2',
u'user': u'u2',
u'orig_v': 1
},
]
def winning_writer():
#will attempt to write version 2 as well
with ArcticTransaction(library, 'FOO', 'user', 'log') as cwb:
cwb.write('FOO', ts2, metadata={'foo': 'bar'})
e2.wait()
def losing_writer():
#will attempt to write version 2, should find that version 2 is there and it ends up writing version 3
with pytest.raises(ConcurrentModificationException):
with ArcticTransaction(library, 'FOO', 'user', 'log') as cwb:
cwb.write('FOO', ts1_append, metadata={'foo': 'bar'})
e1.wait()
def test_ArcticTransaction_can_do_first_writes(library):
with ArcticTransaction(library, 'SYMBOL_NOT_HERE', 'user', 'log') as cwb:
cwb.write('SYMBOL_NOT_HERE', ts1)
wrote_vi = library.read('SYMBOL_NOT_HERE')
assert_frame_equal(wrote_vi.data, ts1)
def winning_writer():
#will attempt to write version 2 as well
with ArcticTransaction(vs, sentinel.symbol, sentinel.user,
sentinel.log) as cwb:
cwb.write(sentinel.symbol, pd.DataFrame([1.0, 2.0], [5, 6]))
e2.wait()
