def test_metadata_changes_writes(library): with ArcticTransaction(library, symbol, 'u1', 'l1') as mt: mt.write(symbol, ts1, metadata={'original': 'data'}) with ArcticTransaction(library, symbol, 'u2', 'l2') as mt: mt.write(symbol, ts1, metadata={'some': 'data', 'original': 'data'}) audit_log = library.read_audit_log(symbol) assert audit_log == [{ u'new_v': 2, u'symbol': u'TS1', u'message': u'l2', u'user': u'u2', u'orig_v': 1 }, { u'new_v': 1, u'symbol': u'TS1', u'message': u'l1', u'user': u'u1', u'orig_v': 0 }] assert_frame_equal(ts1, library.read(symbol, audit_log[0]['orig_v']).data) assert_frame_equal(ts1, library.read(symbol, audit_log[0]['new_v']).data) assert library.read(symbol, audit_log[0]['orig_v']).metadata == { 'original': 'data' } assert library.read(symbol, audit_log[0]['new_v']).metadata == { 'some': 'data', 'original': 'data' }
def test_corrupted_read_writes_new(library): with ArcticTransaction(library, symbol, 'u1', 'l1') as mt: mt.write(symbol, ts1) res = library.read(symbol) assert res.version == 1 with ArcticTransaction(library, symbol, 'u1', 'l2') as mt: mt.write(symbol, ts2) res = library.read(symbol) assert res.version == 2 with patch.object(library, 'read') as l: l.side_effect = OperationFailure('some failure') with ArcticTransaction(library, symbol, 'u1', 'l2') as mt: mt.write(symbol, ts3, metadata={'a': 1, 'b': 2}) res = library.read(symbol) # Corrupted data still increments on write to next version correctly with new data assert res.version == 3 assert_frame_equal(ts3, library.read(symbol, 3).data) assert res.metadata == {'a': 1, 'b': 2} with patch.object(library, 'read') as l: l.side_effect = OperationFailure('some failure') with ArcticTransaction(library, symbol, 'u1', 'l2') as mt: mt.write(symbol, ts3, metadata={'a': 1, 'b': 2}) res = library.read(symbol) # Corrupted data still increments to next version correctly with ts & metadata unchanged assert res.version == 4 assert_frame_equal(ts3, library.read(symbol, 4).data) assert res.metadata == {'a': 1, 'b': 2}
def test_write_after_delete(library): with ArcticTransaction(library, symbol, 'u1', 'l') as mt: mt.write(symbol, ts1) library.delete(symbol) with ArcticTransaction(library, symbol, 'u1', 'l') as mt: mt.write(symbol, ts1_append) assert_frame_equal(library.read(symbol).data, ts1_append)
def test_ArcticTransaction_writes_no_data_found(): vs = Mock(spec=VersionStore) ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]}) vs.read.side_effect = NoDataFoundException('no data') vs.write.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=1, metadata=None, data=None) vs.list_versions.side_effect = [ [], [{ 'version': 1 }], ] with ArcticTransaction(vs, sentinel.symbol, sentinel.user, sentinel.log) as cwb: cwb.write(sentinel.symbol, ts1, metadata={1: 2}) assert vs.write.call_args_list == [ call(sentinel.symbol, ANY, prune_previous_version=True, metadata={1: 2}) ] assert vs.list_versions.call_args_list == [ call(sentinel.symbol, latest_only=True), call(sentinel.symbol) ]
def test_ArcticTransaction_writes_if_base_data_corrupted(): vs = Mock(spec=VersionStore) ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]}) vs.read.side_effect = OperationFailure('some failure') vs.write.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=2, metadata=None, data=None) vs.read_metadata.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=1, metadata=None, data=None) vs.list_versions.return_value = [{'version': 2}, {'version': 1}] with ArcticTransaction(vs, sentinel.symbol, sentinel.user, sentinel.log) as cwb: cwb.write(sentinel.symbol, ts1, metadata={1: 2}) vs.write.assert_called_once_with(sentinel.symbol, ANY, prune_previous_version=True, metadata={1: 2}) assert vs.list_versions.call_args_list == [call(sentinel.symbol)]
def test_ArcticTransaction_simple(): vs = Mock(spec=VersionStore) ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]}) vs.read.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=1, metadata=None, data=ts1) vs.write.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=2, metadata=None, data=None) vs.list_versions.return_value = [{'version': 2}, {'version': 1}] with ArcticTransaction(vs, sentinel.symbol, sentinel.user, sentinel.log) as cwb: cwb.write(sentinel.symbol, pd.DataFrame(index=[3, 4], data={'a': [1.0, 2.0]}), metadata=sentinel.meta) assert not vs._delete_version.called assert vs.write.call_args_list == [ call(sentinel.symbol, ANY, prune_previous_version=True, metadata=sentinel.meta) ] assert vs.list_versions.call_args_list == [call(sentinel.symbol)] assert vs._write_audit.call_args_list == [ call(sentinel.user, sentinel.log, ANY) ]
def _copy_symbol(symbols): for symbol in symbols: with ArcticTransaction(dest, symbol, USER, log) as mt: existing_data = dest.has_symbol(symbol) if existing_data: if force: logger.warn("Symbol: %s already exists in destination, OVERWRITING" % symbol) elif splice: logger.warn("Symbol: %s already exists in destination, splicing in new data" % symbol) else: logger.warn("Symbol: {} already exists in {}@{}, use --force to overwrite or --splice to join " "with existing data".format(symbol, _get_host(dest).get('l'), _get_host(dest).get('mhost'))) continue version = src.read(symbol) new_data = version.data if existing_data and splice: original_data = dest.read(symbol).data preserve_start = to_pandas_closed_closed(DateRange(None, new_data.index[0].to_pydatetime(), interval=CLOSED_OPEN)).end preserve_end = to_pandas_closed_closed(DateRange(new_data.index[-1].to_pydatetime(), None, interval=OPEN_CLOSED)).start if not original_data.index.tz: # No timezone on the original, should we even allow this? preserve_start = preserve_start.replace(tzinfo=None) preserve_end = preserve_end.replace(tzinfo=None) before = original_data.loc[:preserve_start] after = original_data[preserve_end:] new_data = before.append(new_data).append(after) mt.write(symbol, new_data, metadata=version.metadata)
def test_ConcurrentWriteBlock_simple(): vs = create_autospec(VersionStore, _collection=Mock()) ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]}) vs.read.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=1, metadata=None, data=ts1) vs.write.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=2, metadata=None, data=None) vs.list_versions.return_value = [{'version': 2}, {'version': 1}] with ArcticTransaction(vs, sentinel.symbol, sentinel.user, sentinel.log) as cwb: cwb.write(sentinel.symbol, pd.DataFrame(index=[3, 4], data={'a': [1.0, 2.0]}), metadata=sentinel.meta) assert not vs._delete_version.called vs.write.assert_called_once_with(sentinel.symbol, ANY, prune_previous_version=True, metadata=sentinel.meta) vs.list_versions.assert_called_once_with(sentinel.symbol)
def losing_writer(): #will attempt to write version 2, should find that version 2 is there and it ends up writing version 3 with pytest.raises(ArcticTransaction): with ArcticTransaction(vs, sentinel.symbol, sentinel.user, sentinel.log) as cwb: cwb.write(sentinel.symbol, pd.DataFrame([1.0, 2.0], [3, 4])) e1.wait()
def test_ArticTransaction_no_audit(): vs = Mock(spec=VersionStore) ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]}) vs.read.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=1, metadata=None, data=ts1) vs.write.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=2, metadata=None, data=None) vs.list_versions.return_value = [{'version': 2}, {'version': 1}] with ArcticTransaction(vs, sentinel.symbol, sentinel.user, sentinel.log, audit=False) as cwb: cwb.write(sentinel.symbol, pd.DataFrame(index=[3, 4], data={'a': [1.0, 2.0]}), metadata=sentinel.meta) assert vs.write.call_count == 1 assert vs._write_audit.call_count == 0
def test_ArcticTransaction_guards_against_inconsistent_ts(): vs = Mock(spec=VersionStore) ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]}) vs.read.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=1, metadata=None, data=ts1, host=sentinel.host) vs.write.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=2, metadata=None, data=None, host=sentinel.host) vs.list_versions.side_effect = [{'version': 2}, {'version': 1}] ts1 = pd.DataFrame(index=[1, 2], data={'a': [2.0, 3.0]}) with pytest.raises(ConcurrentModificationException): with ArcticTransaction(vs, sentinel.symbol, sentinel.user, sentinel.log, modify_timeseries=ts1) as cwb: pass
def test_ArcticTransaction_write_skips_for_exact_match(library): ts = read_str_as_pandas("""times | PX_LAST 2014-10-31 21:30:00.000 | 204324.674 2014-11-13 21:30:00.000 | 193964.45 2014-11-14 21:30:00.000 | 193650.403""") with ArcticTransaction(library, symbol, 'u1', 'l1') as mt: mt.write(symbol, ts) version = library.read(symbol).version # try and store same TimeSeries again with ArcticTransaction(library, symbol, 'u1', 'l2') as mt: mt.write(symbol, ts) assert library.read(symbol).version == version
def test_cleanup_orphaned_versions_integration(library): _id = ObjectId.from_datetime(dt(2013, 1, 1)) with patch('bson.ObjectId', return_value=_id): with ArcticTransaction(library, symbol, 'u1', 'l1') as mt: mt.write(symbol, ts1) assert mongo_count(library._versions, filter={'parent': {'$size': 1}}) == 1 library._cleanup_orphaned_versions(False) assert mongo_count(library._versions, filter={'parent': {'$size': 1}}) == 1
def test_ArcticTransaction_writes_if_metadata_changed(): vs = Mock(spec=VersionStore) ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]}) vs.read.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=1, metadata=None, data=ts1, host=sentinel.host) vs.write.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=2, metadata=None, data=None, host=sentinel.host) vs.list_versions.return_value = [{'version': 2}, {'version': 1}] with ArcticTransaction(vs, sentinel.symbol, sentinel.user, sentinel.log) as cwb: assert cwb._do_write is False cwb.write(sentinel.symbol, ts1, metadata={1: 2}) assert cwb._do_write is True assert not vs._delete_version.called vs.write.assert_called_once_with(sentinel.symbol, ANY, prune_previous_version=True, metadata={1: 2}) vs.list_versions.assert_called_once_with(sentinel.symbol) # Won't write on exit with same data and metadata vs.read.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=2, metadata={1: 2}, data=ts1, host=sentinel.host) with ArcticTransaction(vs, sentinel.symbol, sentinel.user, sentinel.log) as cwb: assert cwb._do_write is False cwb.write(sentinel.symbol, ts1, metadata={1: 2}) assert cwb._do_write is False
def test_ArcticTransaction_write_doesnt_skip_for_close_ts(library): orig_ts = read_str_as_pandas("""times | PX_LAST 2014-10-31 21:30:00.000 | 204324.674 2014-11-13 21:30:00.000 | 193964.45 2014-11-14 21:30:00.000 | 193650.403""") with ArcticTransaction(library, symbol, 'u1', 'l1') as mt: mt.write(symbol, orig_ts) assert_frame_equal(library.read(symbol).data, orig_ts) # try and store slighty different TimeSeries new_ts = read_str_as_pandas("""times | PX_LAST 2014-10-31 21:30:00.000 | 204324.672 2014-11-13 21:30:00.000 | 193964.453 2014-11-14 21:30:00.000 | 193650.406""") with ArcticTransaction(library, symbol, 'u1', 'l2') as mt: mt.write(symbol, new_ts) assert_frame_equal(library.read(symbol).data, new_ts)
def test_audit_writes(library): with ArcticTransaction(library, symbol, 'u1', 'l1') as mt: mt.write(symbol, ts1) with ArcticTransaction(library, symbol, 'u2', 'l2') as mt: mt.write(symbol, ts2) audit_log = library.read_audit_log(symbol) assert audit_log == [{ u'new_v': 2, u'symbol': u'TS1', u'message': u'l2', u'user': u'u2', u'orig_v': 1 }, { u'new_v': 1, u'symbol': u'TS1', u'message': u'l1', u'user': u'u1', u'orig_v': 0 }] assert_frame_equal(ts1, library.read(symbol, audit_log[0]['orig_v']).data) assert_frame_equal(ts2, library.read(symbol, audit_log[0]['new_v']).data)
def test_ArcticTransaction_does_nothing_when_data_is_None(): vs = Mock(spec=VersionStore) ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]}) vs.read.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=1, metadata=None, data=ts1) vs.write.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=2, metadata=None, data=None) vs.list_versions.return_value = [{'version': 1}, {'version': 2}] with ArcticTransaction(vs, sentinel.symbol, sentinel.user, sentinel.log) as cwb: pass assert not vs._delete_version.called assert not vs.write.called
def test_ArcticTransaction_does_nothing_when_data_not_modified(): vs = create_autospec(VersionStore, _collection=Mock()) ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]}) vs.read.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=1, metadata=None, data=ts1) vs.write.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=2, metadata=None, data=None) vs.list_versions.side_effect = [{'version': 2}, {'version': 1}] with ArcticTransaction(vs, sentinel.symbol, sentinel.user, sentinel.log) as cwb: cwb.write(sentinel.symbol, pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]})) assert not vs._delete_version.called assert not vs.write.called
def _copy_symbol(symbols): for symbol in symbols: with ArcticTransaction(dest, symbol, USER, log) as mt: existing_data = dest.has_symbol(symbol) if existing_data: if force: logger.warn( "Symbol: %s already exists in destination, OVERWRITING" % symbol) elif splice: logger.warn( "Symbol: %s already exists in destination, splicing in new data" % symbol) else: logger.warn( "Symbol: {} already exists in {}@{}, use --force to overwrite or --splice to join with existing data" .format(symbol, _get_host(dest).get('l'), _get_host(dest).get('mhost'))) continue version = src.read(symbol) new_data = version.data if existing_data and splice: original_data = dest.read(symbol).data before = original_data.ix[:to_pandas_closed_closed( DateRange(None, new_data.index[0].to_pydatetime(), interval=CLOSED_OPEN)).end] after = original_data.ix[to_pandas_closed_closed( DateRange(new_data.index[-1].to_pydatetime(), None, interval=OPEN_CLOSED)).start:] new_data = before.append(new_data).append(after) mt.write(symbol, new_data, metadata=version.metadata)
def winning_writer(): #will attempt to write version 2 as well with ArcticTransaction(library, 'FOO', 'user', 'log') as cwb: cwb.write('FOO', ts2, metadata={'foo': 'bar'}) e2.wait()
def losing_writer(): #will attempt to write version 2, should find that version 2 is there and it ends up writing version 3 with pytest.raises(ConcurrentModificationException): with ArcticTransaction(library, 'FOO', 'user', 'log') as cwb: cwb.write('FOO', ts1_append, metadata={'foo': 'bar'}) e1.wait()
def test_ArcticTransaction_can_do_first_writes(library): with ArcticTransaction(library, 'SYMBOL_NOT_HERE', 'user', 'log') as cwb: cwb.write('SYMBOL_NOT_HERE', ts1) wrote_vi = library.read('SYMBOL_NOT_HERE') assert_frame_equal(wrote_vi.data, ts1)
def test_audit_read(library): with ArcticTransaction(library, symbol3, 'u3', 'foo') as mt: mt.write(symbol3, ts1) with ArcticTransaction(library, symbol, 'u1', 'l1') as mt: mt.write(symbol, ts1) with ArcticTransaction(library, symbol, 'u2', 'l2') as mt: mt.write(symbol, ts2) with ArcticTransaction(library, symbol2, 'u2', 'l2') as mt: mt.write(symbol2, ts2) audit_log = library.read_audit_log() assert audit_log == [ { u'new_v': 1, u'symbol': u'TS2', u'message': u'l2', u'user': u'u2', u'orig_v': 0 }, { u'new_v': 2, u'symbol': u'TS1', u'message': u'l2', u'user': u'u2', u'orig_v': 1 }, { u'new_v': 1, u'symbol': u'TS1', u'message': u'l1', u'user': u'u1', u'orig_v': 0 }, { u'new_v': 1, u'symbol': u'TS3', u'message': u'foo', u'user': u'u3', u'orig_v': 0 }, ] l2_audit_log = library.read_audit_log(message='l2') assert l2_audit_log == [ { u'new_v': 1, u'symbol': u'TS2', u'message': u'l2', u'user': u'u2', u'orig_v': 0 }, { u'new_v': 2, u'symbol': u'TS1', u'message': u'l2', u'user': u'u2', u'orig_v': 1 }, ] symbol_audit_log = library.read_audit_log(symbol=symbol) assert symbol_audit_log == [{ u'new_v': 2, u'symbol': u'TS1', u'message': u'l2', u'user': u'u2', u'orig_v': 1 }, { u'new_v': 1, u'symbol': u'TS1', u'message': u'l1', u'user': u'u1', u'orig_v': 0 }] symbols_audit_log = library.read_audit_log(symbol=[symbol, symbol2]) assert symbols_audit_log == [{ u'new_v': 1, u'symbol': u'TS2', u'message': u'l2', u'user': u'u2', u'orig_v': 0 }, { u'new_v': 2, u'symbol': u'TS1', u'message': u'l2', u'user': u'u2', u'orig_v': 1 }, { u'new_v': 1, u'symbol': u'TS1', u'message': u'l1', u'user': u'u1', u'orig_v': 0 }] symbol_message_audit_log = library.read_audit_log(symbol=symbol, message='l2') assert symbol_message_audit_log == [ { u'new_v': 2, u'symbol': u'TS1', u'message': u'l2', u'user': u'u2', u'orig_v': 1 }, ]
def winning_writer(): #will attempt to write version 2 as well with ArcticTransaction(vs, sentinel.symbol, sentinel.user, sentinel.log) as cwb: cwb.write(sentinel.symbol, pd.DataFrame([1.0, 2.0], [5, 6])) e2.wait()