def test_append_error_clean_retry():
read_handler = Mock(append=Mock(__name__=""))
read_handler.append.side_effect = [OperationFailure("mongo failure"), None]
previous_version = TPL_VERSION.copy()
previous_version['version'] = 1
vs = create_autospec(
VersionStore,
instance=True,
_collection=Mock(),
_version_nums=Mock(find_one_and_update=Mock(
return_value={'version': previous_version['version'] + 1})),
_versions=Mock(insert_one=Mock(__name__="insert_one"),
find_one=Mock(__name__="find_one",
return_value=previous_version)),
_arctic_lib=create_autospec(ArcticLibraryBinding,
arctic=create_autospec(
Arctic, mongo_host='some_host')))
vs._insert_version = lambda version: VersionStore._insert_version(
vs, version)
vs._collection.database.connection.nodes = []
vs._read_handler.return_value = read_handler
VersionStore.append(vs,
'sym', [1, 2, 3],
prune_previous_version=False,
upsert=False)
assert vs._version_nums.find_one_and_update.call_count == 2
assert vs._versions.find_one.call_count == 2
assert read_handler.append.call_count == 2
assert vs._versions.insert_one.call_count == 1
def test_write_metadata_insert_version_dupkeyerror():
vs = _create_mock_versionstore()
vs._versions.insert_one.__name__ = 'insert_one'
vs._versions.insert_one.side_effect = [DuplicateKeyError('dup key error'), None]
VersionStore.write_metadata(vs, symbol=TEST_SYMBOL, metadata=META_TO_WRITE)
assert vs._version_nums.find_one_and_update.call_count == 2
assert vs._versions.insert_one.call_count == 2
def _create_mock_versionstore():
vs = create_autospec(VersionStore,
_arctic_lib=Mock(),
_version_nums=Mock(),
_versions=Mock())
vs._insert_version = lambda version: VersionStore._insert_version(
vs, version)
vs._arctic_lib.get_name.return_value = TEST_LIB
vs._read_metadata.return_value = TPL_VERSION
vs._version_nums.find_one_and_update.return_value = {
'version': TPL_VERSION['version'] + 1
}
vs._version_nums.find_one.return_value = {
'version': TPL_VERSION['version'] + 1
}
vs._versions.find_one.return_value = TPL_VERSION
vs._add_new_version_using_reference.side_effect = lambda *args: VersionStore._add_new_version_using_reference(
vs, *args)
vs._last_version_seqnum = lambda version: VersionStore._last_version_seqnum(
vs, version)
vs.write.return_value = VersionedItem(
symbol=TEST_SYMBOL,
library=vs._arctic_lib.get_name(),
version=TPL_VERSION['version'] + 1,
metadata=META_TO_WRITE,
data=None,
host=vs._arctic_lib.arctic.mongo_host)
return vs
def test_write_insert_version_operror():
write_handler = Mock(write=Mock(__name__=""))
vs = create_autospec(VersionStore,
instance=True,
_collection=Mock(),
_version_nums=Mock(find_one_and_update=Mock(
return_value={'version': 1})),
_versions=Mock(insert_one=Mock(__name__="insert_one"),
find_one=Mock(__name__="find_one")),
_arctic_lib=create_autospec(
ArcticLibraryBinding,
arctic=create_autospec(Arctic,
mongo_host='some_host')))
vs._insert_version = lambda version: VersionStore._insert_version(
vs, version)
vs._versions.insert_one.side_effect = [
OperationFailure("mongo op error"), None
]
vs._collection.database.connection.nodes = []
vs._write_handler.return_value = write_handler
VersionStore.write(vs, 'sym', sentinel.data, prune_previous_version=False)
assert vs._version_nums.find_one_and_update.call_count == 1
assert vs._versions.find_one.call_count == 1
assert write_handler.write.call_count == 1
assert vs._versions.insert_one.call_count == 2
def test_read_handles_operation_failure():
self = Mock(spec=VersionStore)
self._read_preference.return_value = sentinel.read_preference
self._collection = create_autospec(Collection)
self._read_metadata.side_effect = [sentinel.meta1, sentinel.meta2]
self._read_metadata.__name__ = 'name'
self._do_read.__name__ = 'name' # feh: mongo_retry decorator cares about this
self._do_read.side_effect = [OperationFailure('error'), sentinel.read]
VersionStore.read(self,
sentinel.symbol,
sentinel.as_of,
from_version=sentinel.from_version,
date_range=sentinel.date_range,
other_kwarg=sentinel.other_kwarg)
# Assert that, for the two read calls, the second uses the new metadata
assert self._do_read.call_args_list == [
call(sentinel.symbol,
sentinel.meta1,
sentinel.from_version,
date_range=sentinel.date_range,
other_kwarg=sentinel.other_kwarg,
read_preference=sentinel.read_preference)
]
assert self._do_read_retry.call_args_list == [
call(sentinel.symbol,
sentinel.meta2,
sentinel.from_version,
date_range=sentinel.date_range,
other_kwarg=sentinel.other_kwarg,
read_preference=ReadPreference.PRIMARY)
]
def test_list_symbols_default_pipeline():
versions = Mock()
vs = create_autospec(VersionStore, _versions=versions)
versions.aggregate.return_value = []
VersionStore.list_symbols(vs)
pipeline = [{
'$group': {
'_id': '$symbol',
'version_custom': {
'$max': {
'$add': [{
'$multiply': ['$version', 2]
}, {
'$cond': [{
'$eq': ['$metadata.deleted', True]
}, 1, 0]
}]
}
},
}
}, {
'$match': {
'version_custom': {
'$mod': [2, 0]
}
}
}]
versions.aggregate.assert_called_once_with(pipeline, allowDiskUse=True)
def test_get_version_allow_secondary_user_override_False():
"""Ensure user can override read preference when calling read"""
vs = create_autospec(VersionStore, instance=True, _versions=Mock())
vs._read_preference.return_value = sentinel.read_preference
vs._find_snapshots.return_value = 'snap'
vs._versions.find.return_value = [{
'_id':
bson.ObjectId.from_datetime(dt(2013, 4, 1, 9, 0)),
'symbol':
's',
'version':
10
}]
VersionStore.read(vs, "symbol", allow_secondary=False)
assert vs._read_metadata.call_args_list == [
call('symbol', as_of=None, read_preference=sentinel.read_preference)
]
assert vs._do_read.call_args_list == [
call('symbol',
vs._read_metadata.return_value,
None,
date_range=None,
read_preference=sentinel.read_preference)
]
vs._read_preference.assert_called_once_with(False)
def test_append_insert_version_duplicatekey():
read_handler = Mock(append=Mock(__name__=""))
previous_version = TPL_VERSION.copy()
previous_version['version'] = 1
vs = create_autospec(
VersionStore,
instance=True,
_collection=Mock(),
_version_nums=Mock(find_one_and_update=Mock(
return_value={'version': previous_version['version'] + 1})),
_versions=Mock(insert_one=Mock(__name__="insert_one"),
find_one=Mock(__name__="find_one",
return_value=previous_version)),
_arctic_lib=create_autospec(ArcticLibraryBinding),
_publish_changes=False)
vs._insert_version = lambda version: VersionStore._insert_version(
vs, version)
vs._versions.insert_one.side_effect = [
DuplicateKeyError("dup key error"), None
]
vs._collection.database.connection.nodes = []
vs._read_handler.return_value = read_handler
VersionStore.append(vs,
'sym', [1, 2, 3],
prune_previous_version=False,
upsert=False)
assert vs._version_nums.find_one_and_update.call_count == 2
assert vs._versions.find_one.call_count == 2
assert read_handler.append.call_count == 2
assert vs._versions.insert_one.call_count == 2
assert vs._publish_change.call_count == 2
def test_write_metadata_insert_version_opfailure():
vs = _create_mock_versionstore()
vs._versions.insert_one.__name__ = 'insert_one'
vs._versions.insert_one.side_effect = [OperationFailure('op failure'), None]
VersionStore.write_metadata(vs, symbol=TEST_SYMBOL, metadata=META_TO_WRITE)
assert vs._version_nums.find_one_and_update.call_count == 1
assert vs._versions.insert_one.call_count == 2
def test_prune_previous_versions_0_timeout():
self = create_autospec(VersionStore, _versions=Mock())
self.name = sentinel.name
self._versions = create_autospec(Collection)
self._versions.with_options.return_value.find.__name__ = 'find'
self._versions.with_options.return_value.find.return_value = []
with patch('arctic.store.version_store.dt') as dt:
dt.utcnow.return_value = datetime.datetime(2013, 10, 1)
VersionStore._find_prunable_version_ids(self,
sentinel.symbol,
keep_mins=0)
assert self._versions.with_options.call_args_list == [
call(read_preference=ReadPreference.PRIMARY)
]
assert self._versions.with_options.return_value.find.call_args_list == [
call(
{
'$or': [{
'parent': {
'$exists': False
}
}, {
'parent': []
}],
'symbol': sentinel.symbol,
'_id': {
'$lt': bson.ObjectId('524a10810000000000000000')
}
},
sort=[('version', -1)],
skip=1,
projection=['_id'])
]
def test_read_handles_operation_failure():
self = create_autospec(VersionStore,
_versions=Mock(),
_arctic_lib=Mock(),
_allow_secondary=True)
self._collection = create_autospec(Collection)
self._read_metadata.side_effect = [sentinel.meta1, sentinel.meta2]
self._read_metadata.__name__ = 'name'
self._do_read.__name__ = 'name' # feh: mongo_retry decorator cares about this
self._do_read.side_effect = [OperationFailure('error'), sentinel.read]
VersionStore.read(self, sentinel.symbol, sentinel.as_of,
sentinel.from_version)
# Assert that, for the two read calls, the second uses the new metadata
assert self._do_read.call_args_list == [
call(sentinel.symbol,
sentinel.meta1,
sentinel.from_version,
read_preference=ReadPreference.NEAREST)
]
assert self._do_read_retry.call_args_list == [
call(sentinel.symbol,
sentinel.meta2,
sentinel.from_version,
read_preference=ReadPreference.PRIMARY)
]
def test_initialize_library():
arctic_lib = create_autospec(ArcticLibraryBinding)
arctic_lib.arctic = create_autospec(Arctic, _allow_secondary=False)
with patch('arctic.store.version_store.enable_sharding', autospec=True) as enable_sharding:
arctic_lib.get_top_level_collection.return_value.database.create_collection.__name__ = 'some_name'
arctic_lib.get_top_level_collection.return_value.database.collection_names.__name__ = 'some_name'
VersionStore.initialize_library(arctic_lib, hashed=sentinel.hashed)
assert enable_sharding.call_args_list == [call(arctic_lib.arctic, arctic_lib.get_name(), hashed=sentinel.hashed)]
def test_read_metadata_no_asof():
# When we do a read, with naive as_of, that as_of is treated in London Time.
vs = create_autospec(VersionStore, instance=True,
_versions=Mock(), _allow_secondary=False)
VersionStore._read_metadata(vs, sentinel.symbol)
versions = vs._versions.with_options.return_value
assert versions.find_one.call_args_list == [call({'symbol': sentinel.symbol},
sort=[('version', pymongo.DESCENDING)])]
def test_write_metadata_insert_version_opfailure():
vs = _create_mock_versionstore()
vs._versions.insert_one.__name__ = 'insert_one'
vs._versions.insert_one.side_effect = [OperationFailure('op failure'), None]
VersionStore.write_metadata(vs, symbol=TEST_SYMBOL, metadata=META_TO_WRITE)
assert vs._version_nums.find_one_and_update.call_count == 1
assert vs._versions.insert_one.call_count == 2
assert vs._publish_change.call_count == 1
def test_write_metadata_insert_version_dupkeyerror():
vs = _create_mock_versionstore()
vs._versions.insert_one.__name__ = 'insert_one'
vs._versions.insert_one.side_effect = [DuplicateKeyError('dup key error'), None]
VersionStore.write_metadata(vs, symbol=TEST_SYMBOL, metadata=META_TO_WRITE)
assert vs._version_nums.find_one_and_update.call_count == 2
assert vs._versions.insert_one.call_count == 2
assert vs._publish_change.call_count == 1
def test_restore_version_insert_version_opfailure():
vs = _create_mock_versionstore()
vs._versions.insert_one.__name__ = 'insert_one'
vs._versions.insert_one.side_effect = [OperationFailure('op failure'), None]
VersionStore.restore_version(vs, symbol=TEST_SYMBOL,
as_of=TPL_VERSION['version'], prune_previous_version=True)
assert vs._version_nums.find_one_and_update.call_count == 1
assert vs._versions.insert_one.call_count == 2
assert vs._publish_change.call_count == 1
def test_restore_version_insert_version_dupkeyerror():
vs = _create_mock_versionstore()
vs._versions.insert_one.__name__ = 'insert_one'
vs._versions.insert_one.side_effect = [DuplicateKeyError('dup key error'), None]
VersionStore.restore_version(vs, symbol=TEST_SYMBOL,
as_of=TPL_VERSION['version'], prune_previous_version=True)
assert vs._version_nums.find_one_and_update.call_count == 2
assert vs._versions.insert_one.call_count == 2
assert vs._publish_change.call_count == 1
def test_read_as_of_NotNaive():
# When we do a read, with naive as_of, that as_of is treated in London Time.
vs = create_autospec(VersionStore, instance=True,
_versions=Mock(), _allow_secondary=False)
VersionStore._read_metadata(vs, 'symbol', dt(2013, 4, 1, 9, 0, tzinfo=mktz('Europe/Paris')))
versions = vs._versions.with_options.return_value
versions.find_one.assert_called_once_with({'symbol':'symbol', '_id':
{'$lt': bson.ObjectId.from_datetime(dt(2013, 4, 1, 9, 0, tzinfo=mktz('Europe/Paris')) + dtd(seconds=1))}},
sort=[('_id', pymongo.DESCENDING)])
def test_restore_version_data_missing_symbol():
vs = _create_mock_versionstore()
vs._read_metadata.side_effect = NoDataFoundException("no data")
with patch('arctic.store.version_store.mongo_retry') as mock_retry:
mock_retry.side_effect = lambda f: f
with pytest.raises(NoDataFoundException):
VersionStore.restore_version(vs, symbol=TEST_SYMBOL,
as_of=TPL_VERSION['version'], prune_previous_version=True)
assert vs._read_metadata.call_args_list == [call(TEST_SYMBOL, as_of=TPL_VERSION['version'])]
assert vs._versions.insert_one.called is False
def test_ensure_index():
th = Mock()
vs = create_autospec(VersionStore, _collection=Mock())
with patch('arctic.store.version_store._TYPE_HANDLERS', [th]):
VersionStore._ensure_index(vs)
assert vs._collection.snapshots.create_index.call_args_list == [call([('name', 1)], unique=True, background=True)]
assert vs._collection.versions.create_index.call_args_list == [call([('symbol', 1), ('_id', -1)], background=True),
call([('symbol', 1), ('version', -1)], unique=True, background=True)]
assert vs._collection.version_nums.create_index.call_args_list == [call('symbol', unique=True, background=True)]
th._ensure_index.assert_called_once_with(vs._collection)
def test_read_reports_random_errors():
self = create_autospec(VersionStore, _versions=Mock(), _arctic_lib=Mock(),
_allow_secondary=True)
self._collection = create_autospec(Collection)
self._do_read.__name__ = 'name' # feh: mongo_retry decorator cares about this
self._do_read.side_effect = Exception('bad')
with pytest.raises(Exception) as e:
with patch('arctic.store.version_store.log_exception') as le:
VersionStore.read(self, sentinel.symbol, sentinel.as_of, sentinel.from_version)
assert 'bad' in str(e)
assert le.call_count == 1
def test_get_version_allow_secondary_True():
vs = create_autospec(VersionStore, instance=True,
_versions=Mock())
vs._allow_secondary = True
vs._find_snapshots.return_value = 'snap'
vs._versions.find.return_value = [{'_id': bson.ObjectId.from_datetime(dt(2013, 4, 1, 9, 0)),
'symbol': 's', 'version': 10}]
VersionStore.read(vs, "symbol")
assert vs._read_metadata.call_args_list == [call('symbol', as_of=None, read_preference=ReadPreference.NEAREST)]
assert vs._do_read.call_args_list == [call('symbol', vs._read_metadata.return_value, None, read_preference=ReadPreference.NEAREST)]
def test_restore_version_data_missing_symbol():
vs = _create_mock_versionstore()
vs._read_metadata.side_effect = NoDataFoundException("no data")
with patch('arctic.store.version_store.mongo_retry') as mock_retry:
mock_retry.side_effect = lambda f: f
with pytest.raises(NoDataFoundException):
VersionStore.restore_version(vs, symbol=TEST_SYMBOL,
as_of=TPL_VERSION['version'], prune_previous_version=True)
assert vs._read_metadata.call_args_list == [call(TEST_SYMBOL, as_of=TPL_VERSION['version'])]
assert vs._versions.insert_one.called is False
assert vs._publish_change.called is False
def _create_mock_versionstore():
vs = create_autospec(VersionStore, _arctic_lib=Mock(), _version_nums=Mock(), _versions=Mock())
vs._insert_version = lambda version: VersionStore._insert_version(vs, version)
vs._arctic_lib.get_name.return_value = TEST_LIB
vs._read_metadata.return_value = TPL_VERSION
vs._version_nums.find_one_and_update.return_value = {'version': TPL_VERSION['version'] + 1}
vs._versions.find_one.return_value = TPL_VERSION
vs._add_new_version_using_reference.side_effect = lambda *args: VersionStore._add_new_version_using_reference(vs, *args)
vs.write.return_value = VersionedItem(symbol=TEST_SYMBOL, library=vs._arctic_lib.get_name(),
version=TPL_VERSION['version'] + 1, metadata=META_TO_WRITE, data=None)
return vs
def test_snapshot():
vs = create_autospec(VersionStore, _snapshots=Mock(),
_collection=Mock(),
_versions=Mock())
vs._snapshots.find_one.return_value = False
vs._versions.update_one.__name__ = 'name'
vs._snapshots.insert_one.__name__ = 'name'
vs.list_symbols.return_value = ['foo', 'bar']
VersionStore.snapshot(vs, "symbol")
assert vs._read_metadata.call_args_list == [call('foo', as_of=None, read_preference=ReadPreference.PRIMARY),
call('bar', as_of=None, read_preference=ReadPreference.PRIMARY)]
def test_write_ensure_index():
write_handler = Mock(write=Mock(__name__=""))
vs = create_autospec(VersionStore, instance=True,
_collection=Mock(),
_version_nums=Mock(find_one_and_update=Mock(return_value={'version':1})),
_versions=Mock(insert_one=lambda x:None),
_arctic_lib=Mock(),
_publish_changes=False)
vs._collection.database.connection.nodes = []
vs._write_handler.return_value = write_handler
VersionStore.write(vs, 'sym', sentinel.data, prune_previous_version=False)
vs._ensure_index.assert_called_once_with()
def test_write_check_quota():
write_handler = Mock(write=Mock(__name__=""))
vs = create_autospec(VersionStore, instance=True,
_collection=Mock(),
_version_nums=Mock(find_one_and_update=Mock(return_value={'version':1})),
_versions=Mock(insert_one=lambda x:None),
_arctic_lib=create_autospec(ArcticLibraryBinding),
_publish_changes=False)
vs._collection.database.connection.nodes = []
vs._write_handler.return_value = write_handler
VersionStore.write(vs, 'sym', sentinel.data, prune_previous_version=False)
assert vs._arctic_lib.check_quota.call_count == 1
def test_get_version_allow_secondary_user_override_False():
"""Ensure user can override read preference when calling read"""
vs = create_autospec(VersionStore, instance=True,
_versions=Mock())
vs._allow_secondary = True
vs._find_snapshots.return_value = 'snap'
vs._versions.find.return_value = [{'_id': bson.ObjectId.from_datetime(dt(2013, 4, 1, 9, 0)),
'symbol': 's', 'version': 10}]
VersionStore.read(vs, "symbol", allow_secondary=False)
assert vs._read_metadata.call_args_list == [call('symbol', as_of=None, read_preference=ReadPreference.PRIMARY)]
assert vs._do_read.call_args_list == [call('symbol', vs._read_metadata.return_value, None, read_preference=ReadPreference.PRIMARY)]
def test_snapshot():
vs = create_autospec(VersionStore, _snapshots=Mock(),
_collection=Mock(),
_versions=Mock())
vs._snapshots.find_one.return_value = False
vs._versions.update_one.__name__ = 'name'
vs._snapshots.insert_one.__name__ = 'name'
vs.list_symbols.return_value = ['foo', 'bar']
VersionStore.snapshot(vs, "symbol")
assert (vs._read_metadata.call_args_list == [call('foo', as_of=None, read_preference=ReadPreference.PRIMARY),
call('bar', as_of=None, read_preference=ReadPreference.PRIMARY)] or
vs._read_metadata.call_args_list == [call('bar', as_of=None, read_preference=ReadPreference.PRIMARY),
call('foo', as_of=None, read_preference=ReadPreference.PRIMARY)])
def test_list_symbols_default_pipeline():
versions = Mock()
vs = create_autospec(VersionStore, _versions=versions)
versions.aggregate.return_value = []
VersionStore.list_symbols(vs)
pipeline = [
{'$sort': bson.SON([('symbol', pymongo.DESCENDING), ('version', pymongo.DESCENDING)])},
{'$group': {'_id': '$symbol', 'deleted': {'$first': '$metadata.deleted'}}},
{'$match': {'deleted': {'$ne': True}}},
{'$project': {'_id': 0, 'symbol': '$_id'}}
]
versions.aggregate.assert_called_once_with(pipeline)
def test_read_handles_operation_failure():
self = create_autospec(VersionStore, _versions=Mock(), _arctic_lib=Mock(),
_allow_secondary=True)
self._collection = create_autospec(Collection)
self._read_metadata.side_effect = [sentinel.meta1, sentinel.meta2]
self._read_metadata.__name__ = 'name'
self._do_read.__name__ = 'name' # feh: mongo_retry decorator cares about this
self._do_read.side_effect = [OperationFailure('error'), sentinel.read]
VersionStore.read(self, sentinel.symbol, sentinel.as_of, sentinel.from_version)
# Assert that, for the two read calls, the second uses the new metadata
assert self._do_read.call_args_list == [call(sentinel.symbol, sentinel.meta1, sentinel.from_version,
read_preference=ReadPreference.NEAREST)]
assert self._do_read_retry.call_args_list == [call(sentinel.symbol, sentinel.meta2, sentinel.from_version,
read_preference=ReadPreference.PRIMARY)]
def test_restore_version():
vs = _create_mock_versionstore()
LASTEST_VERSION = dict(TPL_VERSION, version=TPL_VERSION['version']+1, metadata={'something': 'different'})
last_item = VersionedItem(symbol=TEST_SYMBOL, library=vs._arctic_lib.get_name(),
host=vs._arctic_lib.arctic.mongo_host,
version=LASTEST_VERSION, metadata=LASTEST_VERSION['metadata'], data="hello world")
new_version = dict(LASTEST_VERSION, version=LASTEST_VERSION['version'] + 1)
new_item = VersionedItem(symbol=TEST_SYMBOL, library=vs._arctic_lib.get_name(),
host=vs._arctic_lib.arctic.mongo_host,
version=new_version, metadata=new_version['metadata'], data=last_item.data)
vs.write.return_value = new_item
vs.read.return_value = last_item
vs._read_metadata.side_effect = [TPL_VERSION, LASTEST_VERSION]
with patch('arctic.store.version_store.bson.ObjectId') as mock_objId, \
patch('arctic.store.version_store.mongo_retry') as mock_retry:
mock_objId.return_value = MOCK_OBJID
mock_retry.side_effect = lambda f: f
ret_item = VersionStore.restore_version(vs, symbol=TEST_SYMBOL, as_of=LASTEST_VERSION['version'], prune_previous_version=True)
assert ret_item == new_item
assert vs._read_metadata.call_args_list == [call(TEST_SYMBOL, as_of=LASTEST_VERSION['version'])]
assert vs._version_nums.find_one.call_args_list == [call({'symbol': TEST_SYMBOL})]
assert vs.read.call_args_list == [call(TEST_SYMBOL, as_of=LASTEST_VERSION['version'])]
assert vs.write.call_args_list == [call(TEST_SYMBOL, data=last_item.data, metadata=last_item.metadata, prune_previous_version=True)]
def test_restore_version():
vs = _create_mock_versionstore()
expected_new_version = TPL_VERSION.copy()
expected_new_version.update({
'_id': MOCK_OBJID,
'version': TPL_VERSION['version'] + 1,
'metadata': None
})
with patch('arctic.store.version_store.bson.ObjectId') as mock_objId, \
patch('arctic.store.version_store.mongo_retry') as mock_retry:
mock_objId.return_value = MOCK_OBJID
mock_retry.side_effect = lambda f: f
ret_val = VersionStore.restore_version(vs,
symbol=TEST_SYMBOL,
as_of=TPL_VERSION['version'],
prune_previous_version=True)
assert ret_val == VersionedItem(symbol=TEST_SYMBOL,
library=vs._arctic_lib.get_name(),
version=TPL_VERSION['version'] + 1,
metadata=None,
data=None)
assert vs._versions.insert_one.call_args_list == [
call(expected_new_version)
]
assert vs._publish_change.call_args_list == [
call(TEST_SYMBOL, expected_new_version)
]
def test_write_empty_metadata():
vs = _create_mock_versionstore()
expected_new_version = TPL_VERSION.copy()
expected_new_version.update({
'_id': MOCK_OBJID,
'version': TPL_VERSION['version'] + 1,
'metadata': None
})
expected_ret_val = VersionedItem(symbol=TEST_SYMBOL,
library=vs._arctic_lib.get_name(),
host=vs._arctic_lib.arctic.mongo_host,
version=TPL_VERSION['version'] + 1,
metadata=None,
data=None)
with patch('arctic.store.version_store.bson.ObjectId') as mock_objId, \
patch('arctic.store.version_store.mongo_retry') as mock_retry:
mock_objId.return_value = MOCK_OBJID
mock_retry.side_effect = lambda f: f
assert expected_ret_val == VersionStore.write_metadata(
vs, symbol=TEST_SYMBOL, metadata=None)
assert vs._versions.insert_one.call_args_list == [
call(expected_new_version)
]
assert vs._versions.delete_one.called is False
assert vs._publish_change.call_args_list == [
call(TEST_SYMBOL, expected_new_version)
]
assert vs.write.called is False
def test_restore_version():
vs = _create_mock_versionstore()
LASTEST_VERSION = dict(TPL_VERSION, version=TPL_VERSION['version']+1, metadata={'something': 'different'})
last_item = VersionedItem(symbol=TEST_SYMBOL, library=vs._arctic_lib.get_name(),
host=vs._arctic_lib.arctic.mongo_host,
version=LASTEST_VERSION, metadata=LASTEST_VERSION['metadata'], data="hello world")
new_version = dict(LASTEST_VERSION, version=LASTEST_VERSION['version'] + 1)
new_item = VersionedItem(symbol=TEST_SYMBOL, library=vs._arctic_lib.get_name(),
host=vs._arctic_lib.arctic.mongo_host,
version=new_version, metadata=new_version['metadata'], data=last_item.data)
vs.write.return_value = new_item
vs.read.return_value = last_item
vs._read_metadata.side_effect = [TPL_VERSION, LASTEST_VERSION]
with patch('arctic.store.version_store.bson.ObjectId') as mock_objId, \
patch('arctic.store.version_store.mongo_retry') as mock_retry:
mock_objId.return_value = MOCK_OBJID
mock_retry.side_effect = lambda f: f
ret_item = VersionStore.restore_version(vs, symbol=TEST_SYMBOL, as_of=LASTEST_VERSION['version'], prune_previous_version=True)
assert ret_item == new_item
assert vs._read_metadata.call_args_list == [call(TEST_SYMBOL, as_of=LASTEST_VERSION['version'])]
assert vs._version_nums.find_one.call_args_list == [call({'symbol': TEST_SYMBOL})]
assert vs.read.call_args_list == [call(TEST_SYMBOL, as_of=LASTEST_VERSION['version'])]
assert vs.write.call_args_list == [call(TEST_SYMBOL, data=last_item.data, metadata=last_item.metadata, prune_previous_version=True)]
def test_list_versions_localTime():
# Object ID's are stored in UTC. We need to ensure that the returned times
# for versions are in the local TimeZone
vs = create_autospec(VersionStore,
instance=True,
_versions=Mock(),
_snapshots=Mock())
mocked_snap_resp = [{'_id': 'abcde', 'name': 'snap'}]
vs._snapshots.find.return_value = mocked_snap_resp
vs._snapshots.find_one.return_value = mocked_snap_resp
date = dt(2013, 4, 1, 9, 0)
vs._versions.find.return_value = [{
'_id': bson.ObjectId.from_datetime(date),
'symbol': 's',
'version': 10,
'metadata': None,
'parent': [mocked_snap_resp[0]['_id']]
}]
version = list(VersionStore.list_versions(vs, "symbol"))[0]
local_date = date.replace(tzinfo=mktz("UTC"))
assert version == {
'symbol': version['symbol'],
'version': version['version'],
# We return naive datetimes in 'default' time, which is London for us
'date': local_date,
'snapshots': ['snap'],
'deleted': False
}
def test_restore_last_version():
vs = _create_mock_versionstore()
vs._version_nums.find_one.return_value = {
'version': TPL_VERSION['version']
}
vs._read_metadata.side_effect = [TPL_VERSION, TPL_VERSION]
with patch('arctic.store.version_store.bson.ObjectId') as mock_objId, \
patch('arctic.store.version_store.mongo_retry') as mock_retry:
mock_objId.return_value = MOCK_OBJID
mock_retry.side_effect = lambda f: f
ret_item = VersionStore.restore_version(vs,
symbol=TEST_SYMBOL,
as_of=TPL_VERSION['version'],
prune_previous_version=True)
assert ret_item.version == TPL_VERSION['version']
assert ret_item.metadata == TPL_VERSION.get('metadata')
assert vs._read_metadata.call_args_list == [
call(TEST_SYMBOL, as_of=TPL_VERSION['version'])
]
assert vs._version_nums.find_one.call_args_list == [
call({'symbol': TEST_SYMBOL})
]
assert not vs.read.called
assert not vs.write.called
def test_write_metadata_no_previous_data():
vs = _create_mock_versionstore()
vs._read_metadata.side_effect = NoDataFoundException("no data found")
assert VersionStore.write_metadata(vs, symbol=TEST_SYMBOL, metadata=META_TO_WRITE, my_custom_arg='hello') == vs.write.return_value
assert vs._read_metadata.call_args_list == [call(TEST_SYMBOL)]
assert vs.write.call_args_list == [call(TEST_SYMBOL, data=None, metadata=META_TO_WRITE,
prune_previous_version=True, my_custom_arg='hello')]
def test_write_metadata_no_previous_data():
vs = _create_mock_versionstore()
vs._read_metadata.side_effect = NoDataFoundException("no data found")
assert VersionStore.write_metadata(vs, symbol=TEST_SYMBOL, metadata=META_TO_WRITE, my_custom_arg='hello') == vs.write.return_value
assert vs._read_metadata.call_args_list == [call(TEST_SYMBOL)]
assert vs.write.call_args_list == [call(TEST_SYMBOL, data=None, metadata=META_TO_WRITE,
prune_previous_version=True, my_custom_arg='hello')]
def test_write_insert_version_operror():
write_handler = Mock(write=Mock(__name__=""))
vs = create_autospec(VersionStore, instance=True,
_collection=Mock(),
_version_nums=Mock(find_one_and_update=Mock(return_value={'version': 1})),
_versions=Mock(insert_one=Mock(__name__="insert_one"), find_one=Mock(__name__="find_one")),
_arctic_lib=create_autospec(ArcticLibraryBinding,
arctic=create_autospec(Arctic, mongo_host='some_host')))
vs._insert_version = lambda version: VersionStore._insert_version(vs, version)
vs._versions.insert_one.side_effect = [OperationFailure("mongo op error"), None]
vs._collection.database.connection.nodes = []
vs._write_handler.return_value = write_handler
VersionStore.write(vs, 'sym', sentinel.data, prune_previous_version=False)
assert vs._version_nums.find_one_and_update.call_count == 1
assert vs._versions.find_one.call_count == 1
assert write_handler.write.call_count == 1
assert vs._versions.insert_one.call_count == 2
def test_write_insert_version_duplicatekey():
write_handler = Mock(write=Mock(__name__=""))
vs = create_autospec(VersionStore, instance=True,
_collection=Mock(),
_version_nums=Mock(find_one_and_update=Mock(return_value={'version': 1})),
_versions=Mock(insert_one=Mock(__name__="insert_one"), find_one=Mock(__name__="find_one")),
_arctic_lib=create_autospec(ArcticLibraryBinding),
_publish_changes=False)
vs._insert_version = lambda version: VersionStore._insert_version(vs, version)
vs._versions.insert_one.side_effect = [DuplicateKeyError("dup key error"), None]
vs._collection.database.connection.nodes = []
vs._write_handler.return_value = write_handler
VersionStore.write(vs, 'sym', sentinel.data, prune_previous_version=False)
assert vs._version_nums.find_one_and_update.call_count == 2
assert vs._versions.find_one.call_count == 2
assert write_handler.write.call_count == 2
assert vs._versions.insert_one.call_count == 2
assert vs._publish_change.call_count == 1
def test_prune_previous_versions_0_timeout():
self = create_autospec(VersionStore, _versions=Mock())
self.name = sentinel.name
self._versions = create_autospec(Collection)
self._versions.with_options.return_value.find.__name__ = 'find'
self._versions.with_options.return_value.find.return_value = []
with patch('arctic.store.version_store.dt') as dt:
dt.utcnow.return_value = datetime.datetime(2013, 10, 1)
VersionStore._prune_previous_versions(self, sentinel.symbol, keep_mins=0)
assert self._versions.with_options.call_args_list == [call(read_preference=ReadPreference.PRIMARY)]
assert self._versions.with_options.return_value.find.call_args_list == [
call({'$or': [{'parent': {'$exists': False}},
{'parent': {'$size': 0}}],
'symbol': sentinel.symbol,
'_id': {'$lt': bson.ObjectId('524a10810000000000000000')}},
sort=[('version', -1)],
skip=1,
projection=['_id', 'type'])]
def test_append_insert_version_operror():
read_handler = Mock(append=Mock(__name__=""))
previous_version = TPL_VERSION.copy()
previous_version['version'] = 1
vs = create_autospec(VersionStore, instance=True,
_collection=Mock(),
_version_nums=Mock(find_one_and_update=Mock(return_value={'version': previous_version['version']+1})),
_versions=Mock(insert_one=Mock(__name__="insert_one"), find_one=Mock(__name__="find_one", return_value=previous_version)),
_arctic_lib=create_autospec(ArcticLibraryBinding,
arctic=create_autospec(Arctic, mongo_host='some_host')))
vs._insert_version = lambda version: VersionStore._insert_version(vs, version)
vs._versions.insert_one.side_effect = [OperationFailure("mongo op error"), None]
vs._collection.database.connection.nodes = []
vs._read_handler.return_value = read_handler
VersionStore.append(vs, 'sym', [1, 2, 3], prune_previous_version=False, upsert=False)
assert vs._version_nums.find_one_and_update.call_count == 1
assert vs._versions.find_one.call_count == 1
assert read_handler.append.call_count == 1
assert vs._versions.insert_one.call_count == 2
def test_append_insert_version_duplicatekey():
read_handler = Mock(append=Mock(__name__=""))
previous_version = TPL_VERSION.copy()
previous_version['version'] = 1
vs = create_autospec(VersionStore, instance=True,
_collection=Mock(),
_version_nums=Mock(find_one_and_update=Mock(return_value={'version': previous_version['version']+1})),
_versions=Mock(insert_one=Mock(__name__="insert_one"), find_one=Mock(__name__="find_one", return_value=previous_version)),
_arctic_lib=create_autospec(ArcticLibraryBinding),
_publish_changes=False)
vs._insert_version = lambda version: VersionStore._insert_version(vs, version)
vs._versions.insert_one.side_effect = [DuplicateKeyError("dup key error"), None]
vs._collection.database.connection.nodes = []
vs._read_handler.return_value = read_handler
VersionStore.append(vs, 'sym', [1, 2, 3], prune_previous_version=False, upsert=False)
assert vs._version_nums.find_one_and_update.call_count == 2
assert vs._versions.find_one.call_count == 2
assert read_handler.append.call_count == 2
assert vs._versions.insert_one.call_count == 2
assert vs._publish_change.call_count == 1
def test_list_symbols_default_pipeline():
versions = Mock()
vs = create_autospec(VersionStore, _versions=versions)
versions.aggregate.return_value = []
VersionStore.list_symbols(vs)
pipeline = [
{'$group': {
'_id': '$symbol',
'version_custom': {
'$max': {
'$add': [
{'$multiply': ['$version', 2]},
{'$cond': [{'$eq': ['$metadata.deleted', True]}, 1, 0]}
]
}
},
}},
{'$match': {'version_custom': {'$mod': [2, 0]}}}
]
versions.aggregate.assert_called_once_with(pipeline, allowDiskUse=True)
def test_read_handles_operation_failure():
self = Mock(spec=VersionStore)
self._read_preference.return_value = sentinel.read_preference
self._collection = create_autospec(Collection)
self._read_metadata.side_effect = [sentinel.meta1, sentinel.meta2]
self._read_metadata.__name__ = 'name'
self._do_read.__name__ = 'name' # feh: mongo_retry decorator cares about this
self._do_read.side_effect = [OperationFailure('error'), sentinel.read]
VersionStore.read(self, sentinel.symbol, sentinel.as_of,
from_version=sentinel.from_version,
date_range=sentinel.date_range,
other_kwarg=sentinel.other_kwarg)
# Assert that, for the two read calls, the second uses the new metadata
assert self._do_read.call_args_list == [call(sentinel.symbol, sentinel.meta1,
sentinel.from_version,
date_range=sentinel.date_range,
other_kwarg=sentinel.other_kwarg,
read_preference=sentinel.read_preference)]
assert self._do_read_retry.call_args_list == [call(sentinel.symbol, sentinel.meta2,
sentinel.from_version,
date_range=sentinel.date_range,
other_kwarg=sentinel.other_kwarg,
read_preference=ReadPreference.PRIMARY)]
def test_list_versions_localTime():
# Object ID's are stored in UTC. We need to ensure that the returned times
# for versions are in the local TimeZone
vs = create_autospec(VersionStore, instance=True,
_versions=Mock())
vs._find_snapshots.return_value = 'snap'
date = dt(2013, 4, 1, 9, 0)
vs._versions.find.return_value = [{'_id': bson.ObjectId.from_datetime(date),
'symbol': 's', 'version': 10}]
version = list(VersionStore.list_versions(vs, "symbol"))[0]
local_date = date.replace(tzinfo=mktz("UTC")).astimezone(mktz()).replace(tzinfo=None)
assert version == {'symbol': version['symbol'], 'version': version['version'],
# We return naive datetimes in 'default' time, which is London for us
'date': local_date,
'snapshots': 'snap'}
def test_list_versions_no_snapshot():
vs = create_autospec(VersionStore, instance=True, _versions=Mock(), _snapshots=Mock())
vs._snapshots.find.return_value = []
vs._snapshots.find_one.return_value = []
date = dt(2013, 4, 1, 9, 0)
vs._versions.find.return_value = [{'_id': bson.ObjectId.from_datetime(date),
'symbol': 's',
'version': 10,
'metadata': None,
'parent': []}]
version = list(VersionStore.list_versions(vs, "symbol"))[0]
local_date = date.replace(tzinfo=mktz("UTC"))
assert version == {'symbol': version['symbol'],
'version': version['version'],
'date': local_date,
'snapshots': [],
'deleted': False}
def test_restore_version():
vs = _create_mock_versionstore()
expected_new_version = TPL_VERSION.copy()
expected_new_version.update({'_id': MOCK_OBJID,
'version': TPL_VERSION['version'] + 1,
'metadata': None})
with patch('arctic.store.version_store.bson.ObjectId') as mock_objId, \
patch('arctic.store.version_store.mongo_retry') as mock_retry:
mock_objId.return_value = MOCK_OBJID
mock_retry.side_effect = lambda f: f
ret_val = VersionStore.restore_version(vs, symbol=TEST_SYMBOL, as_of=TPL_VERSION['version'], prune_previous_version=True)
assert ret_val == VersionedItem(symbol=TEST_SYMBOL,
library=vs._arctic_lib.get_name(),
version=TPL_VERSION['version'] + 1,
metadata=None,
data=None)
assert vs._versions.insert_one.call_args_list == [call(expected_new_version)]
assert vs._publish_change.call_args_list == [call(TEST_SYMBOL, expected_new_version)]
