def test_checksum_handles_p3strs_and_binary():
digest = checksum('test_my_market_data_$ymB0l', {
'key1': u'unicode',
'key2': b'binary_data'
})
expected = b'4O11 ;<[email protected](JRB1.?D[ZEN!8'
assert binascii.b2a_uu(digest).strip() == expected
def test_write_object():
arctic_lib = Mock()
self = create_autospec(PickleStore)
version = {"_id": ObjectId()}
PickleStore.write(self, arctic_lib, version, "sentinel.symbol", sentinel.item, sentinel.previous_version)
assert "data" not in version
assert version["blob"] == "__chunked__"
coll = arctic_lib.get_top_level_collection.return_value
assert coll.update_one.call_args_list == [
call(
{
"sha": checksum(
"sentinel.symbol",
{"data": Binary(lz4.compressHC(cPickle.dumps(sentinel.item, cPickle.HIGHEST_PROTOCOL)))},
),
"symbol": "sentinel.symbol",
},
{
"$set": {
"segment": 0,
"data": Binary(lz4.compressHC(cPickle.dumps(sentinel.item, cPickle.HIGHEST_PROTOCOL)), 0),
},
"$addToSet": {"parent": version["_id"]},
},
upsert=True,
)
]
def write(self, arctic_lib, version, symbol, item, previous_version):
try:
# If it's encodeable, then ship it
b = bson.BSON.encode({'data': item})
if len(b) < _MAX_BSON_ENCODE:
version['data'] = item
return
except InvalidDocument:
pass
# Pickle, chunk and store the data
collection = arctic_lib.get_top_level_collection()
# Try to pickle it. This is best effort
version['blob'] = _MAGIC_CHUNKED
pickled = lz4.compressHC(
cPickle.dumps(item, protocol=cPickle.HIGHEST_PROTOCOL))
for i in xrange(int(len(pickled) / _CHUNK_SIZE + 1)):
segment = {
'data': Binary(pickled[i * _CHUNK_SIZE:(i + 1) * _CHUNK_SIZE])
}
sha = checksum(symbol, segment)
segment['segment'] = i
collection.update_one({
'symbol': symbol,
'sha': sha
}, {
'$set': segment,
'$addToSet': {
'parent': version['_id']
}
},
upsert=True)
def test_write_object():
arctic_lib = Mock()
self = create_autospec(PickleStore)
version = {'_id': ObjectId()}
PickleStore.write(self, arctic_lib, version, 'sentinel.symbol', sentinel.item, sentinel.previous_version)
assert 'data' not in version
assert version['blob'] == '__chunked__V2'
coll = arctic_lib.get_top_level_collection.return_value
assert coll.update_one.call_args_list == [call({'sha': checksum('sentinel.symbol', {'segment':0, 'data': Binary(compress(cPickle.dumps(sentinel.item, cPickle.HIGHEST_PROTOCOL)))}),
'symbol': 'sentinel.symbol'},
{'$set': {'segment': 0, 'data': Binary(compress(cPickle.dumps(sentinel.item, cPickle.HIGHEST_PROTOCOL)), 0)},
'$addToSet': {'parent': version['_id']}}, upsert=True)]
def test_write_object():
arctic_lib = Mock()
self = create_autospec(PickleStore)
version = {'_id': ObjectId()}
PickleStore.write(self, arctic_lib, version, 'sentinel.symbol',
sentinel.item, sentinel.previous_version)
assert 'data' not in version
assert version['blob'] == '__chunked__V2'
coll = arctic_lib.get_top_level_collection.return_value
assert coll.update_one.call_args_list == [
call(
{
'sha':
checksum(
'sentinel.symbol', {
'segment':
0,
'data':
Binary(
compress(
cPickle.dumps(sentinel.item,
cPickle.HIGHEST_PROTOCOL)))
}),
'symbol':
'sentinel.symbol'
}, {
'$set': {
'segment':
0,
'data':
Binary(
compress(
cPickle.dumps(sentinel.item,
cPickle.HIGHEST_PROTOCOL)), 0)
},
'$addToSet': {
'parent': version['_id']
}
},
upsert=True)
]
def write(self, arctic_lib, version, symbol, item, previous_version):
try:
# If it's encodeable, then ship it
b = bson.BSON.encode({'data': item})
if len(b) < _MAX_BSON_ENCODE:
version['data'] = item
return
except InvalidDocument:
pass
# Pickle, chunk and store the data
collection = arctic_lib.get_top_level_collection()
# Try to pickle it. This is best effort
version['blob'] = _MAGIC_CHUNKED
pickled = lz4.compressHC(cPickle.dumps(item, protocol=cPickle.HIGHEST_PROTOCOL))
for i in xrange(int(len(pickled) / _CHUNK_SIZE + 1)):
segment = {'data': Binary(pickled[i * _CHUNK_SIZE : (i + 1) * _CHUNK_SIZE])}
sha = checksum(symbol, segment)
segment['segment'] = i
collection.update_one({'symbol': symbol, 'sha': sha}, {'$set': segment,
'$addToSet': {'parent': version['_id']}},
upsert=True)
def test_checksum_handles_p3strs_and_binary():
digest = checksum('test_my_market_data_$ymB0l', {'key1': u'unicode',
'key2': b'binary_data'})
expected = b'4O11 ;<[email protected](JRB1.?D[ZEN!8'
assert binascii.b2a_uu(digest).strip() == expected
def test_checksum():
digest = checksum('test_my_market_data_$ymB0l', {})
expected = b"""4OZ*3DO'$>XV['VW1MT4I^+7-3H,"""
assert binascii.b2a_uu(digest).strip() == expected