def test_recarrays():
x = np.array([(1.0, 2), (3.0, 4)], dtype=[("x", float), ("y", int)])
assert_array_equal(x, unpack(pack(x)))
x = x.view(np.recarray)
assert_array_equal(x, unpack(pack(x)))
x = np.array([(3, 4)], dtype=[("tmp0", float),
("tmp1", "O")]).view(np.recarray)
assert_array_equal(x, unpack(pack(x)))
def test_recarrays():
x = np.array([(1.0, 2), (3.0, 4)], dtype=[('x', float), ('y', int)])
assert_array_equal(x, unpack(pack(x)))
x = x.view(np.recarray)
assert_array_equal(x, unpack(pack(x)))
x = np.array([(3, 4)], dtype=[('tmp0', float),
('tmp1', 'O')]).view(np.recarray)
assert_array_equal(x, unpack(pack(x)))
def test_pack():
x = np.random.randn(8, 10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.random.randn(10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.float32(np.random.randn(3, 4, 5))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.int16(np.random.randn(1, 2, 3))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = {'name': 'Anonymous', 'age': 15}
assert_true(x == unpack(pack(x), as_dict=True), "Dict do not match!")
x = [1, 2, 3, 4]
assert_array_equal(x, unpack(pack(x)), "List did not pack/unpack correctly")
x = [1, 2, 3, 4]
assert_array_equal(x, unpack(pack(x.__iter__())), "Iterator did not pack/unpack correctly")
x = Decimal('1.24')
assert_true(float(x) == unpack(pack(x)), "Decimal object did not pack/unpack correctly")
x = datetime.now()
assert_true(str(x) == unpack(pack(x)), "Datetime object did not pack/unpack correctly")
def test_pack():
x = np.random.randn(8, 10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.random.randn(10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.float32(np.random.randn(3, 4, 5))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.int16(np.random.randn(1, 2, 3))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = {'name': 'Anonymous', 'age': 15}
assert_true(x == unpack(pack(x), as_dict=True), "Dict do not match!")
x = [1, 2, 3, 4]
assert_array_equal(x, unpack(pack(x)),
"List did not pack/unpack correctly")
x = [1, 2, 3, 4]
assert_array_equal(x, unpack(pack(x.__iter__())),
"Iterator did not pack/unpack correctly")
x = Decimal('1.24')
assert_true(
float(x) == unpack(pack(x)),
"Decimal object did not pack/unpack correctly")
x = datetime.now()
assert_true(
str(x) == unpack(pack(x)),
"Datetime object did not pack/unpack correctly")
def test_complex():
z = np.random.randn(8, 10) + 1j*np.random.randn(8,10)
assert_array_equal(z, unpack(pack(z)), "Arrays do not match!")
z = np.random.randn(10) + 1j*np.random.randn(10)
assert_array_equal(z, unpack(pack(z)), "Arrays do not match!")
x = np.float32(np.random.randn(3, 4, 5)) + 1j*np.float32(np.random.randn(3, 4, 5))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.int16(np.random.randn(1, 2, 3)) + 1j*np.int16(np.random.randn(1, 2, 3))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
def test_pack():
x = np.random.randn(8, 10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.random.randn(10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.float32(np.random.randn(3, 4, 5))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.int16(np.random.randn(1, 2, 3))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
def test_complex():
z = np.random.randn(8, 10) + 1j*np.random.randn(8,10)
assert_array_equal(z, unpack(pack(z)), "Arrays do not match!")
z = np.random.randn(10) + 1j*np.random.randn(10)
assert_array_equal(z, unpack(pack(z)), "Arrays do not match!")
x = np.float32(np.random.randn(3, 4, 5)) + 1j*np.float32(np.random.randn(3, 4, 5))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.int16(np.random.randn(1, 2, 3)) + 1j*np.int16(np.random.randn(1, 2, 3))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
def test_pack():
x = np.random.randn(8, 10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.random.randn(10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.float32(np.random.randn(3, 4, 5))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.int16(np.random.randn(1, 2, 3))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
def request(path):
req = Request("https://nda.seunglab.org/" + path)
req.add_header('Authorization', api_key)
res = urlopen(req)
if (res.info().get_content_type() == 'application/json'):
res2 = res.read()
return json.loads(res2.decode(res.info().get_param('charset') or 'utf-8'), object_pairs_hook=OrderedDict)
else:
return blob.unpack(res.read())
def test_external_put():
"""
external storage put and get and remove
"""
ext = ExternalTable(schema.connection, store='raw', database=schema.database)
initial_length = len(ext)
input_ = np.random.randn(3, 7, 8)
count = 7
extra = 3
for i in range(count):
hash1 = ext.put(pack(input_))
for i in range(extra):
hash2 = ext.put(pack(np.random.randn(4, 3, 2)))
fetched_hashes = ext.fetch('hash')
assert_true(all(hash in fetched_hashes for hash in (hash1, hash2)))
assert_equal(len(ext), initial_length + 1 + extra)
output_ = unpack(ext.get(hash1))
assert_array_equal(input_, output_)
def test_external_put():
"""
external storage put and get and remove
"""
ext = ExternalTable(schema.connection,
store='raw',
database=schema.database)
initial_length = len(ext)
input_ = np.random.randn(3, 7, 8)
count = 7
extra = 3
for i in range(count):
hash1 = ext.put(pack(input_))
for i in range(extra):
hash2 = ext.put(pack(np.random.randn(4, 3, 2)))
fetched_hashes = ext.fetch('hash')
assert_true(all(hash in fetched_hashes for hash in (hash1, hash2)))
assert_equal(len(ext), initial_length + 1 + extra)
output_ = unpack(ext.get(hash1))
assert_array_equal(input_, output_)
def test_complex_matlab_blobs():
"""
test correct de-serialization of various blob types
"""
blobs = Blob().fetch('blob', order_by='KEY')
blob = blobs[0] # 'simple string' 'character string'
assert_equal(blob[0], 'character string')
blob = blobs[1] # '1D vector' 1:15:180
assert_array_equal(blob, np.r_[1:180:15][None, :])
assert_array_equal(blob, unpack(pack(blob)))
blob = blobs[2] # 'string array' {'string1' 'string2'}
assert_true(isinstance(blob, dj.MatCell))
assert_array_equal(blob, np.array([['string1', 'string2']]))
assert_array_equal(blob, unpack(pack(blob)))
blob = blobs[
3] # 'struct array' struct('a', {1,2}, 'b', {struct('c', magic(3)), struct('C', magic(5))})
assert_true(isinstance(blob, dj.MatStruct))
assert_tuple_equal(blob.dtype.names, ('a', 'b'))
assert_array_equal(blob.a[0, 0], np.array([[1.]]))
assert_array_equal(blob.a[0, 1], np.array([[2.]]))
assert_true(isinstance(blob.b[0, 1], dj.MatStruct))
assert_tuple_equal(blob.b[0, 1].C[0, 0].shape, (5, 5))
b = unpack(pack(blob))
assert_array_equal(b[0, 0].b[0, 0].c, blob[0, 0].b[0, 0].c)
assert_array_equal(b[0, 1].b[0, 0].C, blob[0, 1].b[0, 0].C)
blob = blobs[4] # '3D double array' reshape(1:24, [2,3,4])
assert_array_equal(blob, np.r_[1:25].reshape((2, 3, 4), order='F'))
assert_true(blob.dtype == 'float64')
assert_array_equal(blob, unpack(pack(blob)))
blob = blobs[5] # reshape(uint8(1:24), [2,3,4])
assert_true(
np.array_equal(blob, np.r_[1:25].reshape((2, 3, 4), order='F')))
assert_true(blob.dtype == 'uint8')
assert_array_equal(blob, unpack(pack(blob)))
blob = blobs[6] # fftn(reshape(1:24, [2,3,4]))
assert_tuple_equal(blob.shape, (2, 3, 4))
assert_true(blob.dtype == 'complex128')
assert_array_equal(blob, unpack(pack(blob)))
def test_pack():
for x in (32, -3.7e-2, np.float64(3e31), -np.inf, np.int8(-3), np.uint8(-1),
np.int16(-33), np.uint16(-33), np.int32(-3), np.uint32(-1), np.int64(373), np.uint64(-3)):
assert_equal(x, unpack(pack(x)), "Scalars don't match!")
x = np.nan
assert_true(np.isnan(unpack(pack(x))), "nan scalar did not match!")
x = np.random.randn(8, 10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.random.randn(10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.float32(np.random.randn(3, 4, 5))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.int16(np.random.randn(1, 2, 3))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = None
assert_true(x is None, "None did not match")
x = [None]
assert_list_equal(x, unpack(pack(x)))
x = {'name': 'Anonymous', 'age': 15, 99: datetime.now(), 'range': [110, 190], (11,12): None}
assert_dict_equal(x, unpack(pack(x)), "Dict do not match!")
x = uuid.uuid4()
assert_equal(x, unpack(pack(x)), 'UUID did not match')
x = Decimal("-112122121.000003000")
assert_equal(x, unpack(pack(x)), "Decimal did not pack/unpack correctly")
x = [1, datetime.now(), {1: "one", "two": 2}, (1, 2)]
assert_list_equal(x, unpack(pack(x)), "List did not pack/unpack correctly")
x = (1, datetime.now(), {1: "one", "two": 2}, (uuid.uuid4(), 2))
assert_tuple_equal(x, unpack(pack(x)), "Tuple did not pack/unpack correctly")
x = (1, {datetime.now().date(): "today", "now": datetime.now().date()}, {"yes!": [1, 2, np.array((3, 4))]})
y = unpack(pack(x))
assert_dict_equal(x[1], y[1])
assert_array_equal(x[2]['yes!'][2], y[2]['yes!'][2])
x = {'elephant'}
assert_set_equal(x, unpack(pack(x)), "Set did not pack/unpack correctly")
x = tuple(range(10))
assert_tuple_equal(x, unpack(pack(range(10))), "Iterator did not pack/unpack correctly")
x = Decimal('1.24')
assert_true(x == unpack(pack(x)), "Decimal object did not pack/unpack correctly")
x = datetime.now()
assert_true(x == unpack(pack(x)), "Datetime object did not pack/unpack correctly")
def test_recarrays():
x = np.array([(1.0, 2), (3.0, 4)], dtype=[('x', float), ('y', int)])
assert_array_equal(x, unpack(pack(x)))
x = x.view(np.recarray)
assert_array_equal(x, unpack(pack(x)))
def test_object_arrays():
x = np.array(((1, 2, 3), True))
assert_array_equal(x, unpack(pack(x)), "Object array did not serialize correctly")
def test_object_arrays():
x = np.array(((1, 2, 3), True))
assert_array_equal(x, unpack(pack(x)),
"Object array did not serialize correctly")
def test_pack():
for x in (
32,
-3.7e-2,
np.float64(3e31),
-np.inf,
np.int8(-3),
np.uint8(-1),
np.int16(-33),
np.uint16(-33),
np.int32(-3),
np.uint32(-1),
np.int64(373),
np.uint64(-3),
):
assert_equal(x, unpack(pack(x)), "Scalars don't match!")
x = np.nan
assert_true(np.isnan(unpack(pack(x))), "nan scalar did not match!")
x = np.random.randn(8, 10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.random.randn(10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = 7j
assert_equal(x, unpack(pack(x)), "Complex scalar does not match")
x = np.float32(np.random.randn(3, 4, 5))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.int16(np.random.randn(1, 2, 3))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = None
assert_true(unpack(pack(x)) is None, "None did not match")
x = -255
y = unpack(pack(x))
assert_true(
x == y and isinstance(y, int) and not isinstance(y, np.ndarray),
"Scalar int did not match",
)
x = -25523987234234287910987234987098245697129798713407812347
y = unpack(pack(x))
assert_true(
x == y and isinstance(y, int) and not isinstance(y, np.ndarray),
"Unbounded int did not match",
)
x = 7.0
y = unpack(pack(x))
assert_true(
x == y and isinstance(y, float) and not isinstance(y, np.ndarray),
"Scalar float did not match",
)
x = 7j
y = unpack(pack(x))
assert_true(
x == y and isinstance(y, complex) and not isinstance(y, np.ndarray),
"Complex scalar did not match",
)
x = True
assert_true(unpack(pack(x)) is True, "Scalar bool did not match")
x = [None]
assert_list_equal(x, unpack(pack(x)))
x = {
"name": "Anonymous",
"age": 15,
99: datetime.now(),
"range": [110, 190],
(11, 12): None,
}
y = unpack(pack(x))
assert_dict_equal(x, y, "Dict do not match!")
assert_false(isinstance(["range"][0], np.ndarray),
"Scalar int was coerced into arrray.")
x = uuid.uuid4()
assert_equal(x, unpack(pack(x)), "UUID did not match")
x = Decimal("-112122121.000003000")
assert_equal(x, unpack(pack(x)), "Decimal did not pack/unpack correctly")
x = [1, datetime.now(), {1: "one", "two": 2}, (1, 2)]
assert_list_equal(x, unpack(pack(x)), "List did not pack/unpack correctly")
x = (1, datetime.now(), {1: "one", "two": 2}, (uuid.uuid4(), 2))
assert_tuple_equal(x, unpack(pack(x)),
"Tuple did not pack/unpack correctly")
x = (
1,
{
datetime.now().date(): "today",
"now": datetime.now().date()
},
{
"yes!": [1, 2, np.array((3, 4))]
},
)
y = unpack(pack(x))
assert_dict_equal(x[1], y[1])
assert_array_equal(x[2]["yes!"][2], y[2]["yes!"][2])
x = {"elephant"}
assert_set_equal(x, unpack(pack(x)), "Set did not pack/unpack correctly")
x = tuple(range(10))
assert_tuple_equal(x, unpack(pack(range(10))),
"Iterator did not pack/unpack correctly")
x = Decimal("1.24")
assert_true(x == unpack(pack(x)),
"Decimal object did not pack/unpack correctly")
x = datetime.now()
assert_true(x == unpack(pack(x)),
"Datetime object did not pack/unpack correctly")
x = np.bool_(True)
assert_true(x == unpack(pack(x)),
"Numpy bool object did not pack/unpack correctly")
x = "test"
assert_true(x == unpack(pack(x)),
"String object did not pack/unpack correctly")
x = np.array(["yes"])
assert_true(x == unpack(pack(x)),
"Numpy string array object did not pack/unpack correctly")
