def test_recarrays():
x = np.array([(1.0, 2), (3.0, 4)], dtype=[('x', float), ('y', int)])
assert_array_equal(x, unpack(pack(x)))
x = x.view(np.recarray)
assert_array_equal(x, unpack(pack(x)))
x = np.array([(3, 4)], dtype=[('tmp0', float),
('tmp1', 'O')]).view(np.recarray)
assert_array_equal(x, unpack(pack(x)))
def test_recarrays():
x = np.array([(1.0, 2), (3.0, 4)], dtype=[("x", float), ("y", int)])
assert_array_equal(x, unpack(pack(x)))
x = x.view(np.recarray)
assert_array_equal(x, unpack(pack(x)))
x = np.array([(3, 4)], dtype=[("tmp0", float),
("tmp1", "O")]).view(np.recarray)
assert_array_equal(x, unpack(pack(x)))
def test_pack():
x = np.random.randn(8, 10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.random.randn(10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.float32(np.random.randn(3, 4, 5))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.int16(np.random.randn(1, 2, 3))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = {'name': 'Anonymous', 'age': 15}
assert_true(x == unpack(pack(x), as_dict=True), "Dict do not match!")
x = [1, 2, 3, 4]
assert_array_equal(x, unpack(pack(x)),
"List did not pack/unpack correctly")
x = [1, 2, 3, 4]
assert_array_equal(x, unpack(pack(x.__iter__())),
"Iterator did not pack/unpack correctly")
x = Decimal('1.24')
assert_true(
float(x) == unpack(pack(x)),
"Decimal object did not pack/unpack correctly")
x = datetime.now()
assert_true(
str(x) == unpack(pack(x)),
"Datetime object did not pack/unpack correctly")
def test_complex():
z = np.random.randn(8, 10) + 1j*np.random.randn(8,10)
assert_array_equal(z, unpack(pack(z)), "Arrays do not match!")
z = np.random.randn(10) + 1j*np.random.randn(10)
assert_array_equal(z, unpack(pack(z)), "Arrays do not match!")
x = np.float32(np.random.randn(3, 4, 5)) + 1j*np.float32(np.random.randn(3, 4, 5))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.int16(np.random.randn(1, 2, 3)) + 1j*np.int16(np.random.randn(1, 2, 3))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
def test_pack():
x = np.random.randn(8, 10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.random.randn(10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.float32(np.random.randn(3, 4, 5))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.int16(np.random.randn(1, 2, 3))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
def test_remove_object_exception():
# https://github.com/datajoint/datajoint-python/issues/952
# Insert some test data and remove it so that the external table is populated
test = [1, [1, 2, 3]]
SimpleRemote.insert1(test)
SimpleRemote.delete()
# Save the old external table minio client
old_client = schema.external['share'].s3.client
# Apply our new minio client which has a user that does not exist
schema.external['share'].s3.client = Minio(S3_CONN_INFO['endpoint'],
access_key='jeffjeff',
secret_key='jeffjeff',
secure=False)
# This method returns a list of errors
error_list = schema.external['share'].delete(
delete_external_files=True, errors_as_string=False)
# Teardown
schema.external['share'].s3.client = old_client
schema.external['share'].delete(delete_external_files=True)
# Raise the error we want if the error matches the expected uuid
if str(error_list[0][0]) == str(uuid_from_buffer(pack(test[1]))):
raise error_list[0][2]
def test_external_put():
"""
external storage put and get and remove
"""
ext = ExternalTable(schema.connection,
store='raw',
database=schema.database)
initial_length = len(ext)
input_ = np.random.randn(3, 7, 8)
count = 7
extra = 3
for i in range(count):
hash1 = ext.put(pack(input_))
for i in range(extra):
hash2 = ext.put(pack(np.random.randn(4, 3, 2)))
fetched_hashes = ext.fetch('hash')
assert_true(all(hash in fetched_hashes for hash in (hash1, hash2)))
assert_equal(len(ext), initial_length + 1 + extra)
output_ = unpack(ext.get(hash1))
assert_array_equal(input_, output_)
def check_table(schema, rel):
conn = rel.connection
db = rel.database
tab = rel.table_name
hdr = rel.heading
key = rel.primary_key
print('checking {}.{}'.format(db, tab))
ext = [b for b in hdr.blobs if hdr.attributes[b].type == 'external']
if not ext:
print('... skipping {}.{} - no external fields', db, tab)
return
for e in ext:
fields = [*key, e]
q = 'select {} from `{}`.`{}`'.format(','.join(fields), db, tab)
for r in conn.query(q):
try:
h = r[len(fields) - 1]
b = schema.external_table.get(h)
# can get None in some cases.. not sure when/where why.
# XXX: slooow.. double pack
if b is None or h != long_hash(pack(b)):
raise IntegrityError('hash mismatch')
print('... {}: OK!'.format(r)) # todo: verbose flag?
except AssertionError:
print('... {}: size mismatch'.format(r))
except IntegrityError:
print('... {}: hash mismatch'.format(r))
except DataJointError as e:
matched = False
for m in [re.match('^Lost.*?blob (.*)\.$', str(e))]:
print('... {}: missing file {}'.format(r, m.groups()[0]))
matched = True
if not matched:
raise
def test_complex_matlab_blobs():
"""
test correct de-serialization of various blob types
"""
blobs = Blob().fetch('blob', order_by='KEY')
blob = blobs[0] # 'simple string' 'character string'
assert_equal(blob[0], 'character string')
blob = blobs[1] # '1D vector' 1:15:180
assert_array_equal(blob, np.r_[1:180:15][None, :])
assert_array_equal(blob, unpack(pack(blob)))
blob = blobs[2] # 'string array' {'string1' 'string2'}
assert_true(isinstance(blob, dj.MatCell))
assert_array_equal(blob, np.array([['string1', 'string2']]))
assert_array_equal(blob, unpack(pack(blob)))
blob = blobs[
3] # 'struct array' struct('a', {1,2}, 'b', {struct('c', magic(3)), struct('C', magic(5))})
assert_true(isinstance(blob, dj.MatStruct))
assert_tuple_equal(blob.dtype.names, ('a', 'b'))
assert_array_equal(blob.a[0, 0], np.array([[1.]]))
assert_array_equal(blob.a[0, 1], np.array([[2.]]))
assert_true(isinstance(blob.b[0, 1], dj.MatStruct))
assert_tuple_equal(blob.b[0, 1].C[0, 0].shape, (5, 5))
b = unpack(pack(blob))
assert_array_equal(b[0, 0].b[0, 0].c, blob[0, 0].b[0, 0].c)
assert_array_equal(b[0, 1].b[0, 0].C, blob[0, 1].b[0, 0].C)
blob = blobs[4] # '3D double array' reshape(1:24, [2,3,4])
assert_array_equal(blob, np.r_[1:25].reshape((2, 3, 4), order='F'))
assert_true(blob.dtype == 'float64')
assert_array_equal(blob, unpack(pack(blob)))
blob = blobs[5] # reshape(uint8(1:24), [2,3,4])
assert_true(
np.array_equal(blob, np.r_[1:25].reshape((2, 3, 4), order='F')))
assert_true(blob.dtype == 'uint8')
assert_array_equal(blob, unpack(pack(blob)))
blob = blobs[6] # fftn(reshape(1:24, [2,3,4]))
assert_tuple_equal(blob.shape, (2, 3, 4))
assert_true(blob.dtype == 'complex128')
assert_array_equal(blob, unpack(pack(blob)))
def test_pack():
for x in (
32,
-3.7e-2,
np.float64(3e31),
-np.inf,
np.int8(-3),
np.uint8(-1),
np.int16(-33),
np.uint16(-33),
np.int32(-3),
np.uint32(-1),
np.int64(373),
np.uint64(-3),
):
assert_equal(x, unpack(pack(x)), "Scalars don't match!")
x = np.nan
assert_true(np.isnan(unpack(pack(x))), "nan scalar did not match!")
x = np.random.randn(8, 10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.random.randn(10)
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = 7j
assert_equal(x, unpack(pack(x)), "Complex scalar does not match")
x = np.float32(np.random.randn(3, 4, 5))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = np.int16(np.random.randn(1, 2, 3))
assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
x = None
assert_true(unpack(pack(x)) is None, "None did not match")
x = -255
y = unpack(pack(x))
assert_true(
x == y and isinstance(y, int) and not isinstance(y, np.ndarray),
"Scalar int did not match",
)
x = -25523987234234287910987234987098245697129798713407812347
y = unpack(pack(x))
assert_true(
x == y and isinstance(y, int) and not isinstance(y, np.ndarray),
"Unbounded int did not match",
)
x = 7.0
y = unpack(pack(x))
assert_true(
x == y and isinstance(y, float) and not isinstance(y, np.ndarray),
"Scalar float did not match",
)
x = 7j
y = unpack(pack(x))
assert_true(
x == y and isinstance(y, complex) and not isinstance(y, np.ndarray),
"Complex scalar did not match",
)
x = True
assert_true(unpack(pack(x)) is True, "Scalar bool did not match")
x = [None]
assert_list_equal(x, unpack(pack(x)))
x = {
"name": "Anonymous",
"age": 15,
99: datetime.now(),
"range": [110, 190],
(11, 12): None,
}
y = unpack(pack(x))
assert_dict_equal(x, y, "Dict do not match!")
assert_false(isinstance(["range"][0], np.ndarray),
"Scalar int was coerced into arrray.")
x = uuid.uuid4()
assert_equal(x, unpack(pack(x)), "UUID did not match")
x = Decimal("-112122121.000003000")
assert_equal(x, unpack(pack(x)), "Decimal did not pack/unpack correctly")
x = [1, datetime.now(), {1: "one", "two": 2}, (1, 2)]
assert_list_equal(x, unpack(pack(x)), "List did not pack/unpack correctly")
x = (1, datetime.now(), {1: "one", "two": 2}, (uuid.uuid4(), 2))
assert_tuple_equal(x, unpack(pack(x)),
"Tuple did not pack/unpack correctly")
x = (
1,
{
datetime.now().date(): "today",
"now": datetime.now().date()
},
{
"yes!": [1, 2, np.array((3, 4))]
},
)
y = unpack(pack(x))
assert_dict_equal(x[1], y[1])
assert_array_equal(x[2]["yes!"][2], y[2]["yes!"][2])
x = {"elephant"}
assert_set_equal(x, unpack(pack(x)), "Set did not pack/unpack correctly")
x = tuple(range(10))
assert_tuple_equal(x, unpack(pack(range(10))),
"Iterator did not pack/unpack correctly")
x = Decimal("1.24")
assert_true(x == unpack(pack(x)),
"Decimal object did not pack/unpack correctly")
x = datetime.now()
assert_true(x == unpack(pack(x)),
"Datetime object did not pack/unpack correctly")
x = np.bool_(True)
assert_true(x == unpack(pack(x)),
"Numpy bool object did not pack/unpack correctly")
x = "test"
assert_true(x == unpack(pack(x)),
"String object did not pack/unpack correctly")
x = np.array(["yes"])
assert_true(x == unpack(pack(x)),
"Numpy string array object did not pack/unpack correctly")
def test_object_arrays():
x = np.array(((1, 2, 3), True))
assert_array_equal(x, unpack(pack(x)),
"Object array did not serialize correctly")
def mkone(i):
one = np.random.random(10)
hsh = long_hash(pack(one))
return (i, one, hsh)
