def test_recarrays(): x = np.array([(1.0, 2), (3.0, 4)], dtype=[('x', float), ('y', int)]) assert_array_equal(x, unpack(pack(x))) x = x.view(np.recarray) assert_array_equal(x, unpack(pack(x))) x = np.array([(3, 4)], dtype=[('tmp0', float), ('tmp1', 'O')]).view(np.recarray) assert_array_equal(x, unpack(pack(x)))
def test_recarrays(): x = np.array([(1.0, 2), (3.0, 4)], dtype=[("x", float), ("y", int)]) assert_array_equal(x, unpack(pack(x))) x = x.view(np.recarray) assert_array_equal(x, unpack(pack(x))) x = np.array([(3, 4)], dtype=[("tmp0", float), ("tmp1", "O")]).view(np.recarray) assert_array_equal(x, unpack(pack(x)))
def test_pack(): x = np.random.randn(8, 10) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") x = np.random.randn(10) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") x = np.float32(np.random.randn(3, 4, 5)) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") x = np.int16(np.random.randn(1, 2, 3)) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") x = {'name': 'Anonymous', 'age': 15} assert_true(x == unpack(pack(x), as_dict=True), "Dict do not match!") x = [1, 2, 3, 4] assert_array_equal(x, unpack(pack(x)), "List did not pack/unpack correctly") x = [1, 2, 3, 4] assert_array_equal(x, unpack(pack(x.__iter__())), "Iterator did not pack/unpack correctly") x = Decimal('1.24') assert_true( float(x) == unpack(pack(x)), "Decimal object did not pack/unpack correctly") x = datetime.now() assert_true( str(x) == unpack(pack(x)), "Datetime object did not pack/unpack correctly")
def test_complex(): z = np.random.randn(8, 10) + 1j*np.random.randn(8,10) assert_array_equal(z, unpack(pack(z)), "Arrays do not match!") z = np.random.randn(10) + 1j*np.random.randn(10) assert_array_equal(z, unpack(pack(z)), "Arrays do not match!") x = np.float32(np.random.randn(3, 4, 5)) + 1j*np.float32(np.random.randn(3, 4, 5)) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") x = np.int16(np.random.randn(1, 2, 3)) + 1j*np.int16(np.random.randn(1, 2, 3)) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
def test_pack(): x = np.random.randn(8, 10) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") x = np.random.randn(10) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") x = np.float32(np.random.randn(3, 4, 5)) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") x = np.int16(np.random.randn(1, 2, 3)) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
def test_remove_object_exception(): # https://github.com/datajoint/datajoint-python/issues/952 # Insert some test data and remove it so that the external table is populated test = [1, [1, 2, 3]] SimpleRemote.insert1(test) SimpleRemote.delete() # Save the old external table minio client old_client = schema.external['share'].s3.client # Apply our new minio client which has a user that does not exist schema.external['share'].s3.client = Minio(S3_CONN_INFO['endpoint'], access_key='jeffjeff', secret_key='jeffjeff', secure=False) # This method returns a list of errors error_list = schema.external['share'].delete( delete_external_files=True, errors_as_string=False) # Teardown schema.external['share'].s3.client = old_client schema.external['share'].delete(delete_external_files=True) # Raise the error we want if the error matches the expected uuid if str(error_list[0][0]) == str(uuid_from_buffer(pack(test[1]))): raise error_list[0][2]
def test_external_put(): """ external storage put and get and remove """ ext = ExternalTable(schema.connection, store='raw', database=schema.database) initial_length = len(ext) input_ = np.random.randn(3, 7, 8) count = 7 extra = 3 for i in range(count): hash1 = ext.put(pack(input_)) for i in range(extra): hash2 = ext.put(pack(np.random.randn(4, 3, 2))) fetched_hashes = ext.fetch('hash') assert_true(all(hash in fetched_hashes for hash in (hash1, hash2))) assert_equal(len(ext), initial_length + 1 + extra) output_ = unpack(ext.get(hash1)) assert_array_equal(input_, output_)
def check_table(schema, rel): conn = rel.connection db = rel.database tab = rel.table_name hdr = rel.heading key = rel.primary_key print('checking {}.{}'.format(db, tab)) ext = [b for b in hdr.blobs if hdr.attributes[b].type == 'external'] if not ext: print('... skipping {}.{} - no external fields', db, tab) return for e in ext: fields = [*key, e] q = 'select {} from `{}`.`{}`'.format(','.join(fields), db, tab) for r in conn.query(q): try: h = r[len(fields) - 1] b = schema.external_table.get(h) # can get None in some cases.. not sure when/where why. # XXX: slooow.. double pack if b is None or h != long_hash(pack(b)): raise IntegrityError('hash mismatch') print('... {}: OK!'.format(r)) # todo: verbose flag? except AssertionError: print('... {}: size mismatch'.format(r)) except IntegrityError: print('... {}: hash mismatch'.format(r)) except DataJointError as e: matched = False for m in [re.match('^Lost.*?blob (.*)\.$', str(e))]: print('... {}: missing file {}'.format(r, m.groups()[0])) matched = True if not matched: raise
def test_complex_matlab_blobs(): """ test correct de-serialization of various blob types """ blobs = Blob().fetch('blob', order_by='KEY') blob = blobs[0] # 'simple string' 'character string' assert_equal(blob[0], 'character string') blob = blobs[1] # '1D vector' 1:15:180 assert_array_equal(blob, np.r_[1:180:15][None, :]) assert_array_equal(blob, unpack(pack(blob))) blob = blobs[2] # 'string array' {'string1' 'string2'} assert_true(isinstance(blob, dj.MatCell)) assert_array_equal(blob, np.array([['string1', 'string2']])) assert_array_equal(blob, unpack(pack(blob))) blob = blobs[ 3] # 'struct array' struct('a', {1,2}, 'b', {struct('c', magic(3)), struct('C', magic(5))}) assert_true(isinstance(blob, dj.MatStruct)) assert_tuple_equal(blob.dtype.names, ('a', 'b')) assert_array_equal(blob.a[0, 0], np.array([[1.]])) assert_array_equal(blob.a[0, 1], np.array([[2.]])) assert_true(isinstance(blob.b[0, 1], dj.MatStruct)) assert_tuple_equal(blob.b[0, 1].C[0, 0].shape, (5, 5)) b = unpack(pack(blob)) assert_array_equal(b[0, 0].b[0, 0].c, blob[0, 0].b[0, 0].c) assert_array_equal(b[0, 1].b[0, 0].C, blob[0, 1].b[0, 0].C) blob = blobs[4] # '3D double array' reshape(1:24, [2,3,4]) assert_array_equal(blob, np.r_[1:25].reshape((2, 3, 4), order='F')) assert_true(blob.dtype == 'float64') assert_array_equal(blob, unpack(pack(blob))) blob = blobs[5] # reshape(uint8(1:24), [2,3,4]) assert_true( np.array_equal(blob, np.r_[1:25].reshape((2, 3, 4), order='F'))) assert_true(blob.dtype == 'uint8') assert_array_equal(blob, unpack(pack(blob))) blob = blobs[6] # fftn(reshape(1:24, [2,3,4])) assert_tuple_equal(blob.shape, (2, 3, 4)) assert_true(blob.dtype == 'complex128') assert_array_equal(blob, unpack(pack(blob)))
def test_pack(): for x in ( 32, -3.7e-2, np.float64(3e31), -np.inf, np.int8(-3), np.uint8(-1), np.int16(-33), np.uint16(-33), np.int32(-3), np.uint32(-1), np.int64(373), np.uint64(-3), ): assert_equal(x, unpack(pack(x)), "Scalars don't match!") x = np.nan assert_true(np.isnan(unpack(pack(x))), "nan scalar did not match!") x = np.random.randn(8, 10) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") x = np.random.randn(10) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") x = 7j assert_equal(x, unpack(pack(x)), "Complex scalar does not match") x = np.float32(np.random.randn(3, 4, 5)) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") x = np.int16(np.random.randn(1, 2, 3)) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") x = None assert_true(unpack(pack(x)) is None, "None did not match") x = -255 y = unpack(pack(x)) assert_true( x == y and isinstance(y, int) and not isinstance(y, np.ndarray), "Scalar int did not match", ) x = -25523987234234287910987234987098245697129798713407812347 y = unpack(pack(x)) assert_true( x == y and isinstance(y, int) and not isinstance(y, np.ndarray), "Unbounded int did not match", ) x = 7.0 y = unpack(pack(x)) assert_true( x == y and isinstance(y, float) and not isinstance(y, np.ndarray), "Scalar float did not match", ) x = 7j y = unpack(pack(x)) assert_true( x == y and isinstance(y, complex) and not isinstance(y, np.ndarray), "Complex scalar did not match", ) x = True assert_true(unpack(pack(x)) is True, "Scalar bool did not match") x = [None] assert_list_equal(x, unpack(pack(x))) x = { "name": "Anonymous", "age": 15, 99: datetime.now(), "range": [110, 190], (11, 12): None, } y = unpack(pack(x)) assert_dict_equal(x, y, "Dict do not match!") assert_false(isinstance(["range"][0], np.ndarray), "Scalar int was coerced into arrray.") x = uuid.uuid4() assert_equal(x, unpack(pack(x)), "UUID did not match") x = Decimal("-112122121.000003000") assert_equal(x, unpack(pack(x)), "Decimal did not pack/unpack correctly") x = [1, datetime.now(), {1: "one", "two": 2}, (1, 2)] assert_list_equal(x, unpack(pack(x)), "List did not pack/unpack correctly") x = (1, datetime.now(), {1: "one", "two": 2}, (uuid.uuid4(), 2)) assert_tuple_equal(x, unpack(pack(x)), "Tuple did not pack/unpack correctly") x = ( 1, { datetime.now().date(): "today", "now": datetime.now().date() }, { "yes!": [1, 2, np.array((3, 4))] }, ) y = unpack(pack(x)) assert_dict_equal(x[1], y[1]) assert_array_equal(x[2]["yes!"][2], y[2]["yes!"][2]) x = {"elephant"} assert_set_equal(x, unpack(pack(x)), "Set did not pack/unpack correctly") x = tuple(range(10)) assert_tuple_equal(x, unpack(pack(range(10))), "Iterator did not pack/unpack correctly") x = Decimal("1.24") assert_true(x == unpack(pack(x)), "Decimal object did not pack/unpack correctly") x = datetime.now() assert_true(x == unpack(pack(x)), "Datetime object did not pack/unpack correctly") x = np.bool_(True) assert_true(x == unpack(pack(x)), "Numpy bool object did not pack/unpack correctly") x = "test" assert_true(x == unpack(pack(x)), "String object did not pack/unpack correctly") x = np.array(["yes"]) assert_true(x == unpack(pack(x)), "Numpy string array object did not pack/unpack correctly")
def test_object_arrays(): x = np.array(((1, 2, 3), True)) assert_array_equal(x, unpack(pack(x)), "Object array did not serialize correctly")
def mkone(i): one = np.random.random(10) hsh = long_hash(pack(one)) return (i, one, hsh)