def test_input_compose(self, db, args, upload_schema_str):
if args and args[0].startswith('foo'):
if args[0].endswith('not_null'):
args = ('foo', ) + args[1:]
db.create_array('foo', '<val:int64 not null>[i]')
else:
db.create_array('foo', '<val:int64>[i]')
attr_name = 'val'
if (not args or args[0].startswith('{')) and not upload_schema_str:
attr_name = 'x'
assert type(
db.store(
db.input(*args,
upload_data=foo_np,
upload_schema=(Schema.fromstring(upload_schema_str)
if upload_schema_str else None)),
'foo')) == Array
assert type(
db.store(
db.apply(
db.input(
*args,
upload_data=foo_np,
upload_schema=(Schema.fromstring(upload_schema_str)
if upload_schema_str else None)),
'val2', attr_name), 'foo2')) == Array
db.remove('foo')
db.remove('foo2')
def test_input_compose(self, db, args, upload_schema_str):
if args and args[0].startswith('foo'):
if args[0].endswith('not_null'):
args = ('foo',) + args[1:]
db.create_array('foo', '<val:int64 not null>[i]')
else:
db.create_array('foo', '<val:int64>[i]')
attr_name = 'val'
if (not args or args[0].startswith('{')) and not upload_schema_str:
attr_name = 'x'
assert type(
db.store(
db.input(*args,
upload_data=foo_np,
upload_schema=(Schema.fromstring(upload_schema_str)
if upload_schema_str else None)),
'foo')) == Array
assert type(
db.store(
db.apply(
db.input(*args,
upload_data=foo_np,
upload_schema=(
Schema.fromstring(upload_schema_str)
if upload_schema_str else None)),
'val2', attr_name),
'foo2')) == Array
db.remove('foo')
db.remove('foo2')
class TestSchema:
@pytest.mark.parametrize(('string', 'expected_str', 'expected_obj'), [
('foo<x:int64>[i=0:*]', 'foo<x:int64> [i=0:*]',
Schema('foo', (Attribute('x', 'int64'), ),
(Dimension('i', 0, '*'), ))),
('foo@10<x:int64>[i=0:*]', 'foo@10<x:int64> [i=0:*]',
Schema('foo@10', (Attribute('x', 'int64'), ),
(Dimension('i', 0, '*'), ))),
('list<name:string NOT NULL,uaid:int64 NOT NULL,' +
'aid:int64 NOT NULL,schema:string NOT NULL,' +
'availability:bool NOT NULL,temporary:bool NOT NULL> ' +
'[No=0:*:0:1000000]',
'list<name:string NOT NULL,uaid:int64 NOT NULL,' +
'aid:int64 NOT NULL,schema:string NOT NULL,' +
'availability:bool NOT NULL,temporary:bool NOT NULL> ' +
'[No=0:*:0:1000000]',
Schema(
'list',
(Attribute('name', 'string', True),
Attribute('uaid', 'int64', True), Attribute(
'aid', 'int64', True), Attribute('schema', 'string', True),
Attribute('availability', 'bool',
True), Attribute('temporary', 'bool', True)),
(Dimension('No', 0, '*', 0, 1000000), ))),
('not empty operators' +
'<name:string NOT NULL,library:string NOT NULL> [No=0:52:0:53]',
'operators' +
'<name:string NOT NULL,library:string NOT NULL> [No=0:52:0:53]',
Schema('operators',
(Attribute('name', 'string', True, None, None),
Attribute('library', 'string', True, None, None)),
(Dimension('No', 0, 52, 0, 53), ))),
])
def test_fromstring(self, string, expected_str, expected_obj):
assert str(Schema.fromstring(string)) == expected_str
assert Schema.fromstring(string) == expected_obj
@pytest.mark.parametrize(('string', 'expected'), [
('<x:int64,y:double>', ['x:int64', 'y:double']),
])
def test_regex_atts(self, string, expected):
assert (
Schema._regex_atts.search(string).group(1).split(',') == expected)
@pytest.mark.parametrize(('string', 'expected'), [
('[i=0:*;j=-100:0:0:10]', ['i=0:*', 'j=-100:0:0:10']),
])
def test_regex_dims(self, string, expected):
assert (
Schema._regex_dims.search(string).group(1).split(';') == expected)
def test_iquery_numpy_null_bytes(self, db, query, upload_schema_str):
db.create_array('foo', '<val:int64>[i]')
assert db.iquery(query,
upload_data=foo_np_null.tobytes(),
upload_schema=(Schema.fromstring(upload_schema_str) if
upload_schema_str else None)) is None
db.remove('foo')
def test_iquery_numpy_null_bytes(self, db, query, upload_schema_str):
db.create_array('foo', '<val:int64>[i]')
assert db.iquery(
query,
upload_data=foo_np_null.tobytes(),
upload_schema=(Schema.fromstring(upload_schema_str)
if upload_schema_str else None)) is None
db.remove('foo')
def test_iquery_numpy(self, db, query, upload_schema_str):
db.create_array('pytest_2', '<val:int64>[i]')
assert db.iquery(
query,
upload_data=pytest_np,
upload_schema=(Schema.fromstring(upload_schema_str)
if upload_schema_str else None)) is None
db.remove('pytest_2')
def test_input_numpy(self, db, type_name, default, compression):
schema_str = (
'<x:{type_name} {default_exp} {compression_exp}>[i=0:9]'.format(
type_name=type_name,
default_exp='default {}'.format(default) if default else '',
compression_exp=("compression '{}'".format(compression)
if compression else '')))
schema = Schema.fromstring(schema_str)
assert type(db.create_array('foo', schema) == Array)
assert type(db.build(schema_str, 'null').store('foo') == Array)
db.remove('foo')
def test_input_numpy(self, db, type_name, default, compression):
schema_str = (
'<x:{type_name} {default_exp} {compression_exp}>[i=0:9]'.format(
type_name=type_name,
default_exp='default {}'.format(default) if default else '',
compression_exp=("compression '{}'".format(compression)
if compression else '')))
schema = Schema.fromstring(schema_str)
assert type(db.create_array('foo', schema) == Array)
assert type(db.build(schema_str, 'null').store('foo') == Array)
db.remove('foo')
def test_load_numpy_null_bytes(self, db, args, upload_schema_str):
if args[0].endswith('not_null'):
args = ('foo', ) + args[1:]
db.create_array('foo', '<val:int64 not null>[i]')
else:
db.create_array('foo', '<val:int64>[i]')
assert type(
db.load(*args,
upload_data=foo_np_null.tobytes(),
upload_schema=(Schema.fromstring(upload_schema_str)
if upload_schema_str else None))) == Array
db.remove('foo')
def test_load_numpy_null_bytes(self, db, args, upload_schema_str):
if args[0].endswith('not_null'):
args = ('foo',) + args[1:]
db.create_array('foo', '<val:int64 not null>[i]')
else:
db.create_array('foo', '<val:int64>[i]')
assert type(
db.load(
*args,
upload_data=foo_np_null.tobytes(),
upload_schema=(Schema.fromstring(upload_schema_str)
if upload_schema_str else None))) == Array
db.remove('foo')
def test_load_numpy(self, db, args, upload_schema_str):
if args and args[0].startswith('pytest_15'):
if args[0].endswith('not_null'):
args = ('pytest_15',) + args[1:]
db.create_array('pytest_15', '<val:int64 not null>[i]')
else:
db.create_array('pytest_15', '<val:int64>[i]')
assert type(
db.load('pytest_15',
*args[1:],
upload_data=pytest_np,
upload_schema=(Schema.fromstring(upload_schema_str)
if upload_schema_str else None))) == Array
db.remove('pytest_15')
def test_input_numpy_null(self, db, args, upload_schema_str):
if args and args[0].startswith('foo'):
if args[0].endswith('not_null'):
args = ('foo', ) + args[1:]
db.create_array('foo', '<val:int64 not null>[i]')
else:
db.create_array('foo', '<val:int64>[i]')
assert type(
db.input(*args,
upload_data=foo_np_null,
upload_schema=(Schema.fromstring(upload_schema_str)
if upload_schema_str else
None)).store('foo')) == Array
db.remove('foo')
def test_input_numpy_null(self, db, args, upload_schema_str):
if args and args[0].startswith('foo'):
if args[0].endswith('not_null'):
args = ('foo',) + args[1:]
db.create_array('foo', '<val:int64 not null>[i]')
else:
db.create_array('foo', '<val:int64>[i]')
assert type(
db.input(*args,
upload_data=foo_np_null,
upload_schema=(Schema.fromstring(upload_schema_str)
if upload_schema_str else None)).store(
'foo')) == Array
db.remove('foo')
def test_input_chain(self, db, args, upload_schema_str):
is_pytest = False
if args and args[0].startswith('pytest_12'):
is_pytest = True
if args[0].endswith('not_null'):
args = ('pytest_12',) + args[1:]
db.create_array('pytest_12', '<val:int64 not null>[i]')
else:
db.create_array('pytest_12', '<val:int64>[i]')
attr_name = 'val'
if (not args or args[0].startswith('{')) and not upload_schema_str:
attr_name = 'x'
assert type(
db.input(*args,
upload_data=pytest_np,
upload_schema=(Schema.fromstring(upload_schema_str)
if upload_schema_str else None)).apply(
'val2', attr_name).store(
'pytest_13')) == Array
if is_pytest:
db.remove('pytest_12')
db.remove('pytest_13')
class TestVariety:
@pytest.mark.parametrize('schema', [
None,
variety_schema,
Schema.fromstring(variety_schema),
])
def test_variety_numpy(self, db, variety, schema):
# NumPy array
ar = iquery(db,
'scan({})'.format(variety),
fetch=True,
as_dataframe=False,
schema=schema)
assert ar.shape == (12, )
assert ar.ndim == 1
assert ar[0] == variety_array_struct[0]
assert ar[4] == variety_array_struct[1]
assert ar[8] == variety_array_struct[2]
@pytest.mark.parametrize('schema', [
None,
variety_schema,
Schema.fromstring(variety_schema),
])
def test_variety_numpy_atts(self, db, variety, schema):
# NumPy array, atts_only
ar = iquery(db,
'scan({})'.format(variety),
fetch=True,
atts_only=True,
as_dataframe=False,
schema=schema)
assert ar.shape == (12, )
assert ar.ndim == 1
assert ar[0] == variety_array_struct[variety_atts][0]
assert ar[4] == variety_array_struct[variety_atts][1]
assert ar[8] == variety_array_struct[variety_atts][2]
@pytest.mark.parametrize('schema', [
None,
variety_schema,
Schema.fromstring(variety_schema),
])
def test_variety_dataframe(self, db, variety, schema):
# Pandas DataFrame, atts_only
ar = iquery(db, 'scan({})'.format(variety), fetch=True)
assert ar.shape == (12, 16)
assert ar.ndim == 2
assert numpy.all(ar[0:1].values[0] == variety_array_promo[0])
# Values which differ have to be NAN
ln = ar[4:5]
assert numpy.all(
numpy.isnan(ln[ar.columns[(ln.values[0] !=
variety_array_promo[1]).tolist()]]))
ln = ar[8:9]
assert numpy.all(
numpy.isnan(ln[ar.columns[(ln.values[0] !=
variety_array_promo[2]).tolist()]]))
@pytest.mark.parametrize('schema', [
None,
variety_schema,
Schema.fromstring(variety_schema),
])
def test_variety_dataframe_atts(self, db, variety, schema):
# Pandas DataFrame, atts_only
ar = iquery(db, 'scan({})'.format(variety), fetch=True, atts_only=True)
assert ar.shape == (12, 13)
assert ar.ndim == 2
assert numpy.all(ar[0:1].values == variety_array_promo[0][3:])
# Values which differ have to be NAN
ln = ar[4:5]
assert numpy.all(
numpy.isnan(
ln[ar.columns[(ln.values != variety_array_promo[1][3:])[0]]]))
ln = ar[8:9]
assert numpy.all(
numpy.isnan(
ln[ar.columns[(ln.values != variety_array_promo[2][3:])[0]]]))
@pytest.mark.parametrize('schema', [
None,
variety_schema,
Schema.fromstring(variety_schema),
])
def test_variety_dataframe_no_promo(self, db, variety, schema):
# Pandas DataFrame, atts_only
ar = iquery(db,
'scan({})'.format(variety),
fetch=True,
dataframe_promo=False)
assert ar.shape == (12, 16)
assert ar.ndim == 2
assert ar[0:1].to_records(index=False) == variety_array_obj[0]
assert ar[4:5].to_records(index=False) == variety_array_obj[1]
assert ar[8:9].to_records(index=False) == variety_array_obj[2]
def test_fromstring(self, string, expected_str, expected_obj):
assert str(Schema.fromstring(string)) == expected_str
assert Schema.fromstring(string) == expected_obj
class TestTypes:
@pytest.mark.parametrize(
('mode', 'schema'),
((mode, schema) for mode in modes
for schema in (None, types_schema[mode],
Schema.fromstring(types_schema[mode]))))
def test_types_numpy(self, db, setup, mode, schema):
# NumPy array
ar = iquery(db,
'scan(types_{})'.format(mode),
fetch=True,
as_dataframe=False,
schema=schema)
assert ar.shape == (12, )
assert ar.ndim == 1
assert ar[0] == types_array_struct[mode][0]
assert ar[4] == types_array_struct[mode][1]
assert ar[8] == types_array_struct[mode][2]
@pytest.mark.parametrize(
('mode', 'schema'),
((mode, schema) for mode in modes
for schema in (None, types_schema[mode],
Schema.fromstring(types_schema[mode]))))
def test_types_numpy_atts(self, db, setup, mode, schema):
# NumPy array, atts_only
ar = iquery(db,
'scan(types_{})'.format(mode),
fetch=True,
atts_only=True,
as_dataframe=False,
schema=schema)
assert ar.shape == (12, )
assert ar.ndim == 1
assert ar[0] == types_array_struct[mode][types_atts[mode]][0]
assert ar[4] == types_array_struct[mode][types_atts[mode]][1]
assert ar[8] == types_array_struct[mode][types_atts[mode]][2]
@pytest.mark.parametrize(
('mode', 'schema'),
((mode, schema) for mode in modes
for schema in (None, types_schema[mode],
Schema.fromstring(types_schema[mode]))))
def test_types_dataframe(self, db, setup, mode, schema):
# Pandas DataFrame, atts_only
ar = iquery(db, 'scan(types_{})'.format(mode), fetch=True)
assert ar.shape == types_shape[mode][0]
assert ar.ndim == 2
assert numpy.all(ar[0:1].values[0] == types_array_promo[mode][0])
# Values which differ have to be NAN
ln = ar[4:5]
assert numpy.all(
pandas.isnull(ln[ar.columns[(
ln.values[0] != types_array_promo[mode][1]).tolist()]]))
ln = ar[8:9]
assert numpy.all(
pandas.isnull(ln[ar.columns[(
ln.values[0] != types_array_promo[mode][2]).tolist()]]))
@pytest.mark.parametrize(
('mode', 'schema'),
((mode, schema) for mode in modes
for schema in (None, types_schema[mode],
Schema.fromstring(types_schema[mode]))))
def test_types_dataframe_atts(self, db, setup, mode, schema):
# Pandas DataFrame, atts_only
ar = iquery(db,
'scan(types_{})'.format(mode),
fetch=True,
atts_only=True)
assert ar.shape == types_shape[mode][1]
assert ar.ndim == 2
assert numpy.all(ar[0:1].values == types_array_promo[mode][0][3:])
# Values which differ have to be NAN
ln = ar[4:5]
assert numpy.all(
pandas.isnull(ln[ar.columns[(ln.values !=
types_array_promo[mode][1][3:])[0]]]))
ln = ar[8:9]
assert numpy.all(
pandas.isnull(ln[ar.columns[(ln.values !=
types_array_promo[mode][2][3:])[0]]]))
@pytest.mark.parametrize(
('mode', 'schema'),
((mode, schema) for mode in modes
for schema in (None, types_schema[mode],
Schema.fromstring(types_schema[mode]))))
def test_types_dataframe_no_promo(self, db, setup, mode, schema):
# Pandas DataFrame, atts_only
ar = iquery(db,
'scan(types_{})'.format(mode),
fetch=True,
dataframe_promo=False)
assert ar.shape == types_shape[mode][0]
assert ar.ndim == 2
assert ar[0:1].to_records(index=False) == types_array_obj[mode][0]
assert ar[4:5].to_records(index=False) == types_array_obj[mode][1]
assert ar[8:9].to_records(index=False) == types_array_obj[mode][2]
