def load_sample_data_into_datasource(schema_only_db):
"""
load our sample test data into a dataspace
This is a function not a fixture so you can
run it on any datasource providing the right API.
"""
logging.getLogger().debug("Loading Sample data for tests")
_pk = schema_only_db.store_taskmanager(
"taskmanager1",
"11111111-1111-1111-1111-111111111111",
datetime.datetime(2016, 3, 14),
) # _pk=1 probably
header = Header(_pk)
metadata = Metadata(_pk)
schema_only_db.insert(_pk, 1, "my_test_key", b"my_test_value", header,
metadata)
schema_only_db.insert(_pk, 1, "a_test_key", b"a_test_value", header,
metadata)
_pk = schema_only_db.store_taskmanager(
"taskmanager2",
"22222222-2222-2222-2222-222222222222") # _pk=2 probably
header = Header(_pk)
metadata = Metadata(_pk)
schema_only_db.insert(_pk, 2, "other_test_key", b"other_test_value",
header, metadata)
# return the connection now that it isn't just the schema
return schema_only_db
def test_update_bad(datasource): # noqa: F811
"""Do updates fail to work on bogus taskmanager as expected"""
metadata_row = datasource.get_metadata(
taskmanager_id=1,
generation_id=1,
key="my_test_key",
)
header_row = datasource.get_header(
taskmanager_id=1,
generation_id=1,
key="my_test_key",
)
with pytest.raises(Exception):
datasource.update(
taskmanager_id=100,
generation_id=1,
key="my_test_key",
value=b"I changed IT",
header=Header(
header_row[0],
create_time=header_row[4],
expiration_time=header_row[5],
scheduled_create_time=header_row[6],
creator=header_row[7],
schema_id=header_row[8],
),
metadata=Metadata(
metadata_row[0],
state=metadata_row[4],
generation_id=metadata_row[2],
generation_time=metadata_row[5],
missed_update_count=metadata_row[6],
),
)
def test_insert(datasource): # noqa: F811
"""Can we insert new elements"""
primary_key = datasource.store_taskmanager("taskmanager3", "33333333-3333-3333-3333-333333333333")
assert primary_key > 1
header = Header(primary_key)
metadata = Metadata(primary_key)
datasource.insert(
primary_key,
1,
"sample_test_key",
b"sample_test_value",
header,
metadata,
)
result1 = datasource.get_dataproducts(taskmanager_id=primary_key)
assert result1 == [
{
"key": "sample_test_key",
"taskmanager_id": primary_key,
"generation_id": 1,
"value": b"sample_test_value",
}
]
result2 = datasource.get_dataproducts(taskmanager_id=primary_key, key="sample_test_key")
assert result1 == result2
def test_update(datasource): # noqa: F811
"""Do updates work as expected"""
metadata_row = datasource.get_metadata(
taskmanager_id=1,
generation_id=1,
key="my_test_key",
)
header_row = datasource.get_header(
taskmanager_id=1,
generation_id=1,
key="my_test_key",
)
datasource.update(
taskmanager_id=1,
generation_id=1,
key="my_test_key",
value=b"I changed IT",
header=Header(
header_row[0],
create_time=header_row[4],
expiration_time=header_row[5],
scheduled_create_time=header_row[6],
creator=header_row[7],
schema_id=header_row[8],
),
metadata=Metadata(
metadata_row[0],
state=metadata_row[4],
generation_id=metadata_row[2],
generation_time=metadata_row[5],
missed_update_count=metadata_row[6],
),
)
result1 = datasource.get_dataproduct(
taskmanager_id=1,
generation_id=1,
key="my_test_key",
)
assert result1 == b"I changed IT"
def header(data):
return Header(
data["taskmanager"][0]["taskmanager_id"]
)
def test_datablock(self):
print "Hello test_datablock"
if config['dataspace']['db_driver']['name'] == 'SQLite3DB':
filename = config['dataspace']['db_driver']['config']['filename']
if os.path.exists(filename):
os.unlink(filename)
dataspace = DataSpace(config)
taskmanager_id = 'E0B9A7F5-B55E-47F6-B5EF-DCCB8B977AFE'
generation_id = 9999
datablock = DataBlock(dataspace, taskmanager_id, generation_id)
timestamp = time.time()
key = 'aKey'
value = {"m1": "v1"}
header = Header(taskmanager_id,
create_time=timestamp,
scheduled_create_time=timestamp + 600,
schema_id=0)
metadata = Metadata(taskmanager_id,
generation_time=timestamp,
generation_id=generation_id)
print 'Doing put:\nkey=%s\nvalue=%s\n\nheader=%s\n\nmetadata=%s\n\n' % (
key, value, header, metadata)
datablock.put(key, value, header, metadata)
datablock.put('zKey', {'mz': 'vz'}, header, metadata)
print 'Doing get: key=%s ...\n' % key
db_value = datablock.get(key)
print db_value
print 'Doing get_header: key=%s ...\n' % key
db_header = datablock.get_header(key)
print db_header
print 'Doing get_metadata: key=%s ...\n' % key
db_metadata = datablock.get_metadata(key)
print db_metadata
print 'Performing comparison of value, header and metadata ...'
if (are_dicts_same(value, db_value)
and are_dicts_same(header, db_header)
and are_dicts_same(metadata, db_metadata)):
print 'DICTS CONSISTENCY CHECK PASSED\n'
else:
print 'DICTS CONSISTENCY CHECK FAILED\n'
assert (are_dicts_same(value, db_value)
and are_dicts_same(header, db_header)
and are_dicts_same(metadata, db_metadata))
# TEST: Insert new value for same key
new_value = {"m2": "v2"}
print 'Doing put:\nkey=%s\nvalue=%s\nheader=%s\nmetadata=%s\n' % (
key, new_value, header, metadata)
datablock.put(key, new_value, header, metadata)
print 'Doing get: key=%s ...\n' % key
print datablock.get(key)
db_new_value = datablock.get(key)
db_header = datablock.get_header(key)
db_metadata = datablock.get_metadata(key)
assert are_dicts_same(new_value, db_new_value)
assert (are_dicts_same(new_value, db_new_value)
and are_dicts_same(header, db_header)
and are_dicts_same(metadata, db_metadata))
# TEST: Duplicate functionality
print '-----------------------'
print 'Duplicating datablock ...\n'
dup_datablock = datablock.duplicate()
print '---'
print 'datablock.generation_id = ', datablock.generation_id
print 'Doing get: key=%s ...\n' % key
print datablock.get(key)
print 'Doing get_header: key=%s ...\n' % key
print datablock.get_header(key)
print 'Doing get_metadata: key=%s ...\n' % key
print datablock.get_metadata(key)
print '---'
print 'dup_datablock.generation_id = ', dup_datablock.generation_id
print 'Doing get on dup_datablock: key=%s\n' % key
print dup_datablock.get(key)
print 'Doing get_header on dup_datablock: key=%s ...\n' % key
print dup_datablock.get_header(key)
print 'Doing get_metadata on dup_datablock: key=%s ...\n' % key
print dup_datablock.get_metadata(key)
print '---'
# TEST: Insert new value on duplicated datablock
new_value3 = {"m3": "v3"}
dup_datablock.put(key, new_value3, dup_datablock.get_header(key),
dup_datablock.get_metadata(key))
print dup_datablock.get(key)
print dup_datablock.get_header(key)
print dup_datablock.get_metadata(key)
db_dup_value = dup_datablock.get(key)
db_dup_header = dup_datablock.get_header(key)
db_dup_metadata = dup_datablock.get_metadata(key)
assert (are_dicts_same(new_value3, db_dup_value)
and are_dicts_same(header, db_dup_header)
and are_dicts_same(metadata, db_dup_metadata))
metadata1 = Metadata(taskmanager_id,
generation_time=timestamp,
generation_id=(generation_id + 1))
db_orig_value = datablock.get(key)
db_orig_header = datablock.get_header(key)
db_orig_metadata = datablock.get_metadata(key)
assert (are_dicts_same(new_value, db_orig_value)
and are_dicts_same(header, db_orig_header)
and are_dicts_same(metadata1, db_orig_metadata))
dataspace.close()
