def test_record_1(self):
# Make source collection
source_collection_id = self.database.get_or_create_collection_id(
"test", datetime.datetime.now(), False)
source_collection = self.database.get_collection(source_collection_id)
# Load some data
store = Store(self.config, self.database)
store.set_collection(source_collection)
json_filename = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'data',
'sample_1_0_record.json')
store.store_file_from_local("test.json", "http://example.com",
"record_package", "utf-8", json_filename)
# Make destination collection
destination_collection_id = self.database.get_or_create_collection_id(
source_collection.source_id,
source_collection.data_version,
source_collection.sample,
transform_from_collection_id=source_collection_id,
transform_type=TRANSFORM_TYPE_UPGRADE_1_0_TO_1_1)
destination_collection = self.database.get_collection(
destination_collection_id)
# transform!
transform = Upgrade10To11Transform(self.config, self.database,
destination_collection)
transform.process()
# check
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.record_table])
result = connection.execute(s)
assert 2 == result.rowcount
s = sa.sql.select([self.database.release_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([
self.database.transform_upgrade_1_0_to_1_1_status_record_table
])
result = connection.execute(s)
assert 1 == result.rowcount
s = sa.sql.select([
self.database.transform_upgrade_1_0_to_1_1_status_release_table
])
result = connection.execute(s)
assert 0 == result.rowcount
# transform again! This should be fine
transform = Upgrade10To11Transform(self.config, self.database,
destination_collection)
transform.process()
# check
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.record_table])
result = connection.execute(s)
assert 2 == result.rowcount
s = sa.sql.select([self.database.release_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([
self.database.transform_upgrade_1_0_to_1_1_status_record_table
])
result = connection.execute(s)
assert 1 == result.rowcount
s = sa.sql.select([
self.database.transform_upgrade_1_0_to_1_1_status_release_table
])
result = connection.execute(s)
assert 0 == result.rowcount
# destination collection will not be closed (because source is still open!)
destination_collection = self.database.get_collection(
destination_collection_id)
assert destination_collection.store_end_at == None # noqa
# Mark source collection as finished
self.database.mark_collection_store_done(source_collection_id)
# transform again! This should be fine
transform = Upgrade10To11Transform(self.config, self.database,
destination_collection)
transform.process()
# destination collection should be closed
destination_collection = self.database.get_collection(
destination_collection_id)
assert destination_collection.store_end_at != None # noqa
def test_releases_via_process_file_item_id_method(self):
collection_id = self.database.get_or_create_collection_id(
"test", datetime.datetime.now(), False)
self.database.mark_collection_check_data(collection_id, True)
self.database.mark_collection_check_older_data_with_schema_version_1_1(
collection_id, True)
collection = self.database.get_collection(collection_id)
store = Store(self.config, self.database)
store.set_collection(collection)
json_filename = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'fixtures',
'sample_1_0_release.json')
store.store_file_from_local("test.json", "http://example.com",
"release_package", "utf-8", json_filename)
file_item = self.database.get_all_files_items_in_file(
self.database.get_all_files_in_collection(collection_id)[0])[0]
# Check Number of check results
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.record_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.record_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
# Call Checks
checks = Checks(self.database, collection)
checks.process_file_item_id(file_item.database_id)
# Check Number of check results
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.record_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_table])
result = connection.execute(s)
assert 2 == result.rowcount
s = sa.sql.select([self.database.record_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
# Call Checks Again - that should be fine
checks = Checks(self.database, collection)
checks.process_file_item_id(file_item.database_id)
# Check Number of check results
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.record_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_table])
result = connection.execute(s)
assert 2 == result.rowcount
s = sa.sql.select([self.database.record_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
def test_two_collections_with_upgrade_1_0_to_1_1(self):
# Source Collection
source_collection_id = self.database.get_or_create_collection_id(
"test", datetime.datetime.now(), False)
source_collection = self.database.get_collection(source_collection_id)
store = Store(self.config, self.database)
store.set_collection(source_collection)
json_filename = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'fixtures',
'sample_1_0_record.json')
store.store_file_from_local("test.json", "http://example.com",
"record", "utf-8", json_filename)
self.database.mark_collection_store_done(source_collection_id)
# Destination Collection
destination_collection_id = self.database.get_or_create_collection_id(
"test",
datetime.datetime.now(),
False,
transform_from_collection_id=source_collection_id,
transform_type=TRANSFORM_TYPE_UPGRADE_1_0_TO_1_1)
destination_collection = self.database.get_collection(
destination_collection_id)
transform = Upgrade10To11Transform(self.config, self.database,
destination_collection)
transform.process()
# Check Number of rows in various tables
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.collection_table])
result = connection.execute(s)
assert 2 == result.rowcount
s = sa.sql.select([self.database.collection_file_table])
result = connection.execute(s)
assert 2 == result.rowcount
s = sa.sql.select([self.database.collection_file_item_table])
result = connection.execute(s)
assert 2 == result.rowcount
s = sa.sql.select([self.database.record_table])
result = connection.execute(s)
assert 2 == result.rowcount
s = sa.sql.select([
self.database.transform_upgrade_1_0_to_1_1_status_record_table
])
result = connection.execute(s)
assert 1 == result.rowcount
s = sa.sql.select([self.database.data_table])
result = connection.execute(s)
assert 0 < result.rowcount
s = sa.sql.select([self.database.package_data_table])
result = connection.execute(s)
assert 0 < result.rowcount
# Delete
self.database.mark_collection_deleted_at(source_collection_id)
self.database.delete_collection(source_collection_id)
self.database.delete_orphan_data()
self.database.mark_collection_deleted_at(destination_collection_id)
self.database.delete_collection(destination_collection_id)
self.database.delete_orphan_data()
# Check Number of rows in various tables
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.collection_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.collection_file_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.collection_file_item_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.record_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([
self.database.transform_upgrade_1_0_to_1_1_status_record_table
])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.data_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.package_data_table])
result = connection.execute(s)
assert 0 == result.rowcount
def test_records(self):
collection_id = self.database.get_or_create_collection_id(
"test", datetime.datetime.now(), False)
collection = self.database.get_collection(collection_id)
store = Store(self.config, self.database)
store.set_collection(collection)
json_filename = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'data',
'sample_1_0_record.json')
store.store_file_from_local("test.json", "http://example.com",
"record", "utf-8", json_filename)
# Check Number of check results
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.record_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.record_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
# Call Checks
checks = Checks(self.database, collection)
checks.process_all_files()
# Check Number of check results
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.record_check_table])
result = connection.execute(s)
assert 2 == result.rowcount
s = sa.sql.select([self.database.release_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.record_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
# Call Checks Again - that should be fine
checks = Checks(self.database, collection)
checks.process_all_files()
# Check Number of check results
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.record_check_table])
result = connection.execute(s)
assert 2 == result.rowcount
s = sa.sql.select([self.database.release_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.record_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
def test_a_single_collection(self):
collection_id = self.database.get_or_create_collection_id(
"test", datetime.datetime.now(), False)
collection = self.database.get_collection(collection_id)
store = Store(self.config, self.database)
store.set_collection(collection)
# Load several records to test data delete
json_filename1 = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'fixtures',
'sample_1_0_record.json')
store.store_file_from_local("test1.json", "http://example.com",
"record", "utf-8", json_filename1)
json_filename2 = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'fixtures',
'sample_1_1_record.json')
store.store_file_from_local("test2.json", "http://example.com",
"record", "utf-8", json_filename2)
# Check Number of rows in various tables
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.collection_table])
result = connection.execute(s)
assert 1 == result.rowcount
s = sa.sql.select([self.database.collection_file_table])
result = connection.execute(s)
assert 2 == result.rowcount
s = sa.sql.select([self.database.collection_file_item_table])
result = connection.execute(s)
assert 2 == result.rowcount
s = sa.sql.select([self.database.record_table])
result = connection.execute(s)
assert 2 == result.rowcount
s = sa.sql.select([self.database.data_table])
result = connection.execute(s)
assert 2 == result.rowcount
s = sa.sql.select([self.database.package_data_table])
result = connection.execute(s)
assert 2 == result.rowcount
# Delete
self.database.mark_collection_deleted_at(collection_id)
self.database.delete_collection(collection_id)
self.database.delete_orphan_data()
# Check Number of rows in various tables
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.collection_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.collection_file_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.collection_file_item_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.record_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.data_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.package_data_table])
result = connection.execute(s)
assert 0 == result.rowcount
