def check_file(source,
source_session_id,
file_info,
override_schema_version=None):
file_id = database.get_id_of_store_file(source_session_id, file_info)
with database.get_engine().begin() as connection:
release_rows = connection.execute(
database.release_table.select().where(
database.release_table.c.collection_file_status_id == file_id))
for release_row in release_rows:
if not database.is_release_check_done(
release_row['id'],
override_schema_version=override_schema_version):
check_release_row(source,
release_row,
override_schema_version=override_schema_version)
del release_rows
with database.get_engine().begin() as connection:
record_rows = connection.execute(database.record_table.select().where(
database.record_table.c.collection_file_status_id == file_id))
for record_row in record_rows:
if not database.is_record_check_done(
record_row['id'],
override_schema_version=override_schema_version):
check_record_row(source,
record_row,
override_schema_version=override_schema_version)
def get_data(data_id):
with database.get_engine().begin() as connection:
s = sa.sql.select([database.data_table]) \
.where(database.data_table.c.id == data_id)
result = connection.execute(s)
data_row = result.fetchone()
return data_row['data']
def check_release_row(source, release_row, override_schema_version=None):
package = get_package_data(release_row.package_data_id)
package['releases'] = [get_data(release_row.data_id)]
if override_schema_version:
package['version'] = override_schema_version
package = source.before_check_data(package, override_schema_version)
try:
cove_output = handle_package(package)
checks = [{
'release_id': release_row.id,
'cove_output': cove_output,
'override_schema_version': override_schema_version
}]
with database.get_engine().begin() as connection:
connection.execute(database.release_check_table.insert(), checks)
except APIException as err:
checks = [{
'release_id': release_row.id,
'error': str(err),
'override_schema_version': override_schema_version
}]
with database.get_engine().begin() as connection:
connection.execute(database.release_check_error_table.insert(),
checks)
def test_checks_records_error():
setup_main_database()
with tempfile.TemporaryDirectory() as tmpdir:
source = EmptySource(tmpdir)
metadata_db = MetadataDB(tmpdir)
metadata_db.create_session_metadata("Test", True,
"http://www.test.com",
"2018-01-01-10-00-00")
metadata_db.add_filestatus({
'filename': 'test1.json',
'url': 'http://www.test.com',
'data_type': 'record_package'
})
# store details
source_session_id = database.start_store("test_source",
"2018-01-01-10-00-00", True,
metadata_db)
for data in metadata_db.list_filestatus():
with database.add_file(source_session_id, data) as database_file:
database_file.insert_record({'record': 'totally'}, {
'version': '0.1-does-not-exist',
'extensions': []
})
database.end_store(source_session_id)
record_id = 1
# Don't like hard coding ID in. Relies on DB assigning 1 to this new row. But I think we can assume that.
# Test
assert not database.is_record_check_done(record_id)
assert not database.is_check_done(source_session_id)
# check!
for data in metadata_db.list_filestatus():
checks.check_file(source, source_session_id, data)
# Test
assert database.is_record_check_done(record_id)
assert database.is_check_done(source_session_id)
with database.get_engine().begin() as connection:
s = sa.sql.select([database.record_check_error_table])
result = connection.execute(s)
data = result.fetchone()
assert 'The schema version in your data is not valid. Accepted values:' in data[
'error']
def test_checks_releases():
setup_main_database()
with tempfile.TemporaryDirectory() as tmpdir:
source = EmptySource(tmpdir)
metadata_db = MetadataDB(tmpdir)
metadata_db.create_session_metadata("Test", True,
"http://www.test.com",
"2018-01-01-10-00-00")
metadata_db.add_filestatus({
'filename': 'test1.json',
'url': 'http://www.test.com',
'data_type': 'release_package'
})
# store details
source_session_id = database.start_store("test_source",
"2018-01-01-10-00-00", True,
metadata_db)
for data in metadata_db.list_filestatus():
with database.add_file(source_session_id, data) as database_file:
database_file.insert_release({'release': 'totally'},
{'extensions': []})
database.end_store(source_session_id)
release_id = 1
# Don't like hard coding ID in. Relies on DB assigning 1 to this new row. But I think we can assume that.
# Test
assert not database.is_release_check_done(release_id)
# check!
for data in metadata_db.list_filestatus():
checks.check_file(source, source_session_id, data)
# Test
assert database.is_release_check_done(release_id)
with database.get_engine().begin() as connection:
s = sa.sql.select([database.release_check_table])
result = connection.execute(s)
data = result.fetchone()
assert data['cove_output']['file_type'] == 'json'
assert len(data['cove_output']['validation_errors']) > 0