def run_command(self, args):
run_until_timestamp = None
run_for_seconds = int(args.runforseconds) if args.runforseconds else 0
if run_for_seconds > 0:
run_until_timestamp = datetime.datetime.utcnow().timestamp(
) + run_for_seconds
# This is a safeguard - the process should stop itself but this will kill it if it does not.
def exitfunc():
os._exit(0)
Timer(run_for_seconds + 60, exitfunc).start()
for collection in self.database.get_all_collections():
if not args.quiet:
print("Collection " + str(collection.database_id))
checks = Checks(self.database,
collection,
run_until_timestamp=run_until_timestamp)
checks.process_all_files()
# Early return?
if run_until_timestamp and run_until_timestamp < datetime.datetime.utcnow(
).timestamp():
break
# If the code above took less than 60 seconds the process will stay open, waiting for the Timer to execute.
# So just kill it to make sure.
os._exit(0)
def test_releases(self):
self.config.default_value_collection_check_data = False
self.config.default_value_collection_check_older_data_with_schema_version_1_1 = False
collection_id = self.database.get_or_create_collection_id(
"test", datetime.datetime.now(), False)
collection = self.database.get_collection(collection_id)
store = Store(self.config, self.database)
store.set_collection(collection)
json_filename = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'data',
'sample_1_0_release.json')
store.store_file_from_local("test.json", "http://example.com",
"release_package", "utf-8", json_filename)
# Check Number of check results
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.record_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.record_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
# Call Checks
checks = Checks(self.database, collection)
checks.process_all_files()
# Check Number of check results
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.record_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.record_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
def process(self, message_as_string, run_until_timestamp=None):
message_as_data = json.loads(message_as_string)
if message_as_data['type'] == 'collection-data-store-finished':
collection = self.database.get_collection(
message_as_data['collection_id'])
if collection:
checks = Checks(self.database,
collection,
run_until_timestamp=run_until_timestamp)
checks.process_all_files()
def process(self, message_as_string, run_until_timestamp=None):
message_as_data = json.loads(message_as_string)
if message_as_data['type'] == 'collection-data-store-finished':
collection = self.database.get_collection(
message_as_data['collection_id'])
if collection:
checks = Checks(self.database,
collection,
run_until_timestamp=run_until_timestamp)
# Older messages might not have the extra data in, so we need to check for this.
if 'collection_file_item_id' in message_as_data and message_as_data[
'collection_file_item_id']:
checks.process_file_item_id(
message_as_data['collection_file_item_id'])
else:
checks.process_all_files()
def test_records(self):
collection_id = self.database.get_or_create_collection_id(
"test", datetime.datetime.now(), False)
self.database.mark_collection_check_older_data_with_schema_version_1_1(
collection_id, True)
collection = self.database.get_collection(collection_id)
store = Store(self.config, self.database)
store.set_collection(collection)
json_filename = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'data',
'sample_1_0_record.json')
store.store_file_from_local("test.json", "http://example.com",
"record", "utf-8", json_filename)
# Check Number of check results
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.record_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.record_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
# Call Checks
checks = Checks(self.database, collection)
checks.process_all_files()
# Check Number of check results
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.record_check_table])
result = connection.execute(s)
assert 1 == result.rowcount
data = result.fetchone()
assert '1.1' == data.override_schema_version
s = sa.sql.select([self.database.release_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.record_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
# Call Checks Again - that should be fine
checks = Checks(self.database, collection)
checks.process_all_files()
# Check Number of check results
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.record_check_table])
result = connection.execute(s)
assert 1 == result.rowcount
data = result.fetchone()
assert '1.1' == data.override_schema_version
s = sa.sql.select([self.database.release_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.record_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
def run_command(self, args):
self.run_command_for_selecting_existing_collection(args)
checks = Checks(self.database, self.collection)
checks.process_all_files()
def test_releases_via_process_file_item_id_method(self):
collection_id = self.database.get_or_create_collection_id(
"test", datetime.datetime.now(), False)
self.database.mark_collection_check_data(collection_id, True)
self.database.mark_collection_check_older_data_with_schema_version_1_1(
collection_id, True)
collection = self.database.get_collection(collection_id)
store = Store(self.config, self.database)
store.set_collection(collection)
json_filename = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'fixtures',
'sample_1_0_release.json')
store.store_file_from_local("test.json", "http://example.com",
"release_package", "utf-8", json_filename)
file_item = self.database.get_all_files_items_in_file(
self.database.get_all_files_in_collection(collection_id)[0])[0]
# Check Number of check results
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.record_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.record_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
# Call Checks
checks = Checks(self.database, collection)
checks.process_file_item_id(file_item.database_id)
# Check Number of check results
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.record_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_table])
result = connection.execute(s)
assert 2 == result.rowcount
s = sa.sql.select([self.database.record_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
# Call Checks Again - that should be fine
checks = Checks(self.database, collection)
checks.process_file_item_id(file_item.database_id)
# Check Number of check results
with self.database.get_engine().begin() as connection:
s = sa.sql.select([self.database.record_check_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_table])
result = connection.execute(s)
assert 2 == result.rowcount
s = sa.sql.select([self.database.record_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount
s = sa.sql.select([self.database.release_check_error_table])
result = connection.execute(s)
assert 0 == result.rowcount