def test_save(self):
"Test database saving (and reload)."
self.dbname1 = dbutil.random_name("test_save_db")
self.filename1 = self.dbname1 + ".sql"
# Create a new database.
dbutil.TESTSERVER.create(self.dbname1, self.SAVE_DIR, self.SAVE_FILE)
# Save the database to disk.
dbutil.TESTSERVER.save(self.dbname1, self.SAVE_DIR, self.filename1, "processing_level")
# Now reload the file as a new database...
self.dbname2 = dbutil.random_name("test_save_db_copy")
dbutil.TESTSERVER.create(self.dbname2, self.SAVE_DIR, self.filename1)
# and check that it exists.
maint_conn = dbutil.TESTSERVER.connect(self.MAINTENANCE_DB, superuser=True)
try:
maint_conn = dbutil.MaintenanceWrapper(maint_conn)
self.assertTrue(
maint_conn.exists(self.dbname2), "Saved and reloaded database " + "does not seem to be there."
)
finally:
maint_conn.close()
def test_random_name(self):
"Test random_name random database name generator."
basename = "Fred"
rnd_name1 = dbutil.random_name(basename)
rnd_name2 = dbutil.random_name(basename)
self.assertRegexpMatches(rnd_name1, r"^Fred_[\d]{9}", "Random name has unexpected format '%s'" % rnd_name1)
self.assertRegexpMatches(rnd_name2, r"^Fred_[\d]{9}", "Random name has unexpected format '%s'" % rnd_name2)
self.assertNotEqual(rnd_name1, rnd_name2, "Random names are equal: '%s'" % rnd_name1)
def test_recreate(self):
"Test ablility to recreate a database on top of itself"
self.dbname1 = dbutil.random_name("test_recreate_db")
# Create a new database.
dbutil.TESTSERVER.create(self.dbname1, self.SAVE_DIR, self.SAVE_FILE)
# Connect to the newly created database, to make sure it is
# there, and to create a pgbouncer pool.
conn = dbutil.TESTSERVER.connect(self.dbname1)
try:
conn = dbutil.ConnectionWrapper(conn)
dbname = conn.database_name()
self.assertEqual(dbname, self.dbname1)
finally:
conn.close()
# Now recreate on top of the existing database...
dbutil.TESTSERVER.create(self.dbname1, self.SAVE_DIR, self.SAVE_FILE)
# and check that it exists.
maint_conn = dbutil.TESTSERVER.connect(self.MAINTENANCE_DB, superuser=True)
try:
maint_conn = dbutil.MaintenanceWrapper(maint_conn)
self.assertTrue(maint_conn.exists(self.dbname1), "Recreated database does not seem to be there.")
finally:
maint_conn.close()
def test_drop(self):
"Test ability to drop a database"
self.dbname1 = dbutil.random_name("test_drop_db")
# Create a new database.
dbutil.TESTSERVER.create(self.dbname1, self.SAVE_DIR, self.SAVE_FILE)
# Connect to the newly created database, to make sure it is
# there, and to create a pgbouncer pool.
conn = dbutil.TESTSERVER.connect(self.dbname1)
try:
conn = dbutil.ConnectionWrapper(conn)
dbname = conn.database_name()
self.assertEqual(dbname, self.dbname1)
finally:
conn.close()
# Now drop the database...
dbutil.TESTSERVER.drop(self.dbname1)
# and make sure it is gone.
maint_conn = dbutil.TESTSERVER.connect(self.MAINTENANCE_DB, superuser=True)
try:
maint_conn = dbutil.MaintenanceWrapper(maint_conn)
self.assertFalse(maint_conn.exists(self.dbname1), "Dropped database still seems to be there.")
finally:
maint_conn.close()
def setUp(self):
self.conn = None
self.dbname = dbutil.random_name("test_connection_wrapper_db")
dbutil.TESTSERVER.create(self.dbname, self.SAVE_DIR, self.TEST_DB_FILE)
self.conn = dbutil.TESTSERVER.connect(self.dbname)
self.conn = dbutil.ConnectionWrapper(self.conn)
def test_expected_directory_4(self):
"Test test expected directory finder/creator, test 4."
dummy_version = dbutil.random_name("version")
expected_path = os.path.join(dbutil.TEST_RESOURCES_ROOT, dummy_version, "expected", "module", "suite")
try:
path = dbutil.expected_directory("module", "suite", version=dummy_version)
self.check_directory(path, expected_path)
finally:
os.removedirs(path)
def test_resources_directory(self):
"Test test resources directory finder/creator."
dummy_user = dbutil.random_name("user")
expected_path = os.path.join(dbutil.TEST_RESOURCES_ROOT, dummy_user, "test", "module", "suite")
try:
path = dbutil.resources_directory(dummy_user, "test", "module", "suite")
self.check_directory(path, expected_path)
finally:
os.removedirs(path)
def test_input_directory(self):
"Test test input directory finder/creator."
dummy_version = dbutil.random_name("version")
input_path = os.path.join(dbutil.TEST_RESOURCES_ROOT, dummy_version, "input", "module", "suite")
try:
path = dbutil.input_directory("module", "suite", version=dummy_version)
self.check_directory(path, input_path)
finally:
os.removedirs(path)
def test_output_directory_1(self):
"Test test output directory finder/creator, test 1."
dummy_user = dbutil.random_name("user")
expected_path = os.path.join(dbutil.TEST_RESOURCES_ROOT, dummy_user, "output", "module", "suite")
try:
path = dbutil.output_directory("module", "suite", user=dummy_user)
self.check_directory(path, expected_path)
finally:
os.removedirs(path)
def setUp(self):
#
# Parse out the name of the test case and use it to name a logfile
#
match = re.search(r'\.([^\.]+)$', self.id())
if match:
name = match.group(1)
else:
name = 'TestIngester'
logfile_name = "%s.log" % name
self.logfile_path = os.path.join(self.OUTPUT_DIR, logfile_name)
self.expected_path = os.path.join(self.EXPECTED_DIR, logfile_name)
if self.POPULATE_EXPECTED:
self.logfile_path = os.path.join(self.EXPECTED_DIR, logfile_name)
#
# Set up a handler to log to the logfile, and attach it to the
# root logger.
#
#logging.basicConfig()
self.handler = logging.FileHandler(self.logfile_path, mode='w')
self.handler.setLevel(logging.INFO)
self.handler.setFormatter(logging.Formatter('%(message)s'))
LOGGER.addHandler(self.handler)
# Add a streamhandler to write output to console
self.stream_handler = logging.StreamHandler(stream=sys.stdout)
self.stream_handler.setLevel(logging.INFO)
self.stream_handler.setFormatter(logging.Formatter('%(message)s'))
LOGGER.addHandler(self.stream_handler)
# Create an empty database
self.test_conn = None
self.test_dbname = dbutil.random_name("test_tile_record")
LOGGER.info('Creating %s', self.test_dbname)
dbutil.TESTSERVER.create(self.test_dbname,
self.INPUT_DIR, "hypercube_empty.sql")
# Set the datacube configuration file to point to the empty database
configuration_dict = {'dbname': self.test_dbname,
'temp_dir': self.TEMP_DIR,
'tile_root': self.TILE_ROOT_DIR}
config_file_path = dbutil.update_config_file2(configuration_dict,
self.INPUT_DIR,
self.OUTPUT_DIR,
"test_datacube.conf")
# Set an instance of the datacube and pass it to an ingester instance
test_args = TestArgs()
test_args.config_file = config_file_path
test_args.debug = False
test_datacube = IngesterDataCube(test_args)
self.ingester = TestIngester(datacube=test_datacube)
self.collection = self.ingester.collection
def test_copy_table_between_databases(self):
"Test copy of a table from one database to another database."
self.dbname1 = dbutil.random_name("test_copy_db")
self.dbname2 = dbutil.random_name("test_copy_db")
self.filename1 = self.dbname1 + ".sql"
# Create the databases.
dbutil.TESTSERVER.create(self.dbname1, self.SAVE_DIR, self.SAVE_FILE)
dbutil.TESTSERVER.create(self.dbname2, self.SAVE_DIR, self.SAVE_FILE)
# Connect to each database
conn1 = dbutil.TESTSERVER.connect(self.dbname1, superuser=True)
conn2 = dbutil.TESTSERVER.connect(self.dbname2, superuser=True)
conn1 = dbcompare.ComparisonWrapper(conn1)
conn2 = dbcompare.ComparisonWrapper(conn2)
# Create a dummy table in Database 1
table_name = "some_dummy_table_name"
sql = "CREATE TABLE " + table_name + " AS " + "\n" + "SELECT * FROM tile_type;"
with conn1.cursor() as cur:
cur.execute(sql)
# Verify that the table exists in Database 1.
exists = conn1.table_exists(table_name)
if not exists:
self.fail("Table " + table_name + " should exist on Database 1")
# Verify that the table does not exist in Database 2.
exists = conn2.table_exists(table_name)
if exists:
self.fail("Table " + table_name + " should not exist in Database 2")
# Copy the table from Database 1 to Database 2
dbutil.TESTSERVER.copy_table_between_databases(self.dbname1, self.dbname2, table_name)
# Verify that the table does exist in Database 2.
exists = conn2.table_exists(table_name)
if not exists:
self.fail("Table " + table_name + " should exist")
def test_exists(self):
"Test database existance check."
self.assertTrue(
dbutil.TESTSERVER.exists(self.MAINTENANCE_DB),
"Unable to verify existance of the " + "maintenance database '%s'." % self.MAINTENANCE_DB,
)
dummy_dbname = dbutil.random_name("dummy")
self.assertFalse(
dbutil.TESTSERVER.exists(dummy_dbname), "Dummy database '%s' reported as existing." % dummy_dbname
)
def xxxtest_create_database(self):
"Test random_name random database name generator."
self.dbname1 = 'hypercube_v0'
self.dbname2 = dbutil.random_name('test_create_database')
LOGGER.info('Creating database %s', self.dbname2)
dbutil.TESTSERVER.create(self.dbname2, self.INPUT_DIR,
'hypercube_test_ingest.sql')
self.conn1 = dbutil.TESTSERVER.connect(self.dbname1)
self.conn2 = dbutil.TESTSERVER.connect(self.dbname2)
LOGGER.info('About to create database from file')
dbutil.TESTSERVER.create(self.dbname, self.INPUT_DIR,
'hypercube_test_ingest.sql')
LOGGER.info('.done')
def test_output_directory_2(self):
"Test test output directory finder/creator, test 2."
dummy_user = dbutil.random_name("user")
expected_path = os.path.join(dbutil.TEST_RESOURCES_ROOT, dummy_user, "output", "module", "suite")
old_user = os.environ["USER"]
try:
os.environ["USER"] = dummy_user
path = dbutil.output_directory("module", "suite")
self.check_directory(path, expected_path)
finally:
os.environ["USER"] = old_user
os.removedirs(path)
def test_create_from_template(self):
"Test database creation from template"
self.dbname1 = dbutil.random_name("test_create_from_template_db")
# Create a new database.
dbutil.TESTSERVER.create(self.dbname1, template_db=self.TEST_TEMPLATE_DB)
# Check if the newly created database exists.
maint_conn = dbutil.TESTSERVER.connect(self.MAINTENANCE_DB, superuser=True)
try:
maint_conn = dbutil.MaintenanceWrapper(maint_conn)
self.assertTrue(maint_conn.exists(self.dbname1), "New database does not seem to be there.")
finally:
maint_conn.close()
def test_expected_directory_5(self):
"Test test expected directory finder/creator, test 5."
dummy_version = dbutil.random_name("version")
old_version = os.environ.get("DATACUBE_VERSION", None)
expected_path = os.path.join(dbutil.TEST_RESOURCES_ROOT, dummy_version, "expected", "module", "suite")
try:
os.environ["DATACUBE_VERSION"] = dummy_version
path = dbutil.expected_directory("module", "suite")
self.check_directory(path, expected_path)
finally:
if old_version is None:
del os.environ["DATACUBE_VERSION"]
else:
os.environ["DATACUBE_VERSION"] = old_version
os.removedirs(path)
def __init__(self, test_name, config):
self.test_name = test_name
self.result = ''
self.error_message = None
try:
self.test_dir = self.create_dir('.', test_name)
self.scenes_dir = self.create_dir(self.test_dir, 'scenes')
self.remove_old_links()
if config.has_option(test_name, 'scenes'):
scene_list = self.link_scenes(test_name, config)
else:
scene_list = None
# export the list of scenes as environment variables for use by
# called shell script
for iscene in range(len(scene_list)):
os.environ['SCENE_DIR%d' %iscene] = scene_list[iscene]
os.environ['Nscenes'] = str(len(scene_list))
os.environ['SYSTEST_DIR'] = self.test_dir
self.temp_dir = self.create_dir(self.test_dir, 'temp')
self.tile_dir = self.create_dir(self.test_dir, 'tiles')
self.dbname = dbutil.random_name(test_name)
self.load_initial_database(test_name, config)
self.make_datacube_config(test_name, config)
self.command = None
if config.has_option(test_name, 'command'):
self.command = config.get(test_name, 'command')
os.environ['DATACUBE_ROOT'] = \
config.get(test_name, 'datacube_root')
self.logfile = self.open_logfile(test_name)
except AssertionError as e:
self.result = 'ERROR'
self.error_message = e.message
def setUp(self):
#
# Parse out the name of the test case and use it to name a logfile
#
match = re.search(r'\.([^\.]+)$', self.id())
if match:
name = match.group(1)
else:
name = 'TestIngester'
logfile_name = "%s.log" % name
self.logfile_path = os.path.join(self.OUTPUT_DIR, logfile_name)
self.expected_path = os.path.join(self.EXPECTED_DIR, logfile_name)
#
# Set up a handler to log to the logfile, and attach it to the
# root logger.
#
self.handler = logging.FileHandler(self.logfile_path, mode='w')
self.handler.setLevel(logging.INFO)
self.handler.setFormatter(logging.Formatter('%(message)s'))
# Create an empty database
self.test_conn = None
print 'Create an empty database'
self.test_dbname = dbutil.random_name("test_dataset_record")
print 'Creating %s' %self.test_dbname
dbutil.TESTSERVER.create(self.test_dbname,
self.INPUT_DIR, "hypercube_empty.sql")
# Set the datacube configuration file to point to the empty database
configuration_dict = {'dbname': self.test_dbname}
config_file_path = dbutil.update_config_file2(configuration_dict,
self.INPUT_DIR,
self.OUTPUT_DIR,
"test_datacube.conf")
# Set an instance of the datacube and pass it to an ingester instance
test_args = TestArgs()
test_args.config_file = config_file_path
test_args.debug = False
test_datacube = IngesterDataCube(test_args)
self.ingester = TestIngester(datacube=test_datacube)
self.collection = self.ingester.collection
def test_compare_tile_stores(self):
"Test creation of tile_acquisition_info table."""
self.dbname1 = 'hypercube_test2_v0'
self.dbname2 = dbutil.random_name('test_tilecompare')
LOGGER.info('Creating database %s', self.dbname2)
dbutil.TESTSERVER.create(self.dbname2, self.INPUT_DIR,
'hypercube_test_ingest.sql')
#Temp
#print 'Loading production database %s' %self.dbname1
#dbutil.TESTSERVER.create(self.dbname1, '/g/data/v10/test_resources/databases',
# 'hypercube_v0.sql')
#self.dbname1 = 'hypercube_test2_v0'
#print 'Loading production database %s' %self.dbname1
#dbutil.TESTSERVER.create(self.dbname1, '/g/data/v10/test_resources/databases',
# 'hypercube_v0.sql')
#return
#Temp
self.conn1 = dbutil.TESTSERVER.connect(self.dbname1, autocommit=False)
self.conn2 = dbutil.TESTSERVER.connect(self.dbname2, autocommit=False)
LOGGER.info('About to call compare_tile_stores')
fout = open(os.path.join(self.OUTPUT_DIR,
'tile_comparison_output.txt'), 'w')
fout = sys.stdout #temp
difference_pairs = tilecompare.compare_tile_stores(self.conn1,
self.conn2,
output=fout)
LOGGER.info('Finished calling compare_tile_stores')
if difference_pairs != []:
report_string = "Fresh ingest tile content differs from the " \
"benchmark:\n"
for pair in difference_pairs:
report_string = report_string + "Benchmark tile:\n%s\nFresh" \
"Ingest tile:\n%s\n" %(pair[0], pair[1])
self.fail(report_string)
else:
print 'Everything passed'
def xxxtest_create_tile_acqusition_info(self):
"Test creation of tile_acquisition_info table."""
self.dbname1 = 'hypercube_test'
self.dbname2 = dbutil.random_name('test_tilecompare')
LOGGER.info('Creating database %s', self.dbname2)
dbutil.TESTSERVER.create(self.dbname2, self.INPUT_DIR,
'hypercube_test_ingest.sql')
self.conn1 = dbutil.TESTSERVER.connect(self.dbname1, autocommit=False)
self.conn2 = dbutil.TESTSERVER.connect(self.dbname2, autocommit=False)
LOGGER.info('About to create comparision pair')
pair = tilecompare.TileComparisonPair(self.conn1, self.conn2,
'public', 'public')
LOGGER.info('About to create table from fresh ingest')
fresh_ingest_info_table = 'fresh_ingest_info'
comparison_table = 'ingest_comparison'
tilecompare._copy_ingest_tile_acquisition_info(pair,
fresh_ingest_info_table)
LOGGER.info('About to create comparison table')
tilecompare._create_comparison_table(pair, fresh_ingest_info_table,
comparison_table)
LOGGER.info('About to compare the tile contents')
tilecompare._compare_tile_contents(pair, comparison_table)
def test_landsat_tiler(self):
"""Test the cataloging and tiling of Landsat scences and compare
resulting database and tile contents with an ingestion benchmark"""
# This test is intended as an example, and so is extensively
# commented.
# Open a log file
if self.mode not in [0, 1, 2, 3]:
self.skipTest('Skipping test_landsat_tiler since flag is not in [0, 1, 2, 3]')
logfile_path = os.path.join(self.OUTPUT_DIR, "test_landsat_tiler.log")
self.logfile = open(logfile_path, "w")
#
# Create the initial database
#
# Randomise the name to avoid collisions with other users.
self.test_dbname = dbutil.random_name("test_tiler")
# Create the database.
print 'About to create dbase from %s' \
%(os.path.join(self.INPUT_DIR, "hypercube_empty.sql"))
if self.mode != 1:
dbutil.TESTSERVER.create(self.test_dbname,
self.INPUT_DIR, "hypercube_empty.sql")
#
# Run dbupdater on the test database and save the result
#
# Create updated datacube_conf file with the new dbname and tile_root
tile_root = os.path.join(self.OUTPUT_DIR, "tiles")
configuration_dict = {'dbname': self.test_dbname,
'tile_root': tile_root}
config_file_path = dbutil.update_config_file2(configuration_dict,
self.INPUT_DIR,
self.OUTPUT_DIR,
"test_datacube.conf")
# Run dbupdater
ingest_dir = os.path.join(self.INPUT_DIR, 'tiler_testing')
dbupdater_cmd = ["python",
"dbupdater.py",
"--debug",
"--config=%s" % config_file_path,
"--source=%s" % ingest_dir,
"--removedblist",
"--followsymlinks"]
if self.mode != 1:
subprocess.check_call(dbupdater_cmd, stdout=self.logfile,
stderr=subprocess.STDOUT)
# Run landsat_tiler
landsat_tiler_cmd = ["python",
"landsat_tiler.py",
"--config=%s" % config_file_path]
if self.mode != 1:
subprocess.check_call(landsat_tiler_cmd, stdout=self.logfile,
stderr=subprocess.STDOUT)
# Save the updated database
if self.mode != 1:
dbutil.TESTSERVER.save(self.test_dbname, self.OUTPUT_DIR,
"tiler_testing.sql")
#
# If an expected result exists then load it and compare
#
# Check for expected result
if self.mode > 0 and os.path.isfile(os.path.join(self.EXPECTED_DIR,
"tiler_testing.sql")):
print 'starting to check differences'
#MPHtemp create the output database
if self.mode == 1:
self.test_dbname = dbutil.random_name("tiler_testing")
dbutil.TESTSERVER.create(self.test_dbname,
self.OUTPUT_DIR, "tiler_testing.sql")
#END MPHtemp
# Create a randomised name...
self.expected_dbname = dbutil.random_name("expected_tiler_testing")
# load the database...
dbutil.TESTSERVER.create(self.expected_dbname,
self.EXPECTED_DIR, "tiler_testing.sql")
# create database connections...
self.test_conn = dbutil.TESTSERVER.connect(self.test_dbname)
self.expected_conn = \
dbutil.TESTSERVER.connect(self.expected_dbname)
# and compare.
dbases_agree = dbcompare.compare_databases(self.test_conn,
self.expected_conn,
output=self.logfile,
verbosity=3)
if self.mode == 2:
#Compare databases and fail test if they differ
assert dbases_agree, "Databases do not match."
#Compare data within corresponding files of the EXPECTED_DIR and
#OUTPUT_DIR. Get list of tile pathnames from EXPECTED and OUTPUT
#databases' repsective tile tables. There is an assumption here
#that, within each of the expected and output databases,
#the tile basename uniquely defines both the tile_type_id and the
#full tile pathname. However, if the tile type table has put ORTHO
#bands to be of a new tile_type, then
#corresponding tiles in expected and output may be of different
#tile_type. So we need to have self.bands_expected and
#self.bands_output
expected_tile_dict, output_tile_dict = \
self.get_tile_pathnames(self.expected_conn, self.test_conn)
tiles_expected = set(expected_tile_dict.keys())
tiles_output = set(output_tile_dict.keys())
tiles_expected_or_output = tiles_expected | tiles_output
#Construct band source table as per datacube module
self.bands_expected = \
self.construct_bands_source_dict(self.expected_conn)
self.bands_output =\
self.construct_bands_source_dict(self.test_conn)
#file_pattern to parse file name for information
file_pattern = [r'(?P<sat>\w+)_(?P<sensor>\w+)_',
r'(?P<processing_level>\w+)_',
r'(?P<xindex>-*\d+)_(?P<yindex>-*\d+)_'
r'(?P<year>\d+)-(?P<month>\d+)-'
r'(?P<day>\d+)T(?P<hour>\d+)-(?P<minute>\d+)-',
r'(?P<second_whole>\d+)\.(?P<second_fraction>\d+)'
r'\.(?P<file_extension>.+)']
pattern = re.compile(''.join(file_pattern))
#Set up dictionary of pixel counts to be accumulated per
#(procesing_level, tile_layer) over all tiles
#0: total_pixel_count_expected
#1: total_pixel_count_output
#2: total_pixel_count_both
#3: total_pixel_count_expected_not_output
#4: total_pixel_count_output_not_expected
pixel_count_dict = {}
#Set up nested dicts of differece counts
difference_count_dict = {}
#For each tile in EXPECTED_DIR and OUTPUT_DIR, get pixel counts and
#difference histograms
#There are five dictionaries involved:
### tile_name_dict {'sat': LS5, 'sensor': TM, ...}
### bands_dict_expected: those bands from self.bands_expected
### corresponding to current tile's
### tile_type_id and (satellite, sensor)
### bands_dict_output: output database's correspondent to
### bands_dict_expected
### level_dict_expected: those bands from bands_dict_expected
### for which the processing level
### matches that for the current tile
### level_dict_output: output database's correspondent to
### level_dict_expected
### all_levels_info_dict [level_dict_expected,
#### level_dict_output]
### for each processing level
all_levels_info_dict = {}
for tile_name in tiles_expected_or_output:
print 'processing tile %s' %tile_name
tile_type_id_expected = None
tile_type_id_output = None
fname_expected = None
fname_output = None
#If tile is in either database, extract tile_type and pathname
if tile_name in tiles_expected:
tile_type_id_expected, fname_expected = \
expected_tile_dict[tile_name]
if tile_name in tiles_output:
tile_type_id_output, fname_output = \
output_tile_dict[tile_name]
#Extract information from the tile name and select
#nested dictionary for this tile from bands table,
#given the (sat, sensor) [or("DERIVED', 'PQA') for PQA],
#which will be common to expected and output tiles, and the
#tile_type_id, which may be different for expected and output
matchobj = re.match(pattern, tile_name)
tile_name_dict = matchobj.groupdict()
full_key_expected = \
self.get_tiletype_sat_sens_level(tile_type_id_expected,
tile_name_dict)
full_key_output = \
self.get_tiletype_sat_sens_level(tile_type_id_output,
tile_name_dict)
#Following will raise assertion error if a tile's
#tile_type_id has changed since benchmark ingestion
full_key = self.check_equal_or_null(full_key_expected,
full_key_output)
level_dict_expected = {}
level_dict_output = {}
#full_key is (tile_type, sat, sensor, processing_level)
if full_key in all_levels_info_dict:
(level_dict_expected, level_dict_output) = \
all_levels_info_dict[full_key]
if level_dict_expected == {} and full_key_expected != None:
level_dict_expected = \
self.collect_source_bands(self.bands_expected,
full_key)
if level_dict_output == {} and full_key_output != None:
level_dict_output = \
self.collect_source_bands(self.bands_output,
full_key)
if full_key not in all_levels_info_dict:
all_levels_info_dict[full_key] = [level_dict_expected,
level_dict_output]
if all_levels_info_dict[full_key][0] == {} and \
level_dict_expected != {}:
all_levels_info_dict[full_key][0] = level_dict_expected
if all_levels_info_dict[full_key][1] == {} and \
level_dict_output != {}:
all_levels_info_dict[full_key][1] = level_dict_output
#Check that the number of bands is as expected, adding
#singleton dimension if only one band
([data_expected, data_output], number_layers) = \
self.load_and_check(fname_expected, fname_output,
level_dict_expected,
level_dict_output)
assert bool(fname_expected) == (data_expected != None) and \
bool(fname_output) == (data_output != None), \
"data array should exist if and only if fname exists"
for ilayer in range(number_layers):
#Define expected and output band data
band_expected, dtype_expected = \
self.get_band_data(data_expected, ilayer)
band_output, dtype_output = \
self.get_band_data(data_output, ilayer)
assert (band_expected == None) == (dtype_expected == None)\
and (band_output == None) == (dtype_output == None), \
"band data should exist if and only if dtype exists"
dtype_this = self.check_equal_or_null(dtype_expected,
dtype_output)
#calculate the number of bins required to store the
#histogram of differences from this datatype
if tile_name_dict['processing_level'] == 'PQA':
#possible difference values are 0 through 16,
#(number of tests which differ)
bin_count = 16 + 1
else:
#possible difference vals are min through max of dtype
bin_count = numpy.iinfo(dtype_this).max - \
numpy.iinfo(dtype_this).min + 1
assert bin_count < 66000, "datatype is more than 16" \
"bits, need to add code to coarsen the" \
"histogram bins or use apriori max and" \
"min values of the data"
#The histograms are per (level, layer).
#Could have one histogram per (sat, sensor, level, layer)
#and then, depending on verbosity, aggregate during report.
#But for now, just key by (level, layer).
result_key = (full_key[3], ilayer + 1)
if result_key not in pixel_count_dict:
pixel_count_dict[result_key] = numpy.zeros(shape=(5),
dtype=numpy.uint64)
difference_count_dict[result_key] = \
numpy.zeros(shape=(bin_count), dtype=numpy.uint64)
pixel_count = pixel_count_dict[result_key]
difference_count = difference_count_dict[result_key]
if tile_name_dict['processing_level'] == 'PQA':
if band_expected is None:
band_expected = 0
if band_output is None:
band_output = 0
#define index as those pixels with contiguity bit set
index_expected = \
numpy.bitwise_and(band_expected,
1 << self.PQA_CONTIGUITY_BIT) > 0
index_output = \
numpy.bitwise_and(band_output,
1 << self.PQA_CONTIGUITY_BIT) > 0
else:
#For NBAR and ORTHO use nodata_value
nodata_value = \
level_dict_output[ilayer + 1]['nodata_value']
if band_expected is None:
band_expected = nodata_value
if band_output is None:
band_output = nodata_value
index_expected = band_expected != nodata_value
index_output = band_output != nodata_value
pixel_count[0] += numpy.count_nonzero(index_expected)
pixel_count[1] += numpy.count_nonzero(index_output)
pixel_count[2] += \
numpy.count_nonzero(numpy.logical_and(index_expected,
index_output))
pixel_count[3] += \
numpy.count_nonzero(numpy.logical_and
(index_expected, ~index_output))
pixel_count[4] += \
numpy.count_nonzero(numpy.logical_and
(~index_expected, index_output))
#Only want to calculate differences at common pixels
index_both = numpy.logical_and(index_expected,
index_output)
if numpy.count_nonzero(index_both) == 0:
continue
valid_data_expected = band_expected[index_both].ravel()
valid_data_output = band_output[index_both].ravel()
#Calculate difference histogram and add to running total
if tile_name_dict['processing_level'] == 'PQA':
difference = \
self.count_bitwise_diffs(valid_data_expected,
valid_data_output)
else:
difference = abs(valid_data_output.astype(numpy.int64)
- valid_data_expected.astype(numpy.int64))
hist, dummy_bin_edges = \
numpy.histogram(difference,
numpy.array(range(bin_count + 1),
dtype=numpy.uint64))
difference_count += hist
#dereference band data
band_expected = None
band_output = None
difference = None
#end of layer loop
#dereference tile data
data_expected = None
data_output = None
#Output
#for sat_sen, band_dict in all_bands_dict:
fp = open(os.path.join(self.OUTPUT_DIR,
'Histogram_output.txt'), 'w')
fp.writelines('##### COMPARISON OF TILED DATA IN FOLLOWING '\
'DIRECTORES\n%s\n%s\n' %(self.EXPECTED_DIR,
self.OUTPUT_DIR))
result_keys_processed = []
for full_key in all_levels_info_dict.keys():
dummy, dummy, dummy, processing_level = full_key
top_layer_result_key = (processing_level, 1)
if top_layer_result_key in result_keys_processed:
continue
fp.writelines('#### Processing Level: %s\n' %processing_level)
level_dict_expected, level_dict_output = \
all_levels_info_dict[full_key]
assert set(level_dict_expected.keys()) == \
set(level_dict_output.keys()), "different key sets"
number_layers = len(level_dict_output.keys())
for this_layer in range(1, number_layers + 1):
result_key = (processing_level, this_layer)
result_keys_processed.append(result_key)
fp.writelines('### tile_layer = %d\n' %this_layer)
for key, val in level_dict_expected[this_layer].items():
if key == 'tile_layer' or key == 'level_name':
continue
outline = '# %s = %s' %(key, val)
if str(level_dict_output[this_layer][key]) != str(val):
outline = '%s (%s in output database)' \
%(outline, level_dict_output[this_layer][key])
fp.writelines('%s\n' %outline)
#get key for pixel_count_dict and difference_count_dict
#Print counts of pixels with valid data
fp.writelines('#Valid data counts\n')
pixel_count = pixel_count_dict[result_key]
count_desc = ['Expected\t', 'Output\t\t', 'Common\t\t',
'Missing\t\t', 'Extra\t\t']
for desc, num in zip(count_desc, pixel_count):
fp.writelines('\t\t%s%d\n' %(desc, num))
#Print histograms of differences in valid data
fp.writelines('#Histogram of differences in valid data\n')
difference_count = difference_count_dict[result_key]
index_nonzero_bins = difference_count > 0
for bin_no in range(len(difference_count)):
if index_nonzero_bins[bin_no]:
fp.writelines('\t\tDifference of %d: %d\n'
%(bin_no, difference_count[bin_no]))
fp.close()
else:
if self.mode > 0:
self.skipTest("Expected database save file not found.")
def test_onescene(self):
"""Test database update for a single scene."""
# This test is intended as an example, and so is extensively
# commented.
# Open a log file
logfile_path = os.path.join(self.OUTPUT_DIR, "test_onescene.log")
self.logfile = open(logfile_path, "w")
#
# Create the initial database
#
# Randomise the name to avoid collisons with other users.
self.test_dbname = dbutil.random_name("test_onescene")
# Create the database.
dbutil.TESTSERVER.create(self.test_dbname, self.INPUT_DIR, "hypercube_empty.sql")
#
# Run dbupdater on the test database and save the result
#
# Create an updated datacube_conf file with the new dbname
config_file_path = dbutil.update_config_file(
self.test_dbname, self.INPUT_DIR, self.OUTPUT_DIR, "test_datacube.conf"
)
# Run dbupdater
ingest_dir = os.path.join(self.INPUT_DIR, "onescene")
dbupdater_cmd = [
"python",
"dbupdater.py",
"--debug",
"--config=%s" % config_file_path,
"--source=%s" % ingest_dir,
"--removedblist",
"--followsymlinks",
]
subprocess.check_call(dbupdater_cmd, stdout=self.logfile, stderr=subprocess.STDOUT)
# Save the updated database
dbutil.TESTSERVER.save(self.test_dbname, self.OUTPUT_DIR, "onescene.sql")
#
# If an expected result exists then load it and compare
#
# Check for expected result
if os.path.isfile(os.path.join(self.EXPECTED_DIR, "onescene.sql")):
# Create a randomised name...
self.expected_dbname = dbutil.random_name("expected_onescene")
# load the database...
dbutil.TESTSERVER.create(self.expected_dbname, self.EXPECTED_DIR, "onescene.sql")
# create database connections...
self.test_conn = dbutil.TESTSERVER.connect(self.test_dbname)
self.expected_conn = dbutil.TESTSERVER.connect(self.expected_dbname)
# and compare.
self.assertTrue(
dbcompare.compare_databases(self.test_conn, self.expected_conn, output=self.logfile, verbosity=3),
"Databases do not match.",
)
else:
self.skipTest("Expected database save file not found.")
