def test_continuous_period_detection(self):
if DEBUG:
print '\nTESTING CONTINUOUS PERIOD DETECTION...'
layer_config = os.path.join(self.testfiles_path, 'conf/test_period_detection.xml')
# Pick the start time for the dates that will be generated
start_datetime = datetime.datetime(2014, 6, 1)
# Set the various period units and lengths we'll be testing
test_periods = (('days', 1), ('days', 5), ('months', 1), ('years', 1))
config = get_layer_config(layer_config, self.archive_config)
for period_unit, period_length in test_periods:
# We test detection with both year and non-year directory setups
for year_dir in (True, False):
make_dir_tree(config['wmts_gc_path'])
make_dir_tree(config['twms_gc_path'])
make_dir_tree(config['wmts_staging_location'])
make_dir_tree(config['twms_staging_location'])
# Generate the empty test files
test_dates = testutils.create_continuous_period_test_files(
config['archive_location'], period_unit, period_length, 5, start_datetime, prefix=config['prefix'], make_year_dirs=year_dir)
# Run layer config command for daily test days
cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format(self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config)
run_command(cmd, ignore_warnings=True)
# Check to see if proper period in GetCapabilities
wmts_gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml')
twms_gc_file = os.path.join(config['twms_gc_path'], 'getCapabilities.xml')
# Build GC search string
search_string = "<Value>" + test_dates[0].date().isoformat() + "/" + test_dates[-1].date().isoformat() + "/P{0}{1}</Value>".format(period_length, period_unit[0].upper())
# Create debug output message
if DEBUG:
print 'Testing with {0} {1} periods'.format(period_length, period_unit)
print 'Creating dates: '
for date in test_dates:
print date.isoformat()
print '\n' + 'Searching for string in GetCapabilities: ' + search_string
# Check to see if string exists in the GC files
wmts_error = "{0} {1} continuous period detection failed -- not found in WMTS GetCapabilities".format(period_length, period_unit)
test_result = find_string(wmts_gc_file, search_string)
# twms_error = "{0} {1} period detection failed -- not found in TMWS GetCapabilities".format(period_length, period_unit)
# self.assertTrue(find_string(twms_gc_file, search_string), twms_error)
# Cleanup -- make sure to get rid of staging files
rmtree(config['wmts_gc_path'])
rmtree(config['wmts_staging_location'])
rmtree(config['twms_staging_location'])
rmtree(config['archive_location'])
# Check result
self.assertTrue(test_result, wmts_error)
def test_legacy_subdaily_continuous(self):
"""
Checks that layer config tool is correctly detecting the period and interval
of subdaily layers that have the datetime in their filenames as opposed to
the z-index.
"""
if DEBUG:
print '\nTESTING LEGACY SUBDAILY CONTINUOUS PERIOD DETECTION...'
layer_config = os.path.join(self.testfiles_path,
'conf/test_subdaily_detect.xml')
start_datetime = datetime.datetime(2014, 6, 1, 12)
config = get_layer_config(layer_config, self.archive_config)
# Set the various period units and lengths we'll be testing
test_periods = (('hours', 1), ('hours', 5), ('minutes', 1),
('minutes', 5), ('seconds', 1), ('seconds', 5))
# Test continuous periods
for period_unit, period_length in test_periods:
for year_dir in (True, False):
# Make the GC dirs
make_dir_tree(config['wmts_gc_path'])
make_dir_tree(config['twms_gc_path'])
make_dir_tree(config['wmts_staging_location'])
make_dir_tree(config['twms_staging_location'])
# Create a continuous set of period files for each time interval
test_dates = testutils.create_continuous_period_test_files(
config['archive_location'],
period_unit,
period_length,
4,
start_datetime,
prefix=config['prefix'],
make_year_dirs=year_dir)
# Create debug output message
if DEBUG:
print '\nTesting with {0} {1} periods'.format(
period_length, period_unit)
print 'Creating legacy subdaily files with dates: '
for date in test_dates:
print date.isoformat()
# Run layer config command
cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format(
self.testfiles_path, self.archive_config, layer_config,
self.projection_config, self.tilematrixset_config)
run_command(cmd, ignore_warnings=True)
# Check to see if proper period in GetCapabilities
wmts_gc_file = os.path.join(config['wmts_gc_path'],
'getCapabilities.xml')
twms_gc_file = os.path.join(config['twms_gc_path'],
'getCapabilities.xml')
# Build search strings
time_string = testutils.get_time_string(
start_datetime, test_dates[-1], config)
search_string = "<Value>" + time_string + "/PT{0}{1}</Value>".format(
period_length, period_unit[0].upper())
search_result = find_string(wmts_gc_file, search_string)
if DEBUG:
print '\n' + 'Searching for string in GetCapabilities: ' + search_string
# Cleanup
rmtree(config['wmts_gc_path'])
rmtree(config['wmts_staging_location'])
rmtree(config['twms_staging_location'])
rmtree(config['archive_location'])
# Check result
error = "{0} {1} continuous subdaily legacy period detection failed -- not found in WMTS GetCapabilities".format(
period_length, period_unit)
self.assertTrue(search_result, error)
def test_continuous_zlevel_period_detection(self):
"""
Checks that the start and end periods of a z-level file are being correctly detected.
"""
if DEBUG:
print '\nTESTING CONTINUOUS Z-LEVEL PERIOD DETECTION...'
layer_config = os.path.join(self.testfiles_path,
'conf/test_zindex_detect.xml')
test_periods = (('minutes', 1), ('minutes', 5), ('hours', 1))
start_datetime = datetime.datetime(2014, 6, 1, 12, 0, 0)
start_datestring = str(start_datetime.year) + str(
start_datetime.timetuple().tm_yday).zfill(3)
config = get_layer_config(layer_config, self.archive_config)
for period_unit, period_length in test_periods:
# Make archive location dir
archive_location = os.path.join(config['archive_location'],
str(start_datetime.year))
make_dir_tree(archive_location)
# Make temp GC and archive directories and dummy MRF
make_dir_tree(config['wmts_gc_path'])
make_dir_tree(config['twms_gc_path'])
make_dir_tree(config['wmts_staging_location'])
make_dir_tree(config['twms_staging_location'])
dummy_mrf = os.path.join(
archive_location,
config['prefix'] + start_datestring + '_.mrf')
open(dummy_mrf, 'a').close()
# Create a ZDB file for seeding with the dates we're looking for
zdb_path = os.path.join(
archive_location,
config['prefix'] + start_datestring + '_.zdb')
conn = sqlite3.connect(zdb_path)
# Create ZINDEX table, generate test dates, and populate ZDB file
conn.execute(
'CREATE TABLE ZINDEX(z INTEGER PRIMARY KEY AUTOINCREMENT, key_str TEXT);'
)
test_dates = testutils.create_continuous_period_test_files(
config['archive_location'],
period_unit,
period_length,
5,
start_datetime,
prefix=config['prefix'],
no_files=True)
# Create debug output message
if DEBUG:
print 'Testing with {0} {1} periods'.format(
period_length, period_unit)
print 'Creating ZDB with dates: '
for date in test_dates:
print date.isoformat()
# Populate the dates in the ZDB
for i, date in enumerate(test_dates):
z_key = date.strftime('%Y%m%d%H%M%S')
sql = 'INSERT INTO ZINDEX(z, key_str) VALUES ({0}, {1})'.format(
i, z_key)
conn.execute(sql)
conn.commit()
# Close ZDB and run layer config
conn.close()
cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format(
self.testfiles_path, self.archive_config, layer_config,
self.projection_config, self.tilematrixset_config)
run_command(cmd, ignore_warnings=True)
# Check to see if proper period in GetCapabilities
wmts_gc_file = os.path.join(config['wmts_gc_path'],
'getCapabilities.xml')
twms_gc_file = os.path.join(config['twms_gc_path'],
'getCapabilities.xml')
# Build GC search string
search_string = "<Value>" + test_dates[0].isoformat(
) + "Z/" + test_dates[-1].isoformat() + "Z</Value>"
if DEBUG:
print '\n' + 'Searching for string in GetCapabilities: ' + search_string
# Check to see if string exists in the GC files
error = "{0} {1} continuous z-level period detection failed -- not found in WMTS GetCapabilities".format(
period_length, period_unit)
check_result = find_string(wmts_gc_file, search_string)
# Cleanup
conn.close()
rmtree(config['wmts_gc_path'])
rmtree(config['wmts_staging_location'])
rmtree(config['twms_staging_location'])
rmtree(config['archive_location'])
# Check result
self.assertTrue(check_result, error)
def test_continuous_period_detection(self):
if DEBUG:
print '\nTESTING CONTINUOUS PERIOD DETECTION...'
layer_config = os.path.join(self.testfiles_path,
'conf/test_period_detection.xml')
# Pick the start time for the dates that will be generated
start_datetime = datetime.datetime(2014, 6, 1)
# Set the various period units and lengths we'll be testing
test_periods = (('days', 1), ('days', 5), ('months', 1), ('years', 1))
config = get_layer_config(layer_config, self.archive_config)
for period_unit, period_length in test_periods:
# We test detection with both year and non-year directory setups
for year_dir in (True, False):
make_dir_tree(config['wmts_gc_path'])
make_dir_tree(config['twms_gc_path'])
make_dir_tree(config['wmts_staging_location'])
make_dir_tree(config['twms_staging_location'])
# Generate the empty test files
test_dates = testutils.create_continuous_period_test_files(
config['archive_location'],
period_unit,
period_length,
5,
start_datetime,
prefix=config['prefix'],
make_year_dirs=year_dir)
# Run layer config command for daily test days
cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format(
self.testfiles_path, self.archive_config, layer_config,
self.projection_config, self.tilematrixset_config)
run_command(cmd, ignore_warnings=True)
# Check to see if proper period in GetCapabilities
wmts_gc_file = os.path.join(config['wmts_gc_path'],
'getCapabilities.xml')
twms_gc_file = os.path.join(config['twms_gc_path'],
'getCapabilities.xml')
# Build GC search string
search_string = "<Value>" + test_dates[0].date().isoformat(
) + "/" + test_dates[-1].date(
).isoformat() + "/P{0}{1}</Value>".format(
period_length, period_unit[0].upper())
# Create debug output message
if DEBUG:
print 'Testing with {0} {1} periods'.format(
period_length, period_unit)
print 'Creating dates: '
for date in test_dates:
print date.isoformat()
print '\n' + 'Searching for string in GetCapabilities: ' + search_string
# Check to see if string exists in the GC files
wmts_error = "{0} {1} continuous period detection failed -- not found in WMTS GetCapabilities".format(
period_length, period_unit)
test_result = find_string(wmts_gc_file, search_string)
# twms_error = "{0} {1} period detection failed -- not found in TMWS GetCapabilities".format(period_length, period_unit)
# self.assertTrue(find_string(twms_gc_file, search_string), twms_error)
# Cleanup -- make sure to get rid of staging files
rmtree(config['wmts_gc_path'])
rmtree(config['wmts_staging_location'])
rmtree(config['twms_staging_location'])
rmtree(config['archive_location'])
# Check result
self.assertTrue(test_result, wmts_error)
def test_legacy_subdaily_continuous(self):
"""
Checks that layer config tool is correctly detecting the period and interval
of subdaily layers that have the datetime in their filenames as opposed to
the z-index.
"""
if DEBUG:
print "\nTESTING LEGACY SUBDAILY CONTINUOUS PERIOD DETECTION..."
layer_config = os.path.join(self.testfiles_path, "conf/test_subdaily_detect.xml")
start_datetime = datetime.datetime(2014, 6, 1, 12)
config = get_layer_config(layer_config, self.archive_config)
# Set the various period units and lengths we'll be testing
test_periods = (("hours", 1), ("hours", 5), ("minutes", 1), ("minutes", 5), ("seconds", 1), ("seconds", 5))
# Test continuous periods
for period_unit, period_length in test_periods:
for year_dir in (True, False):
# Make the GC dirs
make_dir_tree(config["wmts_gc_path"])
make_dir_tree(config["twms_gc_path"])
make_dir_tree(config["wmts_staging_location"])
make_dir_tree(config["twms_staging_location"])
# Create a continuous set of period files for each time interval
test_dates = testutils.create_continuous_period_test_files(
config["archive_location"],
period_unit,
period_length,
4,
start_datetime,
prefix=config["prefix"],
suffix="_.idx",
make_year_dirs=year_dir,
)
# Create debug output message
if DEBUG:
print "\nTesting with {0} {1} periods".format(period_length, period_unit)
print "Creating legacy subdaily files with dates: "
for date in test_dates:
print date.isoformat()
# Run layer config command
cmd = "oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}".format(
self.testfiles_path,
self.archive_config,
layer_config,
self.projection_config,
self.tilematrixset_config,
)
run_command(cmd, ignore_warnings=True)
# Check to see if proper period in GetCapabilities
wmts_gc_file = os.path.join(config["wmts_gc_path"], "getCapabilities.xml")
twms_gc_file = os.path.join(config["twms_gc_path"], "getCapabilities.xml")
# Build search strings
time_string = testutils.get_time_string(start_datetime, test_dates[-1], config)
search_string = (
"<Value>" + time_string + "/PT{0}{1}</Value>".format(period_length, period_unit[0].upper())
)
search_result = find_string(wmts_gc_file, search_string)
if DEBUG:
print "\n" + "Searching for string in GetCapabilities: " + search_string
# Cleanup
rmtree(config["wmts_gc_path"])
rmtree(config["wmts_staging_location"])
rmtree(config["twms_staging_location"])
rmtree(config["archive_location"])
# Check result
error = "{0} {1} continuous subdaily legacy period detection failed -- not found in WMTS GetCapabilities".format(
period_length, period_unit
)
self.assertTrue(search_result, error)
def test_continuous_zlevel_period_detection(self):
"""
Checks that the start and end periods of a z-level file are being correctly detected.
"""
if DEBUG:
print "\nTESTING CONTINUOUS Z-LEVEL PERIOD DETECTION..."
layer_config = os.path.join(self.testfiles_path, "conf/test_zindex_detect.xml")
test_periods = (("minutes", 1), ("minutes", 5), ("hours", 1))
start_datetime = datetime.datetime(2014, 6, 1, 12, 0, 0)
start_datestring = str(start_datetime.year) + str(start_datetime.timetuple().tm_yday).zfill(3)
config = get_layer_config(layer_config, self.archive_config)
for period_unit, period_length in test_periods:
# Make archive location dir
archive_location = os.path.join(config["archive_location"], str(start_datetime.year))
make_dir_tree(archive_location)
# Make temp GC and archive directories and dummy MRF
make_dir_tree(config["wmts_gc_path"])
make_dir_tree(config["twms_gc_path"])
make_dir_tree(config["wmts_staging_location"])
make_dir_tree(config["twms_staging_location"])
dummy_mrf = os.path.join(archive_location, config["prefix"] + start_datestring + "_.idx")
open(dummy_mrf, "a").close()
# Create a ZDB file for seeding with the dates we're looking for
zdb_path = os.path.join(archive_location, config["prefix"] + start_datestring + "_.zdb")
conn = sqlite3.connect(zdb_path)
# Create ZINDEX table, generate test dates, and populate ZDB file
conn.execute("CREATE TABLE ZINDEX(z INTEGER PRIMARY KEY AUTOINCREMENT, key_str TEXT);")
test_dates = testutils.create_continuous_period_test_files(
config["archive_location"],
period_unit,
period_length,
5,
start_datetime,
prefix=config["prefix"],
no_files=True,
)
# Create debug output message
if DEBUG:
print "Testing with {0} {1} periods".format(period_length, period_unit)
print "Creating ZDB with dates: "
for date in test_dates:
print date.isoformat()
# Populate the dates in the ZDB
for i, date in enumerate(test_dates):
z_key = date.strftime("%Y%m%d%H%M%S")
sql = "INSERT INTO ZINDEX(z, key_str) VALUES ({0}, {1})".format(i, z_key)
conn.execute(sql)
conn.commit()
# Close ZDB and run layer config
conn.close()
cmd = "oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}".format(
self.testfiles_path,
self.archive_config,
layer_config,
self.projection_config,
self.tilematrixset_config,
)
run_command(cmd, ignore_warnings=True)
# Check to see if proper period in GetCapabilities
wmts_gc_file = os.path.join(config["wmts_gc_path"], "getCapabilities.xml")
twms_gc_file = os.path.join(config["twms_gc_path"], "getCapabilities.xml")
# Build GC search string
search_string = "<Value>" + test_dates[0].isoformat() + "Z/" + test_dates[-1].isoformat() + "Z</Value>"
if DEBUG:
print "\n" + "Searching for string in GetCapabilities: " + search_string
# Check to see if string exists in the GC files
error = "{0} {1} continuous z-level period detection failed -- not found in WMTS GetCapabilities".format(
period_length, period_unit
)
check_result = find_string(wmts_gc_file, search_string)
# Cleanup
conn.close()
rmtree(config["wmts_gc_path"])
rmtree(config["wmts_staging_location"])
rmtree(config["twms_staging_location"])
rmtree(config["archive_location"])
# Check result
self.assertTrue(check_result, error)
