def test_versioned_colormaps(self):
# Set locations of the config files we're using for this test
layer_config = os.path.join(self.testfiles_path, 'conf/test_versioned_colormaps.xml')
test_metadata = ('<ows:Metadata xlink:type="simple" xlink:role="http://earthdata.nasa.gov/gibs/metadata-type/colormap" xlink:href="http://localhost/colormaps/2.0/MODIS_Aqua_Aerosol-GIBS_2.0.xml" xlink:title="GIBS Color Map: Data - RGB Mapping"/>',
'<ows:Metadata xlink:type="simple" xlink:role="http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.0" xlink:href="http://localhost/colormaps/1.0//MODIS_Aqua_Aerosol-GIBS_1.0.xml" xlink:title="GIBS Color Map: Data - RGB Mapping"/>'
'<ows:Metadata xlink:type="simple" xlink:role="http://earthdata.nasa.gov/gibs/metadata-type/colormap/2.0" xlink:href="http://localhost/colormaps/2.0//MODIS_Aqua_Aerosol-GIBS_2.0.xml" xlink:title="GIBS Color Map: Data - RGB Mapping"/>')
config = get_layer_config(layer_config, self.archive_config)
# Create colormap locations and copy colormap test file
for location in config['colormap_locations']:
make_dir_tree(location.firstChild.nodeValue)
colormap = next(colormap.firstChild.nodeValue for colormap in config['colormaps'] if colormap.attributes['version'].value == location.attributes['version'].value)
copy(os.path.join(self.testfiles_path, 'conf/' + colormap), location.firstChild.nodeValue)
# Create paths for data and GC
make_dir_tree(config['wmts_gc_path'])
make_dir_tree(config['twms_gc_path'])
make_dir_tree(config['archive_location'])
# Run layer config tool
cmd = 'oe_configure_layer -l {0} --skip_empty_tiles -a {1} -c {2} -p {3} -m {4}'.format(self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config)
run_command(cmd)
# Check to see if all required metadata lines are in the getCapabilities
gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml')
metadata_pass = all(line for line in test_metadata if find_string(gc_file, line))
self.assertTrue(metadata_pass, "Can't find all the proper versioned colormap metadata in the WMTS GetCapabilities file or GC file was not created.")
# Cleanup
[rmtree(path) for path in [path.firstChild.nodeValue for path in config['colormap_locations']]]
rmtree(config['wmts_gc_path'])
rmtree(config['wmts_staging_location'])
rmtree(config['twms_staging_location'])
def test_continuous_period_detection(self):
if DEBUG:
print '\nTESTING CONTINUOUS PERIOD DETECTION...'
layer_config = os.path.join(self.testfiles_path, 'conf/test_period_detection.xml')
# Pick the start time for the dates that will be generated
start_datetime = datetime.datetime(2014, 6, 1)
# Set the various period units and lengths we'll be testing
test_periods = (('days', 1), ('days', 5), ('months', 1), ('years', 1))
config = get_layer_config(layer_config, self.archive_config)
for period_unit, period_length in test_periods:
# We test detection with both year and non-year directory setups
for year_dir in (True, False):
make_dir_tree(config['wmts_gc_path'])
make_dir_tree(config['twms_gc_path'])
make_dir_tree(config['wmts_staging_location'])
make_dir_tree(config['twms_staging_location'])
# Generate the empty test files
test_dates = testutils.create_continuous_period_test_files(
config['archive_location'], period_unit, period_length, 5, start_datetime, prefix=config['prefix'], make_year_dirs=year_dir)
# Run layer config command for daily test days
cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format(self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config)
run_command(cmd, ignore_warnings=True)
# Check to see if proper period in GetCapabilities
wmts_gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml')
twms_gc_file = os.path.join(config['twms_gc_path'], 'getCapabilities.xml')
# Build GC search string
search_string = "<Value>" + test_dates[0].date().isoformat() + "/" + test_dates[-1].date().isoformat() + "/P{0}{1}</Value>".format(period_length, period_unit[0].upper())
# Create debug output message
if DEBUG:
print 'Testing with {0} {1} periods'.format(period_length, period_unit)
print 'Creating dates: '
for date in test_dates:
print date.isoformat()
print '\n' + 'Searching for string in GetCapabilities: ' + search_string
# Check to see if string exists in the GC files
wmts_error = "{0} {1} continuous period detection failed -- not found in WMTS GetCapabilities".format(period_length, period_unit)
test_result = find_string(wmts_gc_file, search_string)
# twms_error = "{0} {1} period detection failed -- not found in TMWS GetCapabilities".format(period_length, period_unit)
# self.assertTrue(find_string(twms_gc_file, search_string), twms_error)
# Cleanup -- make sure to get rid of staging files
rmtree(config['wmts_gc_path'])
rmtree(config['wmts_staging_location'])
rmtree(config['twms_staging_location'])
rmtree(config['archive_location'])
# Check result
self.assertTrue(test_result, wmts_error)
def test_versioned_colormaps(self):
# Set locations of the config files we're using for this test
layer_config = os.path.join(self.testfiles_path,
'conf/test_versioned_colormaps.xml')
test_metadata = (
'<ows:Metadata xlink:type="simple" xlink:role="http://earthdata.nasa.gov/gibs/metadata-type/colormap" xlink:href="http://localhost/colormaps/2.0/MODIS_Aqua_Aerosol-GIBS_2.0.xml" xlink:title="GIBS Color Map: Data - RGB Mapping"/>',
'<ows:Metadata xlink:type="simple" xlink:role="http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.0" xlink:href="http://localhost/colormaps/1.0//MODIS_Aqua_Aerosol-GIBS_1.0.xml" xlink:title="GIBS Color Map: Data - RGB Mapping"/>'
'<ows:Metadata xlink:type="simple" xlink:role="http://earthdata.nasa.gov/gibs/metadata-type/colormap/2.0" xlink:href="http://localhost/colormaps/2.0//MODIS_Aqua_Aerosol-GIBS_2.0.xml" xlink:title="GIBS Color Map: Data - RGB Mapping"/>'
)
config = get_layer_config(layer_config, self.archive_config)
# Create colormap locations and copy colormap test file
for location in config['colormap_locations']:
make_dir_tree(location.firstChild.nodeValue)
colormap = next(colormap.firstChild.nodeValue
for colormap in config['colormaps']
if colormap.attributes['version'].value ==
location.attributes['version'].value)
copy(os.path.join(self.testfiles_path, 'conf/' + colormap),
location.firstChild.nodeValue)
# Create paths for data and GC
make_dir_tree(config['wmts_gc_path'])
make_dir_tree(config['twms_gc_path'])
make_dir_tree(config['archive_location'])
# Run layer config tool
cmd = 'oe_configure_layer -l {0} --skip_empty_tiles -a {1} -c {2} -p {3} -m {4}'.format(
self.testfiles_path, self.archive_config, layer_config,
self.projection_config, self.tilematrixset_config)
run_command(cmd)
# Check to see if all required metadata lines are in the getCapabilities
gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml')
metadata_pass = all(line for line in test_metadata
if find_string(gc_file, line))
self.assertTrue(
metadata_pass,
"Can't find all the proper versioned colormap metadata in the WMTS GetCapabilities file or GC file was not created."
)
# Cleanup
[
rmtree(path) for path in [
path.firstChild.nodeValue
for path in config['colormap_locations']
]
]
rmtree(config['wmts_gc_path'])
rmtree(config['wmts_staging_location'])
rmtree(config['twms_staging_location'])
def test_legacy_subdaily_intermittent(self):
"""
Checks that layer config tool is correctly detecting the period and interval
of subdaily layers that have the datetime in their filenames as opposed to
the z-index.
"""
if DEBUG:
print '\nTESTING LEGACY SUBDAILY INTERMITTENT PERIOD DETECTION...'
layer_config = os.path.join(self.testfiles_path,
'conf/test_subdaily_detect.xml')
start_datetime = datetime.datetime(2014, 6, 1, 12)
config = get_layer_config(layer_config, self.archive_config)
# Set the various period units and lengths we'll be testing
test_periods = (('hours', 1), ('hours', 5), ('minutes', 1),
('minutes', 5), ('seconds', 1), ('seconds', 5))
# Test intermittent periods
for period_unit, period_length in test_periods:
for year_dir in (True, False):
# Make the GC dirs
make_dir_tree(config['wmts_gc_path'])
make_dir_tree(config['twms_gc_path'])
make_dir_tree(config['wmts_staging_location'])
make_dir_tree(config['twms_staging_location'])
test_intervals = testutils.create_intermittent_period_test_files(
config['archive_location'],
period_unit,
period_length,
5,
start_datetime,
prefix=config['prefix'],
make_year_dirs=year_dir)
# Create debug output message (flatten array w/ itertools)
if DEBUG:
print '\nTesting with {0} {1} periods'.format(
period_length, period_unit)
print 'Creating legacy subdaily files with dates: '
for date in itertools.chain.from_iterable(test_intervals):
print date.isoformat()
# Run layer config command
cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format(
self.testfiles_path, self.archive_config, layer_config,
self.projection_config, self.tilematrixset_config)
run_command(cmd, ignore_warnings=True)
# Check to see if proper period in GetCapabilities
wmts_gc_file = os.path.join(config['wmts_gc_path'],
'getCapabilities.xml')
twms_gc_file = os.path.join(config['twms_gc_path'],
'getCapabilities.xml')
search_strings = []
for interval in test_intervals:
time_string = testutils.get_time_string(
interval[0], interval[-1], config)
search_string = "<Value>" + time_string + "/PT{0}{1}</Value>".format(
period_length, period_unit[0].upper())
search_strings.append(search_string)
if DEBUG:
print '\n' + 'Searching for strings in GetCapabilities: '
for string in search_strings:
print string
# Check to see if string exists in the GC files
error = "{0} {1} continuous subdaily legacy period detection failed -- not found in WMTS GetCapabilities".format(
period_length, period_unit)
check_result = all(string for string in search_strings
if find_string(wmts_gc_file, string))
# Cleanup
rmtree(config['wmts_gc_path'])
rmtree(config['wmts_staging_location'])
rmtree(config['twms_staging_location'])
rmtree(config['archive_location'])
# Check result
self.assertTrue(check_result, error)
def test_layer_config_default(self):
# Set config files and reference hash for checking empty tile
layer_config = os.path.join(self.testfiles_path,
'conf/test_default_behavior.xml')
config = get_layer_config(layer_config, self.archive_config)
# Make test dirs
make_dir_tree(config['archive_location'])
make_dir_tree(config['wmts_gc_path'])
make_dir_tree(config['twms_gc_path'])
make_dir_tree(config['legend_location'])
# Copy colormaps
colormap_location = config['colormap_locations'][
0].firstChild.nodeValue
colormap = config['colormaps'][0].firstChild.nodeValue
make_dir_tree(colormap_location)
copy(os.path.join(self.testfiles_path, 'conf/' + colormap),
colormap_location)
# Run layer config tool
cmd = 'oe_configure_layer -l {0} -a {1} -c {2} -p {3} -m {4}'.format(
self.testfiles_path, self.archive_config, layer_config,
self.projection_config, self.tilematrixset_config)
run_command(cmd)
# Get all the test results before running the assertions. We do this because a failure ends the test and makes it impossible to clean up
wmts_cache_xml = os.path.join(config['archive_basepath'],
'cache_all_wmts.xml')
wmts_cache_file = os.path.join(config['archive_basepath'],
'cache_all_wmts.config')
wmts_cache = os.path.isfile(wmts_cache_file)
wmts_gc_file = os.path.join(config['wmts_gc_path'],
'getCapabilities.xml')
wmts_gc = os.path.isfile(
os.path.join(config['wmts_gc_path'], 'getCapabilities.xml'))
wmts_staging_file = os.path.join(config['wmts_staging_location'],
config['prefix'] + '.xml')
wmts_staging = os.path.isfile(wmts_staging_file)
twms_cache_xml = os.path.join(config['archive_basepath'],
'cache_all_twms.xml')
twms_cache_file = os.path.join(config['archive_basepath'],
'cache_all_twms.config')
twms_cache = os.path.isfile(twms_cache_file)
twms_gc_file = os.path.join(config['twms_gc_path'],
'getCapabilities.xml')
twms_gc = os.path.isfile(twms_gc_file)
twms_ts_file = os.path.join(config['twms_gc_path'],
'getTileService.xml')
twms_ts = os.path.isfile(twms_ts_file)
twms_staging_file = os.path.join(config['twms_staging_location'],
config['prefix'] + '_gc.xml')
twms_staging = os.path.isfile(twms_staging_file)
twms_staging_gts_file = os.path.join(config['twms_staging_location'],
config['prefix'] + '_gts.xml')
twms_staging_gts = os.path.exists(twms_staging_gts_file)
twms_staging_mrf_file = os.path.join(config['twms_staging_location'],
config['prefix'] + '.mrf')
twms_staging_mrf = os.path.exists(twms_staging_mrf_file)
self.assertTrue(
wmts_cache,
"Default layer config test -- cache_all_wmts.config not created")
self.assertTrue(
wmts_gc,
'Default layer config test -- WMTS getCapabilities.xml does not exist'
)
self.assertTrue(
wmts_staging, 'Default layer config test -- staging file ' +
wmts_staging_file + ' does not exist in WMTS staging area')
self.assertTrue(
twms_cache,
'Default layer config test -- cache_all_twms.config does not exist'
)
self.assertTrue(
twms_gc,
'Default layer config test -- TWMS getCapabilities.xml does not exist'
)
self.assertTrue(
twms_ts,
'Default layer config test -- TWMS getTileService.xml does not exist'
)
self.assertTrue(
twms_staging, 'Default layer config test -- staging file ' +
twms_staging_file + ' does not exist in TWMS staging area')
self.assertTrue(
twms_staging_gts, 'Default layer config test -- staging file ' +
twms_staging_gts_file + ' does not exist in TWMS staging area')
self.assertTrue(
twms_staging_mrf, 'Default layer config test -- staging file ' +
twms_staging_mrf_file + ' does not exist in TWMS staging area')
rmtree(config['legend_location'])
rmtree(colormap_location)
# String searches in the GC and config filenames
search_string = '<ows:Identifier>' + config[
'identifier'] + '</ows:Identifier>'
contains_layer = find_string(wmts_gc_file, search_string)
os.remove(wmts_gc_file)
os.remove(wmts_cache_xml)
self.assertTrue(
contains_layer,
'Default layer config test -- WMTS GetCapabilities does not contain layer'
)
# Unicode weirdness in the binary configs necessitates running str() on the search strings
search_string = str(
'SERVICE=WMTS&REQUEST=GetTile&VERSION=1.0.0&LAYER=' +
config['identifier'] +
'&STYLE=(default)?&TILEMATRIXSET=EPSG3413_500m&TILEMATRIX=[0-9]*&TILEROW=[0-9]*&TILECOL=[0-9]*&FORMAT=image%2Fjpeg'
)
contains_layer = find_string(wmts_cache_file, search_string)
os.remove(wmts_cache_file)
os.remove(twms_cache_xml)
self.assertTrue(
contains_layer,
'Default layer config test -- WMTS cache configuration does not contain layer'
)
search_string = '<Name>' + config['identifier'] + '</Name>'
contains_layer = find_string(twms_gc_file, search_string)
os.remove(twms_gc_file)
self.assertTrue(
contains_layer,
'Default layer config test -- TWMS GetCapabilities does not contain layer'
)
search_string = '<Name>' + config['tiled_group_name'] + '</Name>'
contains_layer = find_string(twms_ts_file, search_string)
os.remove(twms_ts_file)
self.assertTrue(
contains_layer,
'Default layer config test -- GetTileService does not contain layer'
)
search_string = str(
'request=GetMap&layers=' + config['prefix'] +
'&srs=EPSG:3413&format=image%2Fjpeg&styles=&width=512&height=512&bbox=[-,\.0-9+Ee]'
)
contains_layer = find_string(twms_cache_file, search_string)
os.remove(twms_cache_file)
self.assertTrue(
contains_layer,
'Default layer config test -- TWMS cache configuration does not contain layer'
)
rmtree(config['archive_location'])
rmtree(config['wmts_gc_path'])
rmtree(config['wmts_staging_location'])
rmtree(config['twms_staging_location'])
def test_intermittent_zlevel_period_detection(self):
"""
Checks that the start and end periods of a z-level file are being correctly detected.
"""
if DEBUG:
print '\nTESTING INTERMITTENT Z-LEVEL PERIOD DETECTION...'
layer_config = os.path.join(self.testfiles_path,
'conf/test_zindex_detect.xml')
test_periods = (('minutes', 1), ('minutes', 5), ('hours', 1))
start_datetime = datetime.datetime(2014, 6, 1, 12, 0, 0)
start_datestring = str(start_datetime.year) + str(
start_datetime.timetuple().tm_yday).zfill(3)
config = get_layer_config(layer_config, self.archive_config)
for period_unit, period_length in test_periods:
# Make archive location dir
archive_location = os.path.join(config['archive_location'],
str(start_datetime.year))
make_dir_tree(archive_location)
# Make temp GC and archive directories and dummy MRF
make_dir_tree(config['wmts_gc_path'])
make_dir_tree(config['twms_gc_path'])
make_dir_tree(config['wmts_staging_location'])
make_dir_tree(config['twms_staging_location'])
dummy_mrf = os.path.join(
archive_location,
config['prefix'] + start_datestring + '_.mrf')
open(dummy_mrf, 'a').close()
# Create a ZDB file for seeding with the dates we're looking for
zdb_path = os.path.join(
archive_location,
config['prefix'] + start_datestring + '_.zdb')
conn = sqlite3.connect(zdb_path)
# Create ZINDEX table, generate test dates, and populate ZDB file
conn.execute(
'CREATE TABLE ZINDEX(z INTEGER PRIMARY KEY AUTOINCREMENT, key_str TEXT);'
)
test_intervals = testutils.create_intermittent_period_test_files(
config['archive_location'],
period_unit,
period_length,
5,
start_datetime,
prefix=config['prefix'],
no_files=True)
# Create debug output message (flatten array w/ itertools)
if DEBUG:
print 'Testing with {0} {1} periods'.format(
period_length, period_unit)
print 'Creating ZDB with dates: '
for date in itertools.chain.from_iterable(test_intervals):
print date.isoformat()
# Populate the dates in the ZDB (flatten array w/ itertools)
for i, date in enumerate(
itertools.chain.from_iterable(test_intervals)):
z_key = date.strftime('%Y%m%d%H%M%S')
sql = 'INSERT INTO ZINDEX(z, key_str) VALUES ({0}, {1})'.format(
i, z_key)
conn.execute(sql)
conn.commit()
# Close ZDB and run layer config
conn.close()
# Run layer config command
cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format(
self.testfiles_path, self.archive_config, layer_config,
self.projection_config, self.tilematrixset_config)
run_command(cmd, ignore_warnings=True)
# Check to see if proper period in GetCapabilities
wmts_gc_file = os.path.join(config['wmts_gc_path'],
'getCapabilities.xml')
twms_gc_file = os.path.join(config['twms_gc_path'],
'getCapabilities.xml')
# Build GC search string
search_strings = []
for interval in test_intervals:
search_string = "<Value>" + interval[0].isoformat(
) + "Z/" + interval[-1].isoformat() + "Z</Value>"
search_strings.append(search_string)
if DEBUG:
print '\n' + 'Searching for strings in GetCapabilities: '
for string in search_strings:
print string
# Check to see if string exists in the GC files
error = "{0} {1} intermittent z-level period detection failed -- not found in WMTS GetCapabilities".format(
period_length, period_unit)
check_result = all(string for string in search_strings
if find_string(wmts_gc_file, string))
# Cleanup
conn.close()
rmtree(config['wmts_gc_path'])
rmtree(config['wmts_staging_location'])
rmtree(config['twms_staging_location'])
rmtree(config['archive_location'])
# Check result
self.assertTrue(check_result, error)
def test_intermittent_period_detection(self):
if DEBUG:
print '\nTESTING INTERMITTENT PERIOD DETECTION...'
layer_config = os.path.join(self.testfiles_path,
'conf/test_period_detection.xml')
# Pick the start time for the dates that will be generated
start_datetime = datetime.datetime(2014, 6, 1)
# Set the various period units and lengths we'll be testing
test_periods = (('days', 1), ('days', 5), ('months', 1), ('years', 1))
config = get_layer_config(layer_config, self.archive_config)
for period_unit, period_length in test_periods:
# We test detection with both year and non-year directory setups
for year_dir in (True, False):
make_dir_tree(config['wmts_gc_path'])
make_dir_tree(config['twms_gc_path'])
make_dir_tree(config['wmts_staging_location'])
make_dir_tree(config['twms_staging_location'])
# Generate the empty test files
test_intervals = testutils.create_intermittent_period_test_files(
config['archive_location'],
period_unit,
period_length,
5,
start_datetime,
prefix=config['prefix'],
make_year_dirs=year_dir)
# Run layer config command for daily test days
cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format(
self.testfiles_path, self.archive_config, layer_config,
self.projection_config, self.tilematrixset_config)
run_command(cmd, ignore_warnings=True)
# Check to see if proper period in GetCapabilities
wmts_gc_file = os.path.join(config['wmts_gc_path'],
'getCapabilities.xml')
twms_gc_file = os.path.join(config['twms_gc_path'],
'getCapabilities.xml')
# Build a list GC search strings
search_strings = []
for interval in test_intervals:
search_string = "<Value>" + interval[0].isoformat() + "/" + interval[-1].isoformat() + \
"/P{0}{1}</Value>".format(period_length, period_unit[0].upper())
search_strings.append(search_string)
# Create debug output message
if DEBUG:
print '\n' + 'Creating dates: '
dates = [
date for date in interval
for interval in test_intervals
]
for date in dates:
print date.isoformat()
print '\n' + 'Searching for string(s) in GetCapabilities: '
for string in search_strings:
print search_string
# Check to see if string exists in the GC files
wmts_error = "{0} {1} intermittent period detection failed -- not found in WMTS GetCapabilities".format(
period_length, period_unit)
# twms_error = "{0} {1} period detection failed -- not found in TMWS GetCapabilities".format(period_length, period_unit)
# self.assertTrue(find_string(twms_gc_file, search_string), twms_error)
search_result = all(string for string in search_strings
if find_string(wmts_gc_file, string))
# Cleanup -- make sure to get rid of staging files
rmtree(config['wmts_gc_path'])
rmtree(config['wmts_staging_location'])
rmtree(config['twms_staging_location'])
rmtree(config['archive_location'])
# Check result
self.assertTrue(search_result, wmts_error)
def test_layer_config_default(self):
# Set config files and reference hash for checking empty tile
layer_config = os.path.join(self.testfiles_path, "conf/test_default_behavior.xml")
config = get_layer_config(layer_config, self.archive_config)
# Make test dirs
make_dir_tree(config["archive_location"])
make_dir_tree(config["wmts_gc_path"])
make_dir_tree(config["twms_gc_path"])
make_dir_tree(config["legend_location"])
# Copy colormaps
colormap_location = config["colormap_locations"][0].firstChild.nodeValue
colormap = config["colormaps"][0].firstChild.nodeValue
make_dir_tree(colormap_location)
copy(os.path.join(self.testfiles_path, "conf/" + colormap), colormap_location)
# Run layer config tool
cmd = "oe_configure_layer -l {0} -a {1} -c {2} -p {3} -m {4}".format(
self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config
)
run_command(cmd)
# Get all the test results before running the assertions. We do this because a failure ends the test and makes it impossible to clean up
wmts_cache_xml = os.path.join(config["archive_basepath"], "cache_all_wmts.xml")
wmts_cache_file = os.path.join(config["archive_basepath"], "cache_all_wmts.config")
wmts_cache = os.path.isfile(wmts_cache_file)
wmts_gc_file = os.path.join(config["wmts_gc_path"], "getCapabilities.xml")
wmts_gc = os.path.isfile(os.path.join(config["wmts_gc_path"], "getCapabilities.xml"))
wmts_staging_file = os.path.join(config["wmts_staging_location"], config["prefix"] + ".xml")
wmts_staging = os.path.isfile(wmts_staging_file)
twms_cache_xml = os.path.join(config["archive_basepath"], "cache_all_twms.xml")
twms_cache_file = os.path.join(config["archive_basepath"], "cache_all_twms.config")
twms_cache = os.path.isfile(twms_cache_file)
twms_gc_file = os.path.join(config["twms_gc_path"], "getCapabilities.xml")
twms_gc = os.path.isfile(twms_gc_file)
twms_ts_file = os.path.join(config["twms_gc_path"], "getTileService.xml")
twms_ts = os.path.isfile(twms_ts_file)
twms_staging_file = os.path.join(config["twms_staging_location"], config["prefix"] + "_gc.xml")
twms_staging = os.path.isfile(twms_staging_file)
twms_staging_gts_file = os.path.join(config["twms_staging_location"], config["prefix"] + "_gts.xml")
twms_staging_gts = os.path.exists(twms_staging_gts_file)
twms_staging_mrf_file = os.path.join(config["twms_staging_location"], config["prefix"] + ".mrf")
twms_staging_mrf = os.path.exists(twms_staging_mrf_file)
self.assertTrue(wmts_cache, "Default layer config test -- cache_all_wmts.config not created")
self.assertTrue(wmts_gc, "Default layer config test -- WMTS getCapabilities.xml does not exist")
self.assertTrue(
wmts_staging,
"Default layer config test -- staging file " + wmts_staging_file + " does not exist in WMTS staging area",
)
self.assertTrue(twms_cache, "Default layer config test -- cache_all_twms.config does not exist")
self.assertTrue(twms_gc, "Default layer config test -- TWMS getCapabilities.xml does not exist")
self.assertTrue(twms_ts, "Default layer config test -- TWMS getTileService.xml does not exist")
self.assertTrue(
twms_staging,
"Default layer config test -- staging file " + twms_staging_file + " does not exist in TWMS staging area",
)
self.assertTrue(
twms_staging_gts,
"Default layer config test -- staging file "
+ twms_staging_gts_file
+ " does not exist in TWMS staging area",
)
self.assertTrue(
twms_staging_mrf,
"Default layer config test -- staging file "
+ twms_staging_mrf_file
+ " does not exist in TWMS staging area",
)
rmtree(config["legend_location"])
rmtree(colormap_location)
# String searches in the GC and config filenames
search_string = "<ows:Identifier>" + config["identifier"] + "</ows:Identifier>"
contains_layer = find_string(wmts_gc_file, search_string)
os.remove(wmts_gc_file)
os.remove(wmts_cache_xml)
self.assertTrue(contains_layer, "Default layer config test -- WMTS GetCapabilities does not contain layer")
# Unicode weirdness in the binary configs necessitates running str() on the search strings
search_string = str(
"SERVICE=WMTS&REQUEST=GetTile&VERSION=1.0.0&LAYER="
+ config["identifier"]
+ "&STYLE=(default)?&TILEMATRIXSET=EPSG3413_500m&TILEMATRIX=[0-9]*&TILEROW=[0-9]*&TILECOL=[0-9]*&FORMAT=image%2Fjpeg"
)
contains_layer = find_string(wmts_cache_file, search_string)
os.remove(wmts_cache_file)
os.remove(twms_cache_xml)
self.assertTrue(contains_layer, "Default layer config test -- WMTS cache configuration does not contain layer")
search_string = "<Name>" + config["identifier"] + "</Name>"
contains_layer = find_string(twms_gc_file, search_string)
os.remove(twms_gc_file)
self.assertTrue(contains_layer, "Default layer config test -- TWMS GetCapabilities does not contain layer")
search_string = "<Name>" + config["tiled_group_name"] + "</Name>"
contains_layer = find_string(twms_ts_file, search_string)
os.remove(twms_ts_file)
self.assertTrue(contains_layer, "Default layer config test -- GetTileService does not contain layer")
search_string = str(
"request=GetMap&layers="
+ config["prefix"]
+ "&srs=EPSG:3413&format=image%2Fjpeg&styles=&width=512&height=512&bbox=[-,\.0-9+Ee]"
)
contains_layer = find_string(twms_cache_file, search_string)
os.remove(twms_cache_file)
self.assertTrue(contains_layer, "Default layer config test -- TWMS cache configuration does not contain layer")
rmtree(config["archive_location"])
rmtree(config["wmts_gc_path"])
rmtree(config["wmts_staging_location"])
rmtree(config["twms_staging_location"])
def test_legacy_subdaily_intermittent(self):
"""
Checks that layer config tool is correctly detecting the period and interval
of subdaily layers that have the datetime in their filenames as opposed to
the z-index.
"""
if DEBUG:
print "\nTESTING LEGACY SUBDAILY INTERMITTENT PERIOD DETECTION..."
layer_config = os.path.join(self.testfiles_path, "conf/test_subdaily_detect.xml")
start_datetime = datetime.datetime(2014, 6, 1, 12)
config = get_layer_config(layer_config, self.archive_config)
# Set the various period units and lengths we'll be testing
test_periods = (("hours", 1), ("hours", 5), ("minutes", 1), ("minutes", 5), ("seconds", 1), ("seconds", 5))
# Test intermittent periods
for period_unit, period_length in test_periods:
for year_dir in (True, False):
# Make the GC dirs
make_dir_tree(config["wmts_gc_path"])
make_dir_tree(config["twms_gc_path"])
make_dir_tree(config["wmts_staging_location"])
make_dir_tree(config["twms_staging_location"])
test_intervals = testutils.create_intermittent_period_test_files(
config["archive_location"],
period_unit,
period_length,
5,
start_datetime,
prefix=config["prefix"],
make_year_dirs=year_dir,
)
# Create debug output message (flatten array w/ itertools)
if DEBUG:
print "\nTesting with {0} {1} periods".format(period_length, period_unit)
print "Creating legacy subdaily files with dates: "
for date in itertools.chain.from_iterable(test_intervals):
print date.isoformat()
# Run layer config command
cmd = "oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}".format(
self.testfiles_path,
self.archive_config,
layer_config,
self.projection_config,
self.tilematrixset_config,
)
run_command(cmd, ignore_warnings=True)
# Check to see if proper period in GetCapabilities
wmts_gc_file = os.path.join(config["wmts_gc_path"], "getCapabilities.xml")
twms_gc_file = os.path.join(config["twms_gc_path"], "getCapabilities.xml")
search_strings = []
for interval in test_intervals:
time_string = testutils.get_time_string(interval[0], interval[-1], config)
search_string = (
"<Value>" + time_string + "/PT{0}{1}</Value>".format(period_length, period_unit[0].upper())
)
search_strings.append(search_string)
if DEBUG:
print "\n" + "Searching for strings in GetCapabilities: "
for string in search_strings:
print string
# Check to see if string exists in the GC files
error = "{0} {1} continuous subdaily legacy period detection failed -- not found in WMTS GetCapabilities".format(
period_length, period_unit
)
check_result = all(string for string in search_strings if find_string(wmts_gc_file, string))
# Cleanup
rmtree(config["wmts_gc_path"])
rmtree(config["wmts_staging_location"])
rmtree(config["twms_staging_location"])
rmtree(config["archive_location"])
# Check result
self.assertTrue(check_result, error)
def test_intermittent_zlevel_period_detection(self):
"""
Checks that the start and end periods of a z-level file are being correctly detected.
"""
if DEBUG:
print "\nTESTING INTERMITTENT Z-LEVEL PERIOD DETECTION..."
layer_config = os.path.join(self.testfiles_path, "conf/test_zindex_detect.xml")
test_periods = (("minutes", 1), ("minutes", 5), ("hours", 1))
start_datetime = datetime.datetime(2014, 6, 1, 12, 0, 0)
start_datestring = str(start_datetime.year) + str(start_datetime.timetuple().tm_yday).zfill(3)
config = get_layer_config(layer_config, self.archive_config)
for period_unit, period_length in test_periods:
# Make archive location dir
archive_location = os.path.join(config["archive_location"], str(start_datetime.year))
make_dir_tree(archive_location)
# Make temp GC and archive directories and dummy MRF
make_dir_tree(config["wmts_gc_path"])
make_dir_tree(config["twms_gc_path"])
make_dir_tree(config["wmts_staging_location"])
make_dir_tree(config["twms_staging_location"])
dummy_mrf = os.path.join(archive_location, config["prefix"] + start_datestring + "_.mrf")
open(dummy_mrf, "a").close()
# Create a ZDB file for seeding with the dates we're looking for
zdb_path = os.path.join(archive_location, config["prefix"] + start_datestring + "_.zdb")
conn = sqlite3.connect(zdb_path)
# Create ZINDEX table, generate test dates, and populate ZDB file
conn.execute("CREATE TABLE ZINDEX(z INTEGER PRIMARY KEY AUTOINCREMENT, key_str TEXT);")
test_intervals = testutils.create_intermittent_period_test_files(
config["archive_location"],
period_unit,
period_length,
5,
start_datetime,
prefix=config["prefix"],
no_files=True,
)
# Create debug output message (flatten array w/ itertools)
if DEBUG:
print "Testing with {0} {1} periods".format(period_length, period_unit)
print "Creating ZDB with dates: "
for date in itertools.chain.from_iterable(test_intervals):
print date.isoformat()
# Populate the dates in the ZDB (flatten array w/ itertools)
for i, date in enumerate(itertools.chain.from_iterable(test_intervals)):
z_key = date.strftime("%Y%m%d%H%M%S")
sql = "INSERT INTO ZINDEX(z, key_str) VALUES ({0}, {1})".format(i, z_key)
conn.execute(sql)
conn.commit()
# Close ZDB and run layer config
conn.close()
# Run layer config command
cmd = "oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}".format(
self.testfiles_path,
self.archive_config,
layer_config,
self.projection_config,
self.tilematrixset_config,
)
run_command(cmd, ignore_warnings=True)
# Check to see if proper period in GetCapabilities
wmts_gc_file = os.path.join(config["wmts_gc_path"], "getCapabilities.xml")
twms_gc_file = os.path.join(config["twms_gc_path"], "getCapabilities.xml")
# Build GC search string
search_strings = []
for interval in test_intervals:
search_string = "<Value>" + interval[0].isoformat() + "Z/" + interval[-1].isoformat() + "Z</Value>"
search_strings.append(search_string)
if DEBUG:
print "\n" + "Searching for strings in GetCapabilities: "
for string in search_strings:
print string
# Check to see if string exists in the GC files
error = "{0} {1} intermittent z-level period detection failed -- not found in WMTS GetCapabilities".format(
period_length, period_unit
)
check_result = all(string for string in search_strings if find_string(wmts_gc_file, string))
# Cleanup
conn.close()
rmtree(config["wmts_gc_path"])
rmtree(config["wmts_staging_location"])
rmtree(config["twms_staging_location"])
rmtree(config["archive_location"])
# Check result
self.assertTrue(check_result, error)
def test_intermittent_period_detection(self):
if DEBUG:
print "\nTESTING INTERMITTENT PERIOD DETECTION..."
layer_config = os.path.join(self.testfiles_path, "conf/test_period_detection.xml")
# Pick the start time for the dates that will be generated
start_datetime = datetime.datetime(2014, 6, 1)
# Set the various period units and lengths we'll be testing
test_periods = (("days", 1), ("days", 5), ("months", 1), ("years", 1))
config = get_layer_config(layer_config, self.archive_config)
for period_unit, period_length in test_periods:
# We test detection with both year and non-year directory setups
for year_dir in (True, False):
make_dir_tree(config["wmts_gc_path"])
make_dir_tree(config["twms_gc_path"])
make_dir_tree(config["wmts_staging_location"])
make_dir_tree(config["twms_staging_location"])
# Generate the empty test files
test_intervals = testutils.create_intermittent_period_test_files(
config["archive_location"],
period_unit,
period_length,
5,
start_datetime,
prefix=config["prefix"],
make_year_dirs=year_dir,
)
# Run layer config command for daily test days
cmd = "oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}".format(
self.testfiles_path,
self.archive_config,
layer_config,
self.projection_config,
self.tilematrixset_config,
)
run_command(cmd, ignore_warnings=True)
# Check to see if proper period in GetCapabilities
wmts_gc_file = os.path.join(config["wmts_gc_path"], "getCapabilities.xml")
twms_gc_file = os.path.join(config["twms_gc_path"], "getCapabilities.xml")
# Build a list GC search strings
search_strings = []
for interval in test_intervals:
search_string = (
"<Value>"
+ interval[0].isoformat()
+ "/"
+ interval[-1].isoformat()
+ "/P{0}{1}</Value>".format(period_length, period_unit[0].upper())
)
search_strings.append(search_string)
# Create debug output message
if DEBUG:
print "\n" + "Creating dates: "
dates = [date for date in interval for interval in test_intervals]
for date in dates:
print date.isoformat()
print "\n" + "Searching for string(s) in GetCapabilities: "
for string in search_strings:
print search_string
# Check to see if string exists in the GC files
wmts_error = "{0} {1} intermittent period detection failed -- not found in WMTS GetCapabilities".format(
period_length, period_unit
)
# twms_error = "{0} {1} period detection failed -- not found in TMWS GetCapabilities".format(period_length, period_unit)
# self.assertTrue(find_string(twms_gc_file, search_string), twms_error)
search_result = all(string for string in search_strings if find_string(wmts_gc_file, string))
# Cleanup -- make sure to get rid of staging files
rmtree(config["wmts_gc_path"])
rmtree(config["wmts_staging_location"])
rmtree(config["twms_staging_location"])
rmtree(config["archive_location"])
# Check result
self.assertTrue(search_result, wmts_error)
def test_legacy_subdaily_continuous(self):
"""
Checks that layer config tool is correctly detecting the period and interval
of subdaily layers that have the datetime in their filenames as opposed to
the z-index.
"""
if DEBUG:
print '\nTESTING LEGACY SUBDAILY CONTINUOUS PERIOD DETECTION...'
layer_config = os.path.join(self.testfiles_path, 'conf/test_subdaily_detect.xml')
start_datetime = datetime.datetime(2014, 6, 1, 12)
config = get_layer_config(layer_config, self.archive_config)
# Set the various period units and lengths we'll be testing
test_periods = (('hours', 1), ('hours', 5), ('minutes', 1), ('minutes', 5), ('seconds', 1), ('seconds', 5))
# Test continuous periods
for period_unit, period_length in test_periods:
for year_dir in (True, False):
# Make the GC dirs
make_dir_tree(config['wmts_gc_path'])
make_dir_tree(config['twms_gc_path'])
make_dir_tree(config['wmts_staging_location'])
make_dir_tree(config['twms_staging_location'])
# Create a continuous set of period files for each time interval
test_dates = testutils.create_continuous_period_test_files(config['archive_location'], period_unit, period_length, 4, start_datetime, prefix=config['prefix'], make_year_dirs=year_dir)
# Create debug output message
if DEBUG:
print '\nTesting with {0} {1} periods'.format(period_length, period_unit)
print 'Creating legacy subdaily files with dates: '
for date in test_dates:
print date.isoformat()
# Run layer config command
cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format(self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config)
run_command(cmd, ignore_warnings=True)
# Check to see if proper period in GetCapabilities
wmts_gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml')
twms_gc_file = os.path.join(config['twms_gc_path'], 'getCapabilities.xml')
# Build search strings
time_string = testutils.get_time_string(start_datetime, test_dates[-1], config)
search_string = "<Value>" + time_string + "/PT{0}{1}</Value>".format(period_length, period_unit[0].upper())
search_result = find_string(wmts_gc_file, search_string)
if DEBUG:
print '\n' + 'Searching for string in GetCapabilities: ' + search_string
# Cleanup
rmtree(config['wmts_gc_path'])
rmtree(config['wmts_staging_location'])
rmtree(config['twms_staging_location'])
rmtree(config['archive_location'])
# Check result
error = "{0} {1} continuous subdaily legacy period detection failed -- not found in WMTS GetCapabilities".format(period_length, period_unit)
self.assertTrue(search_result, error)
def test_continuous_zlevel_period_detection(self):
"""
Checks that the start and end periods of a z-level file are being correctly detected.
"""
if DEBUG:
print '\nTESTING CONTINUOUS Z-LEVEL PERIOD DETECTION...'
layer_config = os.path.join(self.testfiles_path, 'conf/test_zindex_detect.xml')
test_periods = (('minutes', 1), ('minutes', 5), ('hours', 1))
start_datetime = datetime.datetime(2014, 6, 1, 12, 0, 0)
start_datestring = str(start_datetime.year) + str(start_datetime.timetuple().tm_yday).zfill(3)
config = get_layer_config(layer_config, self.archive_config)
for period_unit, period_length in test_periods:
# Make archive location dir
archive_location = os.path.join(config['archive_location'], str(start_datetime.year))
make_dir_tree(archive_location)
# Make temp GC and archive directories and dummy MRF
make_dir_tree(config['wmts_gc_path'])
make_dir_tree(config['twms_gc_path'])
make_dir_tree(config['wmts_staging_location'])
make_dir_tree(config['twms_staging_location'])
dummy_mrf = os.path.join(archive_location, config['prefix'] + start_datestring + '_.mrf')
open(dummy_mrf, 'a').close()
# Create a ZDB file for seeding with the dates we're looking for
zdb_path = os.path.join(archive_location, config['prefix'] + start_datestring + '_.zdb')
conn = sqlite3.connect(zdb_path)
# Create ZINDEX table, generate test dates, and populate ZDB file
conn.execute('CREATE TABLE ZINDEX(z INTEGER PRIMARY KEY AUTOINCREMENT, key_str TEXT);')
test_dates = testutils.create_continuous_period_test_files(
config['archive_location'], period_unit, period_length, 5, start_datetime, prefix=config['prefix'], no_files=True)
# Create debug output message
if DEBUG:
print 'Testing with {0} {1} periods'.format(period_length, period_unit)
print 'Creating ZDB with dates: '
for date in test_dates:
print date.isoformat()
# Populate the dates in the ZDB
for i, date in enumerate(test_dates):
z_key = date.strftime('%Y%m%d%H%M%S')
sql = 'INSERT INTO ZINDEX(z, key_str) VALUES ({0}, {1})'.format(i, z_key)
conn.execute(sql)
conn.commit()
# Close ZDB and run layer config
conn.close()
cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format(self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config)
run_command(cmd, ignore_warnings=True)
# Check to see if proper period in GetCapabilities
wmts_gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml')
twms_gc_file = os.path.join(config['twms_gc_path'], 'getCapabilities.xml')
# Build GC search string
search_string = "<Value>" + test_dates[0].isoformat() + "Z/" + test_dates[-1].isoformat() + "Z</Value>"
if DEBUG:
print '\n' + 'Searching for string in GetCapabilities: ' + search_string
# Check to see if string exists in the GC files
error = "{0} {1} continuous z-level period detection failed -- not found in WMTS GetCapabilities".format(period_length, period_unit)
check_result = find_string(wmts_gc_file, search_string)
# Cleanup
conn.close()
rmtree(config['wmts_gc_path'])
rmtree(config['wmts_staging_location'])
rmtree(config['twms_staging_location'])
rmtree(config['archive_location'])
# Check result
self.assertTrue(check_result, error)
