def test_versioned_colormaps(self): # Set locations of the config files we're using for this test layer_config = os.path.join(self.testfiles_path, 'conf/test_versioned_colormaps.xml') test_metadata = ('<ows:Metadata xlink:type="simple" xlink:role="http://earthdata.nasa.gov/gibs/metadata-type/colormap" xlink:href="http://localhost/colormaps/2.0/MODIS_Aqua_Aerosol-GIBS_2.0.xml" xlink:title="GIBS Color Map: Data - RGB Mapping"/>', '<ows:Metadata xlink:type="simple" xlink:role="http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.0" xlink:href="http://localhost/colormaps/1.0//MODIS_Aqua_Aerosol-GIBS_1.0.xml" xlink:title="GIBS Color Map: Data - RGB Mapping"/>' '<ows:Metadata xlink:type="simple" xlink:role="http://earthdata.nasa.gov/gibs/metadata-type/colormap/2.0" xlink:href="http://localhost/colormaps/2.0//MODIS_Aqua_Aerosol-GIBS_2.0.xml" xlink:title="GIBS Color Map: Data - RGB Mapping"/>') config = get_layer_config(layer_config, self.archive_config) # Create colormap locations and copy colormap test file for location in config['colormap_locations']: make_dir_tree(location.firstChild.nodeValue) colormap = next(colormap.firstChild.nodeValue for colormap in config['colormaps'] if colormap.attributes['version'].value == location.attributes['version'].value) copy(os.path.join(self.testfiles_path, 'conf/' + colormap), location.firstChild.nodeValue) # Create paths for data and GC make_dir_tree(config['wmts_gc_path']) make_dir_tree(config['twms_gc_path']) make_dir_tree(config['archive_location']) # Run layer config tool cmd = 'oe_configure_layer -l {0} --skip_empty_tiles -a {1} -c {2} -p {3} -m {4}'.format(self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config) run_command(cmd) # Check to see if all required metadata lines are in the getCapabilities gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml') metadata_pass = all(line for line in test_metadata if find_string(gc_file, line)) self.assertTrue(metadata_pass, "Can't find all the proper versioned colormap metadata in the WMTS GetCapabilities file or GC file was not created.") # Cleanup [rmtree(path) for path in [path.firstChild.nodeValue for path in config['colormap_locations']]] rmtree(config['wmts_gc_path']) rmtree(config['wmts_staging_location']) rmtree(config['twms_staging_location'])
def test_continuous_period_detection(self): if DEBUG: print '\nTESTING CONTINUOUS PERIOD DETECTION...' layer_config = os.path.join(self.testfiles_path, 'conf/test_period_detection.xml') # Pick the start time for the dates that will be generated start_datetime = datetime.datetime(2014, 6, 1) # Set the various period units and lengths we'll be testing test_periods = (('days', 1), ('days', 5), ('months', 1), ('years', 1)) config = get_layer_config(layer_config, self.archive_config) for period_unit, period_length in test_periods: # We test detection with both year and non-year directory setups for year_dir in (True, False): make_dir_tree(config['wmts_gc_path']) make_dir_tree(config['twms_gc_path']) make_dir_tree(config['wmts_staging_location']) make_dir_tree(config['twms_staging_location']) # Generate the empty test files test_dates = testutils.create_continuous_period_test_files( config['archive_location'], period_unit, period_length, 5, start_datetime, prefix=config['prefix'], make_year_dirs=year_dir) # Run layer config command for daily test days cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format(self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config) run_command(cmd, ignore_warnings=True) # Check to see if proper period in GetCapabilities wmts_gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml') twms_gc_file = os.path.join(config['twms_gc_path'], 'getCapabilities.xml') # Build GC search string search_string = "<Value>" + test_dates[0].date().isoformat() + "/" + test_dates[-1].date().isoformat() + "/P{0}{1}</Value>".format(period_length, period_unit[0].upper()) # Create debug output message if DEBUG: print 'Testing with {0} {1} periods'.format(period_length, period_unit) print 'Creating dates: ' for date in test_dates: print date.isoformat() print '\n' + 'Searching for string in GetCapabilities: ' + search_string # Check to see if string exists in the GC files wmts_error = "{0} {1} continuous period detection failed -- not found in WMTS GetCapabilities".format(period_length, period_unit) test_result = find_string(wmts_gc_file, search_string) # twms_error = "{0} {1} period detection failed -- not found in TMWS GetCapabilities".format(period_length, period_unit) # self.assertTrue(find_string(twms_gc_file, search_string), twms_error) # Cleanup -- make sure to get rid of staging files rmtree(config['wmts_gc_path']) rmtree(config['wmts_staging_location']) rmtree(config['twms_staging_location']) rmtree(config['archive_location']) # Check result self.assertTrue(test_result, wmts_error)
def test_versioned_colormaps(self): # Set locations of the config files we're using for this test layer_config = os.path.join(self.testfiles_path, 'conf/test_versioned_colormaps.xml') test_metadata = ( '<ows:Metadata xlink:type="simple" xlink:role="http://earthdata.nasa.gov/gibs/metadata-type/colormap" xlink:href="http://localhost/colormaps/2.0/MODIS_Aqua_Aerosol-GIBS_2.0.xml" xlink:title="GIBS Color Map: Data - RGB Mapping"/>', '<ows:Metadata xlink:type="simple" xlink:role="http://earthdata.nasa.gov/gibs/metadata-type/colormap/1.0" xlink:href="http://localhost/colormaps/1.0//MODIS_Aqua_Aerosol-GIBS_1.0.xml" xlink:title="GIBS Color Map: Data - RGB Mapping"/>' '<ows:Metadata xlink:type="simple" xlink:role="http://earthdata.nasa.gov/gibs/metadata-type/colormap/2.0" xlink:href="http://localhost/colormaps/2.0//MODIS_Aqua_Aerosol-GIBS_2.0.xml" xlink:title="GIBS Color Map: Data - RGB Mapping"/>' ) config = get_layer_config(layer_config, self.archive_config) # Create colormap locations and copy colormap test file for location in config['colormap_locations']: make_dir_tree(location.firstChild.nodeValue) colormap = next(colormap.firstChild.nodeValue for colormap in config['colormaps'] if colormap.attributes['version'].value == location.attributes['version'].value) copy(os.path.join(self.testfiles_path, 'conf/' + colormap), location.firstChild.nodeValue) # Create paths for data and GC make_dir_tree(config['wmts_gc_path']) make_dir_tree(config['twms_gc_path']) make_dir_tree(config['archive_location']) # Run layer config tool cmd = 'oe_configure_layer -l {0} --skip_empty_tiles -a {1} -c {2} -p {3} -m {4}'.format( self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config) run_command(cmd) # Check to see if all required metadata lines are in the getCapabilities gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml') metadata_pass = all(line for line in test_metadata if find_string(gc_file, line)) self.assertTrue( metadata_pass, "Can't find all the proper versioned colormap metadata in the WMTS GetCapabilities file or GC file was not created." ) # Cleanup [ rmtree(path) for path in [ path.firstChild.nodeValue for path in config['colormap_locations'] ] ] rmtree(config['wmts_gc_path']) rmtree(config['wmts_staging_location']) rmtree(config['twms_staging_location'])
def test_legacy_subdaily_intermittent(self): """ Checks that layer config tool is correctly detecting the period and interval of subdaily layers that have the datetime in their filenames as opposed to the z-index. """ if DEBUG: print '\nTESTING LEGACY SUBDAILY INTERMITTENT PERIOD DETECTION...' layer_config = os.path.join(self.testfiles_path, 'conf/test_subdaily_detect.xml') start_datetime = datetime.datetime(2014, 6, 1, 12) config = get_layer_config(layer_config, self.archive_config) # Set the various period units and lengths we'll be testing test_periods = (('hours', 1), ('hours', 5), ('minutes', 1), ('minutes', 5), ('seconds', 1), ('seconds', 5)) # Test intermittent periods for period_unit, period_length in test_periods: for year_dir in (True, False): # Make the GC dirs make_dir_tree(config['wmts_gc_path']) make_dir_tree(config['twms_gc_path']) make_dir_tree(config['wmts_staging_location']) make_dir_tree(config['twms_staging_location']) test_intervals = testutils.create_intermittent_period_test_files( config['archive_location'], period_unit, period_length, 5, start_datetime, prefix=config['prefix'], make_year_dirs=year_dir) # Create debug output message (flatten array w/ itertools) if DEBUG: print '\nTesting with {0} {1} periods'.format( period_length, period_unit) print 'Creating legacy subdaily files with dates: ' for date in itertools.chain.from_iterable(test_intervals): print date.isoformat() # Run layer config command cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format( self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config) run_command(cmd, ignore_warnings=True) # Check to see if proper period in GetCapabilities wmts_gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml') twms_gc_file = os.path.join(config['twms_gc_path'], 'getCapabilities.xml') search_strings = [] for interval in test_intervals: time_string = testutils.get_time_string( interval[0], interval[-1], config) search_string = "<Value>" + time_string + "/PT{0}{1}</Value>".format( period_length, period_unit[0].upper()) search_strings.append(search_string) if DEBUG: print '\n' + 'Searching for strings in GetCapabilities: ' for string in search_strings: print string # Check to see if string exists in the GC files error = "{0} {1} continuous subdaily legacy period detection failed -- not found in WMTS GetCapabilities".format( period_length, period_unit) check_result = all(string for string in search_strings if find_string(wmts_gc_file, string)) # Cleanup rmtree(config['wmts_gc_path']) rmtree(config['wmts_staging_location']) rmtree(config['twms_staging_location']) rmtree(config['archive_location']) # Check result self.assertTrue(check_result, error)
def test_layer_config_default(self): # Set config files and reference hash for checking empty tile layer_config = os.path.join(self.testfiles_path, 'conf/test_default_behavior.xml') config = get_layer_config(layer_config, self.archive_config) # Make test dirs make_dir_tree(config['archive_location']) make_dir_tree(config['wmts_gc_path']) make_dir_tree(config['twms_gc_path']) make_dir_tree(config['legend_location']) # Copy colormaps colormap_location = config['colormap_locations'][ 0].firstChild.nodeValue colormap = config['colormaps'][0].firstChild.nodeValue make_dir_tree(colormap_location) copy(os.path.join(self.testfiles_path, 'conf/' + colormap), colormap_location) # Run layer config tool cmd = 'oe_configure_layer -l {0} -a {1} -c {2} -p {3} -m {4}'.format( self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config) run_command(cmd) # Get all the test results before running the assertions. We do this because a failure ends the test and makes it impossible to clean up wmts_cache_xml = os.path.join(config['archive_basepath'], 'cache_all_wmts.xml') wmts_cache_file = os.path.join(config['archive_basepath'], 'cache_all_wmts.config') wmts_cache = os.path.isfile(wmts_cache_file) wmts_gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml') wmts_gc = os.path.isfile( os.path.join(config['wmts_gc_path'], 'getCapabilities.xml')) wmts_staging_file = os.path.join(config['wmts_staging_location'], config['prefix'] + '.xml') wmts_staging = os.path.isfile(wmts_staging_file) twms_cache_xml = os.path.join(config['archive_basepath'], 'cache_all_twms.xml') twms_cache_file = os.path.join(config['archive_basepath'], 'cache_all_twms.config') twms_cache = os.path.isfile(twms_cache_file) twms_gc_file = os.path.join(config['twms_gc_path'], 'getCapabilities.xml') twms_gc = os.path.isfile(twms_gc_file) twms_ts_file = os.path.join(config['twms_gc_path'], 'getTileService.xml') twms_ts = os.path.isfile(twms_ts_file) twms_staging_file = os.path.join(config['twms_staging_location'], config['prefix'] + '_gc.xml') twms_staging = os.path.isfile(twms_staging_file) twms_staging_gts_file = os.path.join(config['twms_staging_location'], config['prefix'] + '_gts.xml') twms_staging_gts = os.path.exists(twms_staging_gts_file) twms_staging_mrf_file = os.path.join(config['twms_staging_location'], config['prefix'] + '.mrf') twms_staging_mrf = os.path.exists(twms_staging_mrf_file) self.assertTrue( wmts_cache, "Default layer config test -- cache_all_wmts.config not created") self.assertTrue( wmts_gc, 'Default layer config test -- WMTS getCapabilities.xml does not exist' ) self.assertTrue( wmts_staging, 'Default layer config test -- staging file ' + wmts_staging_file + ' does not exist in WMTS staging area') self.assertTrue( twms_cache, 'Default layer config test -- cache_all_twms.config does not exist' ) self.assertTrue( twms_gc, 'Default layer config test -- TWMS getCapabilities.xml does not exist' ) self.assertTrue( twms_ts, 'Default layer config test -- TWMS getTileService.xml does not exist' ) self.assertTrue( twms_staging, 'Default layer config test -- staging file ' + twms_staging_file + ' does not exist in TWMS staging area') self.assertTrue( twms_staging_gts, 'Default layer config test -- staging file ' + twms_staging_gts_file + ' does not exist in TWMS staging area') self.assertTrue( twms_staging_mrf, 'Default layer config test -- staging file ' + twms_staging_mrf_file + ' does not exist in TWMS staging area') rmtree(config['legend_location']) rmtree(colormap_location) # String searches in the GC and config filenames search_string = '<ows:Identifier>' + config[ 'identifier'] + '</ows:Identifier>' contains_layer = find_string(wmts_gc_file, search_string) os.remove(wmts_gc_file) os.remove(wmts_cache_xml) self.assertTrue( contains_layer, 'Default layer config test -- WMTS GetCapabilities does not contain layer' ) # Unicode weirdness in the binary configs necessitates running str() on the search strings search_string = str( 'SERVICE=WMTS&REQUEST=GetTile&VERSION=1.0.0&LAYER=' + config['identifier'] + '&STYLE=(default)?&TILEMATRIXSET=EPSG3413_500m&TILEMATRIX=[0-9]*&TILEROW=[0-9]*&TILECOL=[0-9]*&FORMAT=image%2Fjpeg' ) contains_layer = find_string(wmts_cache_file, search_string) os.remove(wmts_cache_file) os.remove(twms_cache_xml) self.assertTrue( contains_layer, 'Default layer config test -- WMTS cache configuration does not contain layer' ) search_string = '<Name>' + config['identifier'] + '</Name>' contains_layer = find_string(twms_gc_file, search_string) os.remove(twms_gc_file) self.assertTrue( contains_layer, 'Default layer config test -- TWMS GetCapabilities does not contain layer' ) search_string = '<Name>' + config['tiled_group_name'] + '</Name>' contains_layer = find_string(twms_ts_file, search_string) os.remove(twms_ts_file) self.assertTrue( contains_layer, 'Default layer config test -- GetTileService does not contain layer' ) search_string = str( 'request=GetMap&layers=' + config['prefix'] + '&srs=EPSG:3413&format=image%2Fjpeg&styles=&width=512&height=512&bbox=[-,\.0-9+Ee]' ) contains_layer = find_string(twms_cache_file, search_string) os.remove(twms_cache_file) self.assertTrue( contains_layer, 'Default layer config test -- TWMS cache configuration does not contain layer' ) rmtree(config['archive_location']) rmtree(config['wmts_gc_path']) rmtree(config['wmts_staging_location']) rmtree(config['twms_staging_location'])
def test_intermittent_zlevel_period_detection(self): """ Checks that the start and end periods of a z-level file are being correctly detected. """ if DEBUG: print '\nTESTING INTERMITTENT Z-LEVEL PERIOD DETECTION...' layer_config = os.path.join(self.testfiles_path, 'conf/test_zindex_detect.xml') test_periods = (('minutes', 1), ('minutes', 5), ('hours', 1)) start_datetime = datetime.datetime(2014, 6, 1, 12, 0, 0) start_datestring = str(start_datetime.year) + str( start_datetime.timetuple().tm_yday).zfill(3) config = get_layer_config(layer_config, self.archive_config) for period_unit, period_length in test_periods: # Make archive location dir archive_location = os.path.join(config['archive_location'], str(start_datetime.year)) make_dir_tree(archive_location) # Make temp GC and archive directories and dummy MRF make_dir_tree(config['wmts_gc_path']) make_dir_tree(config['twms_gc_path']) make_dir_tree(config['wmts_staging_location']) make_dir_tree(config['twms_staging_location']) dummy_mrf = os.path.join( archive_location, config['prefix'] + start_datestring + '_.mrf') open(dummy_mrf, 'a').close() # Create a ZDB file for seeding with the dates we're looking for zdb_path = os.path.join( archive_location, config['prefix'] + start_datestring + '_.zdb') conn = sqlite3.connect(zdb_path) # Create ZINDEX table, generate test dates, and populate ZDB file conn.execute( 'CREATE TABLE ZINDEX(z INTEGER PRIMARY KEY AUTOINCREMENT, key_str TEXT);' ) test_intervals = testutils.create_intermittent_period_test_files( config['archive_location'], period_unit, period_length, 5, start_datetime, prefix=config['prefix'], no_files=True) # Create debug output message (flatten array w/ itertools) if DEBUG: print 'Testing with {0} {1} periods'.format( period_length, period_unit) print 'Creating ZDB with dates: ' for date in itertools.chain.from_iterable(test_intervals): print date.isoformat() # Populate the dates in the ZDB (flatten array w/ itertools) for i, date in enumerate( itertools.chain.from_iterable(test_intervals)): z_key = date.strftime('%Y%m%d%H%M%S') sql = 'INSERT INTO ZINDEX(z, key_str) VALUES ({0}, {1})'.format( i, z_key) conn.execute(sql) conn.commit() # Close ZDB and run layer config conn.close() # Run layer config command cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format( self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config) run_command(cmd, ignore_warnings=True) # Check to see if proper period in GetCapabilities wmts_gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml') twms_gc_file = os.path.join(config['twms_gc_path'], 'getCapabilities.xml') # Build GC search string search_strings = [] for interval in test_intervals: search_string = "<Value>" + interval[0].isoformat( ) + "Z/" + interval[-1].isoformat() + "Z</Value>" search_strings.append(search_string) if DEBUG: print '\n' + 'Searching for strings in GetCapabilities: ' for string in search_strings: print string # Check to see if string exists in the GC files error = "{0} {1} intermittent z-level period detection failed -- not found in WMTS GetCapabilities".format( period_length, period_unit) check_result = all(string for string in search_strings if find_string(wmts_gc_file, string)) # Cleanup conn.close() rmtree(config['wmts_gc_path']) rmtree(config['wmts_staging_location']) rmtree(config['twms_staging_location']) rmtree(config['archive_location']) # Check result self.assertTrue(check_result, error)
def test_intermittent_period_detection(self): if DEBUG: print '\nTESTING INTERMITTENT PERIOD DETECTION...' layer_config = os.path.join(self.testfiles_path, 'conf/test_period_detection.xml') # Pick the start time for the dates that will be generated start_datetime = datetime.datetime(2014, 6, 1) # Set the various period units and lengths we'll be testing test_periods = (('days', 1), ('days', 5), ('months', 1), ('years', 1)) config = get_layer_config(layer_config, self.archive_config) for period_unit, period_length in test_periods: # We test detection with both year and non-year directory setups for year_dir in (True, False): make_dir_tree(config['wmts_gc_path']) make_dir_tree(config['twms_gc_path']) make_dir_tree(config['wmts_staging_location']) make_dir_tree(config['twms_staging_location']) # Generate the empty test files test_intervals = testutils.create_intermittent_period_test_files( config['archive_location'], period_unit, period_length, 5, start_datetime, prefix=config['prefix'], make_year_dirs=year_dir) # Run layer config command for daily test days cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format( self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config) run_command(cmd, ignore_warnings=True) # Check to see if proper period in GetCapabilities wmts_gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml') twms_gc_file = os.path.join(config['twms_gc_path'], 'getCapabilities.xml') # Build a list GC search strings search_strings = [] for interval in test_intervals: search_string = "<Value>" + interval[0].isoformat() + "/" + interval[-1].isoformat() + \ "/P{0}{1}</Value>".format(period_length, period_unit[0].upper()) search_strings.append(search_string) # Create debug output message if DEBUG: print '\n' + 'Creating dates: ' dates = [ date for date in interval for interval in test_intervals ] for date in dates: print date.isoformat() print '\n' + 'Searching for string(s) in GetCapabilities: ' for string in search_strings: print search_string # Check to see if string exists in the GC files wmts_error = "{0} {1} intermittent period detection failed -- not found in WMTS GetCapabilities".format( period_length, period_unit) # twms_error = "{0} {1} period detection failed -- not found in TMWS GetCapabilities".format(period_length, period_unit) # self.assertTrue(find_string(twms_gc_file, search_string), twms_error) search_result = all(string for string in search_strings if find_string(wmts_gc_file, string)) # Cleanup -- make sure to get rid of staging files rmtree(config['wmts_gc_path']) rmtree(config['wmts_staging_location']) rmtree(config['twms_staging_location']) rmtree(config['archive_location']) # Check result self.assertTrue(search_result, wmts_error)
def test_layer_config_default(self): # Set config files and reference hash for checking empty tile layer_config = os.path.join(self.testfiles_path, "conf/test_default_behavior.xml") config = get_layer_config(layer_config, self.archive_config) # Make test dirs make_dir_tree(config["archive_location"]) make_dir_tree(config["wmts_gc_path"]) make_dir_tree(config["twms_gc_path"]) make_dir_tree(config["legend_location"]) # Copy colormaps colormap_location = config["colormap_locations"][0].firstChild.nodeValue colormap = config["colormaps"][0].firstChild.nodeValue make_dir_tree(colormap_location) copy(os.path.join(self.testfiles_path, "conf/" + colormap), colormap_location) # Run layer config tool cmd = "oe_configure_layer -l {0} -a {1} -c {2} -p {3} -m {4}".format( self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config ) run_command(cmd) # Get all the test results before running the assertions. We do this because a failure ends the test and makes it impossible to clean up wmts_cache_xml = os.path.join(config["archive_basepath"], "cache_all_wmts.xml") wmts_cache_file = os.path.join(config["archive_basepath"], "cache_all_wmts.config") wmts_cache = os.path.isfile(wmts_cache_file) wmts_gc_file = os.path.join(config["wmts_gc_path"], "getCapabilities.xml") wmts_gc = os.path.isfile(os.path.join(config["wmts_gc_path"], "getCapabilities.xml")) wmts_staging_file = os.path.join(config["wmts_staging_location"], config["prefix"] + ".xml") wmts_staging = os.path.isfile(wmts_staging_file) twms_cache_xml = os.path.join(config["archive_basepath"], "cache_all_twms.xml") twms_cache_file = os.path.join(config["archive_basepath"], "cache_all_twms.config") twms_cache = os.path.isfile(twms_cache_file) twms_gc_file = os.path.join(config["twms_gc_path"], "getCapabilities.xml") twms_gc = os.path.isfile(twms_gc_file) twms_ts_file = os.path.join(config["twms_gc_path"], "getTileService.xml") twms_ts = os.path.isfile(twms_ts_file) twms_staging_file = os.path.join(config["twms_staging_location"], config["prefix"] + "_gc.xml") twms_staging = os.path.isfile(twms_staging_file) twms_staging_gts_file = os.path.join(config["twms_staging_location"], config["prefix"] + "_gts.xml") twms_staging_gts = os.path.exists(twms_staging_gts_file) twms_staging_mrf_file = os.path.join(config["twms_staging_location"], config["prefix"] + ".mrf") twms_staging_mrf = os.path.exists(twms_staging_mrf_file) self.assertTrue(wmts_cache, "Default layer config test -- cache_all_wmts.config not created") self.assertTrue(wmts_gc, "Default layer config test -- WMTS getCapabilities.xml does not exist") self.assertTrue( wmts_staging, "Default layer config test -- staging file " + wmts_staging_file + " does not exist in WMTS staging area", ) self.assertTrue(twms_cache, "Default layer config test -- cache_all_twms.config does not exist") self.assertTrue(twms_gc, "Default layer config test -- TWMS getCapabilities.xml does not exist") self.assertTrue(twms_ts, "Default layer config test -- TWMS getTileService.xml does not exist") self.assertTrue( twms_staging, "Default layer config test -- staging file " + twms_staging_file + " does not exist in TWMS staging area", ) self.assertTrue( twms_staging_gts, "Default layer config test -- staging file " + twms_staging_gts_file + " does not exist in TWMS staging area", ) self.assertTrue( twms_staging_mrf, "Default layer config test -- staging file " + twms_staging_mrf_file + " does not exist in TWMS staging area", ) rmtree(config["legend_location"]) rmtree(colormap_location) # String searches in the GC and config filenames search_string = "<ows:Identifier>" + config["identifier"] + "</ows:Identifier>" contains_layer = find_string(wmts_gc_file, search_string) os.remove(wmts_gc_file) os.remove(wmts_cache_xml) self.assertTrue(contains_layer, "Default layer config test -- WMTS GetCapabilities does not contain layer") # Unicode weirdness in the binary configs necessitates running str() on the search strings search_string = str( "SERVICE=WMTS&REQUEST=GetTile&VERSION=1.0.0&LAYER=" + config["identifier"] + "&STYLE=(default)?&TILEMATRIXSET=EPSG3413_500m&TILEMATRIX=[0-9]*&TILEROW=[0-9]*&TILECOL=[0-9]*&FORMAT=image%2Fjpeg" ) contains_layer = find_string(wmts_cache_file, search_string) os.remove(wmts_cache_file) os.remove(twms_cache_xml) self.assertTrue(contains_layer, "Default layer config test -- WMTS cache configuration does not contain layer") search_string = "<Name>" + config["identifier"] + "</Name>" contains_layer = find_string(twms_gc_file, search_string) os.remove(twms_gc_file) self.assertTrue(contains_layer, "Default layer config test -- TWMS GetCapabilities does not contain layer") search_string = "<Name>" + config["tiled_group_name"] + "</Name>" contains_layer = find_string(twms_ts_file, search_string) os.remove(twms_ts_file) self.assertTrue(contains_layer, "Default layer config test -- GetTileService does not contain layer") search_string = str( "request=GetMap&layers=" + config["prefix"] + "&srs=EPSG:3413&format=image%2Fjpeg&styles=&width=512&height=512&bbox=[-,\.0-9+Ee]" ) contains_layer = find_string(twms_cache_file, search_string) os.remove(twms_cache_file) self.assertTrue(contains_layer, "Default layer config test -- TWMS cache configuration does not contain layer") rmtree(config["archive_location"]) rmtree(config["wmts_gc_path"]) rmtree(config["wmts_staging_location"]) rmtree(config["twms_staging_location"])
def test_legacy_subdaily_intermittent(self): """ Checks that layer config tool is correctly detecting the period and interval of subdaily layers that have the datetime in their filenames as opposed to the z-index. """ if DEBUG: print "\nTESTING LEGACY SUBDAILY INTERMITTENT PERIOD DETECTION..." layer_config = os.path.join(self.testfiles_path, "conf/test_subdaily_detect.xml") start_datetime = datetime.datetime(2014, 6, 1, 12) config = get_layer_config(layer_config, self.archive_config) # Set the various period units and lengths we'll be testing test_periods = (("hours", 1), ("hours", 5), ("minutes", 1), ("minutes", 5), ("seconds", 1), ("seconds", 5)) # Test intermittent periods for period_unit, period_length in test_periods: for year_dir in (True, False): # Make the GC dirs make_dir_tree(config["wmts_gc_path"]) make_dir_tree(config["twms_gc_path"]) make_dir_tree(config["wmts_staging_location"]) make_dir_tree(config["twms_staging_location"]) test_intervals = testutils.create_intermittent_period_test_files( config["archive_location"], period_unit, period_length, 5, start_datetime, prefix=config["prefix"], make_year_dirs=year_dir, ) # Create debug output message (flatten array w/ itertools) if DEBUG: print "\nTesting with {0} {1} periods".format(period_length, period_unit) print "Creating legacy subdaily files with dates: " for date in itertools.chain.from_iterable(test_intervals): print date.isoformat() # Run layer config command cmd = "oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}".format( self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config, ) run_command(cmd, ignore_warnings=True) # Check to see if proper period in GetCapabilities wmts_gc_file = os.path.join(config["wmts_gc_path"], "getCapabilities.xml") twms_gc_file = os.path.join(config["twms_gc_path"], "getCapabilities.xml") search_strings = [] for interval in test_intervals: time_string = testutils.get_time_string(interval[0], interval[-1], config) search_string = ( "<Value>" + time_string + "/PT{0}{1}</Value>".format(period_length, period_unit[0].upper()) ) search_strings.append(search_string) if DEBUG: print "\n" + "Searching for strings in GetCapabilities: " for string in search_strings: print string # Check to see if string exists in the GC files error = "{0} {1} continuous subdaily legacy period detection failed -- not found in WMTS GetCapabilities".format( period_length, period_unit ) check_result = all(string for string in search_strings if find_string(wmts_gc_file, string)) # Cleanup rmtree(config["wmts_gc_path"]) rmtree(config["wmts_staging_location"]) rmtree(config["twms_staging_location"]) rmtree(config["archive_location"]) # Check result self.assertTrue(check_result, error)
def test_intermittent_zlevel_period_detection(self): """ Checks that the start and end periods of a z-level file are being correctly detected. """ if DEBUG: print "\nTESTING INTERMITTENT Z-LEVEL PERIOD DETECTION..." layer_config = os.path.join(self.testfiles_path, "conf/test_zindex_detect.xml") test_periods = (("minutes", 1), ("minutes", 5), ("hours", 1)) start_datetime = datetime.datetime(2014, 6, 1, 12, 0, 0) start_datestring = str(start_datetime.year) + str(start_datetime.timetuple().tm_yday).zfill(3) config = get_layer_config(layer_config, self.archive_config) for period_unit, period_length in test_periods: # Make archive location dir archive_location = os.path.join(config["archive_location"], str(start_datetime.year)) make_dir_tree(archive_location) # Make temp GC and archive directories and dummy MRF make_dir_tree(config["wmts_gc_path"]) make_dir_tree(config["twms_gc_path"]) make_dir_tree(config["wmts_staging_location"]) make_dir_tree(config["twms_staging_location"]) dummy_mrf = os.path.join(archive_location, config["prefix"] + start_datestring + "_.mrf") open(dummy_mrf, "a").close() # Create a ZDB file for seeding with the dates we're looking for zdb_path = os.path.join(archive_location, config["prefix"] + start_datestring + "_.zdb") conn = sqlite3.connect(zdb_path) # Create ZINDEX table, generate test dates, and populate ZDB file conn.execute("CREATE TABLE ZINDEX(z INTEGER PRIMARY KEY AUTOINCREMENT, key_str TEXT);") test_intervals = testutils.create_intermittent_period_test_files( config["archive_location"], period_unit, period_length, 5, start_datetime, prefix=config["prefix"], no_files=True, ) # Create debug output message (flatten array w/ itertools) if DEBUG: print "Testing with {0} {1} periods".format(period_length, period_unit) print "Creating ZDB with dates: " for date in itertools.chain.from_iterable(test_intervals): print date.isoformat() # Populate the dates in the ZDB (flatten array w/ itertools) for i, date in enumerate(itertools.chain.from_iterable(test_intervals)): z_key = date.strftime("%Y%m%d%H%M%S") sql = "INSERT INTO ZINDEX(z, key_str) VALUES ({0}, {1})".format(i, z_key) conn.execute(sql) conn.commit() # Close ZDB and run layer config conn.close() # Run layer config command cmd = "oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}".format( self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config, ) run_command(cmd, ignore_warnings=True) # Check to see if proper period in GetCapabilities wmts_gc_file = os.path.join(config["wmts_gc_path"], "getCapabilities.xml") twms_gc_file = os.path.join(config["twms_gc_path"], "getCapabilities.xml") # Build GC search string search_strings = [] for interval in test_intervals: search_string = "<Value>" + interval[0].isoformat() + "Z/" + interval[-1].isoformat() + "Z</Value>" search_strings.append(search_string) if DEBUG: print "\n" + "Searching for strings in GetCapabilities: " for string in search_strings: print string # Check to see if string exists in the GC files error = "{0} {1} intermittent z-level period detection failed -- not found in WMTS GetCapabilities".format( period_length, period_unit ) check_result = all(string for string in search_strings if find_string(wmts_gc_file, string)) # Cleanup conn.close() rmtree(config["wmts_gc_path"]) rmtree(config["wmts_staging_location"]) rmtree(config["twms_staging_location"]) rmtree(config["archive_location"]) # Check result self.assertTrue(check_result, error)
def test_intermittent_period_detection(self): if DEBUG: print "\nTESTING INTERMITTENT PERIOD DETECTION..." layer_config = os.path.join(self.testfiles_path, "conf/test_period_detection.xml") # Pick the start time for the dates that will be generated start_datetime = datetime.datetime(2014, 6, 1) # Set the various period units and lengths we'll be testing test_periods = (("days", 1), ("days", 5), ("months", 1), ("years", 1)) config = get_layer_config(layer_config, self.archive_config) for period_unit, period_length in test_periods: # We test detection with both year and non-year directory setups for year_dir in (True, False): make_dir_tree(config["wmts_gc_path"]) make_dir_tree(config["twms_gc_path"]) make_dir_tree(config["wmts_staging_location"]) make_dir_tree(config["twms_staging_location"]) # Generate the empty test files test_intervals = testutils.create_intermittent_period_test_files( config["archive_location"], period_unit, period_length, 5, start_datetime, prefix=config["prefix"], make_year_dirs=year_dir, ) # Run layer config command for daily test days cmd = "oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}".format( self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config, ) run_command(cmd, ignore_warnings=True) # Check to see if proper period in GetCapabilities wmts_gc_file = os.path.join(config["wmts_gc_path"], "getCapabilities.xml") twms_gc_file = os.path.join(config["twms_gc_path"], "getCapabilities.xml") # Build a list GC search strings search_strings = [] for interval in test_intervals: search_string = ( "<Value>" + interval[0].isoformat() + "/" + interval[-1].isoformat() + "/P{0}{1}</Value>".format(period_length, period_unit[0].upper()) ) search_strings.append(search_string) # Create debug output message if DEBUG: print "\n" + "Creating dates: " dates = [date for date in interval for interval in test_intervals] for date in dates: print date.isoformat() print "\n" + "Searching for string(s) in GetCapabilities: " for string in search_strings: print search_string # Check to see if string exists in the GC files wmts_error = "{0} {1} intermittent period detection failed -- not found in WMTS GetCapabilities".format( period_length, period_unit ) # twms_error = "{0} {1} period detection failed -- not found in TMWS GetCapabilities".format(period_length, period_unit) # self.assertTrue(find_string(twms_gc_file, search_string), twms_error) search_result = all(string for string in search_strings if find_string(wmts_gc_file, string)) # Cleanup -- make sure to get rid of staging files rmtree(config["wmts_gc_path"]) rmtree(config["wmts_staging_location"]) rmtree(config["twms_staging_location"]) rmtree(config["archive_location"]) # Check result self.assertTrue(search_result, wmts_error)
def test_legacy_subdaily_continuous(self): """ Checks that layer config tool is correctly detecting the period and interval of subdaily layers that have the datetime in their filenames as opposed to the z-index. """ if DEBUG: print '\nTESTING LEGACY SUBDAILY CONTINUOUS PERIOD DETECTION...' layer_config = os.path.join(self.testfiles_path, 'conf/test_subdaily_detect.xml') start_datetime = datetime.datetime(2014, 6, 1, 12) config = get_layer_config(layer_config, self.archive_config) # Set the various period units and lengths we'll be testing test_periods = (('hours', 1), ('hours', 5), ('minutes', 1), ('minutes', 5), ('seconds', 1), ('seconds', 5)) # Test continuous periods for period_unit, period_length in test_periods: for year_dir in (True, False): # Make the GC dirs make_dir_tree(config['wmts_gc_path']) make_dir_tree(config['twms_gc_path']) make_dir_tree(config['wmts_staging_location']) make_dir_tree(config['twms_staging_location']) # Create a continuous set of period files for each time interval test_dates = testutils.create_continuous_period_test_files(config['archive_location'], period_unit, period_length, 4, start_datetime, prefix=config['prefix'], make_year_dirs=year_dir) # Create debug output message if DEBUG: print '\nTesting with {0} {1} periods'.format(period_length, period_unit) print 'Creating legacy subdaily files with dates: ' for date in test_dates: print date.isoformat() # Run layer config command cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format(self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config) run_command(cmd, ignore_warnings=True) # Check to see if proper period in GetCapabilities wmts_gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml') twms_gc_file = os.path.join(config['twms_gc_path'], 'getCapabilities.xml') # Build search strings time_string = testutils.get_time_string(start_datetime, test_dates[-1], config) search_string = "<Value>" + time_string + "/PT{0}{1}</Value>".format(period_length, period_unit[0].upper()) search_result = find_string(wmts_gc_file, search_string) if DEBUG: print '\n' + 'Searching for string in GetCapabilities: ' + search_string # Cleanup rmtree(config['wmts_gc_path']) rmtree(config['wmts_staging_location']) rmtree(config['twms_staging_location']) rmtree(config['archive_location']) # Check result error = "{0} {1} continuous subdaily legacy period detection failed -- not found in WMTS GetCapabilities".format(period_length, period_unit) self.assertTrue(search_result, error)
def test_continuous_zlevel_period_detection(self): """ Checks that the start and end periods of a z-level file are being correctly detected. """ if DEBUG: print '\nTESTING CONTINUOUS Z-LEVEL PERIOD DETECTION...' layer_config = os.path.join(self.testfiles_path, 'conf/test_zindex_detect.xml') test_periods = (('minutes', 1), ('minutes', 5), ('hours', 1)) start_datetime = datetime.datetime(2014, 6, 1, 12, 0, 0) start_datestring = str(start_datetime.year) + str(start_datetime.timetuple().tm_yday).zfill(3) config = get_layer_config(layer_config, self.archive_config) for period_unit, period_length in test_periods: # Make archive location dir archive_location = os.path.join(config['archive_location'], str(start_datetime.year)) make_dir_tree(archive_location) # Make temp GC and archive directories and dummy MRF make_dir_tree(config['wmts_gc_path']) make_dir_tree(config['twms_gc_path']) make_dir_tree(config['wmts_staging_location']) make_dir_tree(config['twms_staging_location']) dummy_mrf = os.path.join(archive_location, config['prefix'] + start_datestring + '_.mrf') open(dummy_mrf, 'a').close() # Create a ZDB file for seeding with the dates we're looking for zdb_path = os.path.join(archive_location, config['prefix'] + start_datestring + '_.zdb') conn = sqlite3.connect(zdb_path) # Create ZINDEX table, generate test dates, and populate ZDB file conn.execute('CREATE TABLE ZINDEX(z INTEGER PRIMARY KEY AUTOINCREMENT, key_str TEXT);') test_dates = testutils.create_continuous_period_test_files( config['archive_location'], period_unit, period_length, 5, start_datetime, prefix=config['prefix'], no_files=True) # Create debug output message if DEBUG: print 'Testing with {0} {1} periods'.format(period_length, period_unit) print 'Creating ZDB with dates: ' for date in test_dates: print date.isoformat() # Populate the dates in the ZDB for i, date in enumerate(test_dates): z_key = date.strftime('%Y%m%d%H%M%S') sql = 'INSERT INTO ZINDEX(z, key_str) VALUES ({0}, {1})'.format(i, z_key) conn.execute(sql) conn.commit() # Close ZDB and run layer config conn.close() cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format(self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config) run_command(cmd, ignore_warnings=True) # Check to see if proper period in GetCapabilities wmts_gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml') twms_gc_file = os.path.join(config['twms_gc_path'], 'getCapabilities.xml') # Build GC search string search_string = "<Value>" + test_dates[0].isoformat() + "Z/" + test_dates[-1].isoformat() + "Z</Value>" if DEBUG: print '\n' + 'Searching for string in GetCapabilities: ' + search_string # Check to see if string exists in the GC files error = "{0} {1} continuous z-level period detection failed -- not found in WMTS GetCapabilities".format(period_length, period_unit) check_result = find_string(wmts_gc_file, search_string) # Cleanup conn.close() rmtree(config['wmts_gc_path']) rmtree(config['wmts_staging_location']) rmtree(config['twms_staging_location']) rmtree(config['archive_location']) # Check result self.assertTrue(check_result, error)