def test_continuous_period_detection(self): if DEBUG: print '\nTESTING CONTINUOUS PERIOD DETECTION...' layer_config = os.path.join(self.testfiles_path, 'conf/test_period_detection.xml') # Pick the start time for the dates that will be generated start_datetime = datetime.datetime(2014, 6, 1) # Set the various period units and lengths we'll be testing test_periods = (('days', 1), ('days', 5), ('months', 1), ('years', 1)) config = get_layer_config(layer_config, self.archive_config) for period_unit, period_length in test_periods: # We test detection with both year and non-year directory setups for year_dir in (True, False): make_dir_tree(config['wmts_gc_path']) make_dir_tree(config['twms_gc_path']) make_dir_tree(config['wmts_staging_location']) make_dir_tree(config['twms_staging_location']) # Generate the empty test files test_dates = testutils.create_continuous_period_test_files( config['archive_location'], period_unit, period_length, 5, start_datetime, prefix=config['prefix'], make_year_dirs=year_dir) # Run layer config command for daily test days cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format(self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config) run_command(cmd, ignore_warnings=True) # Check to see if proper period in GetCapabilities wmts_gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml') twms_gc_file = os.path.join(config['twms_gc_path'], 'getCapabilities.xml') # Build GC search string search_string = "<Value>" + test_dates[0].date().isoformat() + "/" + test_dates[-1].date().isoformat() + "/P{0}{1}</Value>".format(period_length, period_unit[0].upper()) # Create debug output message if DEBUG: print 'Testing with {0} {1} periods'.format(period_length, period_unit) print 'Creating dates: ' for date in test_dates: print date.isoformat() print '\n' + 'Searching for string in GetCapabilities: ' + search_string # Check to see if string exists in the GC files wmts_error = "{0} {1} continuous period detection failed -- not found in WMTS GetCapabilities".format(period_length, period_unit) test_result = find_string(wmts_gc_file, search_string) # twms_error = "{0} {1} period detection failed -- not found in TMWS GetCapabilities".format(period_length, period_unit) # self.assertTrue(find_string(twms_gc_file, search_string), twms_error) # Cleanup -- make sure to get rid of staging files rmtree(config['wmts_gc_path']) rmtree(config['wmts_staging_location']) rmtree(config['twms_staging_location']) rmtree(config['archive_location']) # Check result self.assertTrue(test_result, wmts_error)
def test_legacy_subdaily_continuous(self): """ Checks that layer config tool is correctly detecting the period and interval of subdaily layers that have the datetime in their filenames as opposed to the z-index. """ if DEBUG: print '\nTESTING LEGACY SUBDAILY CONTINUOUS PERIOD DETECTION...' layer_config = os.path.join(self.testfiles_path, 'conf/test_subdaily_detect.xml') start_datetime = datetime.datetime(2014, 6, 1, 12) config = get_layer_config(layer_config, self.archive_config) # Set the various period units and lengths we'll be testing test_periods = (('hours', 1), ('hours', 5), ('minutes', 1), ('minutes', 5), ('seconds', 1), ('seconds', 5)) # Test continuous periods for period_unit, period_length in test_periods: for year_dir in (True, False): # Make the GC dirs make_dir_tree(config['wmts_gc_path']) make_dir_tree(config['twms_gc_path']) make_dir_tree(config['wmts_staging_location']) make_dir_tree(config['twms_staging_location']) # Create a continuous set of period files for each time interval test_dates = testutils.create_continuous_period_test_files( config['archive_location'], period_unit, period_length, 4, start_datetime, prefix=config['prefix'], make_year_dirs=year_dir) # Create debug output message if DEBUG: print '\nTesting with {0} {1} periods'.format( period_length, period_unit) print 'Creating legacy subdaily files with dates: ' for date in test_dates: print date.isoformat() # Run layer config command cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format( self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config) run_command(cmd, ignore_warnings=True) # Check to see if proper period in GetCapabilities wmts_gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml') twms_gc_file = os.path.join(config['twms_gc_path'], 'getCapabilities.xml') # Build search strings time_string = testutils.get_time_string( start_datetime, test_dates[-1], config) search_string = "<Value>" + time_string + "/PT{0}{1}</Value>".format( period_length, period_unit[0].upper()) search_result = find_string(wmts_gc_file, search_string) if DEBUG: print '\n' + 'Searching for string in GetCapabilities: ' + search_string # Cleanup rmtree(config['wmts_gc_path']) rmtree(config['wmts_staging_location']) rmtree(config['twms_staging_location']) rmtree(config['archive_location']) # Check result error = "{0} {1} continuous subdaily legacy period detection failed -- not found in WMTS GetCapabilities".format( period_length, period_unit) self.assertTrue(search_result, error)
def test_continuous_zlevel_period_detection(self): """ Checks that the start and end periods of a z-level file are being correctly detected. """ if DEBUG: print '\nTESTING CONTINUOUS Z-LEVEL PERIOD DETECTION...' layer_config = os.path.join(self.testfiles_path, 'conf/test_zindex_detect.xml') test_periods = (('minutes', 1), ('minutes', 5), ('hours', 1)) start_datetime = datetime.datetime(2014, 6, 1, 12, 0, 0) start_datestring = str(start_datetime.year) + str( start_datetime.timetuple().tm_yday).zfill(3) config = get_layer_config(layer_config, self.archive_config) for period_unit, period_length in test_periods: # Make archive location dir archive_location = os.path.join(config['archive_location'], str(start_datetime.year)) make_dir_tree(archive_location) # Make temp GC and archive directories and dummy MRF make_dir_tree(config['wmts_gc_path']) make_dir_tree(config['twms_gc_path']) make_dir_tree(config['wmts_staging_location']) make_dir_tree(config['twms_staging_location']) dummy_mrf = os.path.join( archive_location, config['prefix'] + start_datestring + '_.mrf') open(dummy_mrf, 'a').close() # Create a ZDB file for seeding with the dates we're looking for zdb_path = os.path.join( archive_location, config['prefix'] + start_datestring + '_.zdb') conn = sqlite3.connect(zdb_path) # Create ZINDEX table, generate test dates, and populate ZDB file conn.execute( 'CREATE TABLE ZINDEX(z INTEGER PRIMARY KEY AUTOINCREMENT, key_str TEXT);' ) test_dates = testutils.create_continuous_period_test_files( config['archive_location'], period_unit, period_length, 5, start_datetime, prefix=config['prefix'], no_files=True) # Create debug output message if DEBUG: print 'Testing with {0} {1} periods'.format( period_length, period_unit) print 'Creating ZDB with dates: ' for date in test_dates: print date.isoformat() # Populate the dates in the ZDB for i, date in enumerate(test_dates): z_key = date.strftime('%Y%m%d%H%M%S') sql = 'INSERT INTO ZINDEX(z, key_str) VALUES ({0}, {1})'.format( i, z_key) conn.execute(sql) conn.commit() # Close ZDB and run layer config conn.close() cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format( self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config) run_command(cmd, ignore_warnings=True) # Check to see if proper period in GetCapabilities wmts_gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml') twms_gc_file = os.path.join(config['twms_gc_path'], 'getCapabilities.xml') # Build GC search string search_string = "<Value>" + test_dates[0].isoformat( ) + "Z/" + test_dates[-1].isoformat() + "Z</Value>" if DEBUG: print '\n' + 'Searching for string in GetCapabilities: ' + search_string # Check to see if string exists in the GC files error = "{0} {1} continuous z-level period detection failed -- not found in WMTS GetCapabilities".format( period_length, period_unit) check_result = find_string(wmts_gc_file, search_string) # Cleanup conn.close() rmtree(config['wmts_gc_path']) rmtree(config['wmts_staging_location']) rmtree(config['twms_staging_location']) rmtree(config['archive_location']) # Check result self.assertTrue(check_result, error)
def test_continuous_period_detection(self): if DEBUG: print '\nTESTING CONTINUOUS PERIOD DETECTION...' layer_config = os.path.join(self.testfiles_path, 'conf/test_period_detection.xml') # Pick the start time for the dates that will be generated start_datetime = datetime.datetime(2014, 6, 1) # Set the various period units and lengths we'll be testing test_periods = (('days', 1), ('days', 5), ('months', 1), ('years', 1)) config = get_layer_config(layer_config, self.archive_config) for period_unit, period_length in test_periods: # We test detection with both year and non-year directory setups for year_dir in (True, False): make_dir_tree(config['wmts_gc_path']) make_dir_tree(config['twms_gc_path']) make_dir_tree(config['wmts_staging_location']) make_dir_tree(config['twms_staging_location']) # Generate the empty test files test_dates = testutils.create_continuous_period_test_files( config['archive_location'], period_unit, period_length, 5, start_datetime, prefix=config['prefix'], make_year_dirs=year_dir) # Run layer config command for daily test days cmd = 'oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}'.format( self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config) run_command(cmd, ignore_warnings=True) # Check to see if proper period in GetCapabilities wmts_gc_file = os.path.join(config['wmts_gc_path'], 'getCapabilities.xml') twms_gc_file = os.path.join(config['twms_gc_path'], 'getCapabilities.xml') # Build GC search string search_string = "<Value>" + test_dates[0].date().isoformat( ) + "/" + test_dates[-1].date( ).isoformat() + "/P{0}{1}</Value>".format( period_length, period_unit[0].upper()) # Create debug output message if DEBUG: print 'Testing with {0} {1} periods'.format( period_length, period_unit) print 'Creating dates: ' for date in test_dates: print date.isoformat() print '\n' + 'Searching for string in GetCapabilities: ' + search_string # Check to see if string exists in the GC files wmts_error = "{0} {1} continuous period detection failed -- not found in WMTS GetCapabilities".format( period_length, period_unit) test_result = find_string(wmts_gc_file, search_string) # twms_error = "{0} {1} period detection failed -- not found in TMWS GetCapabilities".format(period_length, period_unit) # self.assertTrue(find_string(twms_gc_file, search_string), twms_error) # Cleanup -- make sure to get rid of staging files rmtree(config['wmts_gc_path']) rmtree(config['wmts_staging_location']) rmtree(config['twms_staging_location']) rmtree(config['archive_location']) # Check result self.assertTrue(test_result, wmts_error)
def test_legacy_subdaily_continuous(self): """ Checks that layer config tool is correctly detecting the period and interval of subdaily layers that have the datetime in their filenames as opposed to the z-index. """ if DEBUG: print "\nTESTING LEGACY SUBDAILY CONTINUOUS PERIOD DETECTION..." layer_config = os.path.join(self.testfiles_path, "conf/test_subdaily_detect.xml") start_datetime = datetime.datetime(2014, 6, 1, 12) config = get_layer_config(layer_config, self.archive_config) # Set the various period units and lengths we'll be testing test_periods = (("hours", 1), ("hours", 5), ("minutes", 1), ("minutes", 5), ("seconds", 1), ("seconds", 5)) # Test continuous periods for period_unit, period_length in test_periods: for year_dir in (True, False): # Make the GC dirs make_dir_tree(config["wmts_gc_path"]) make_dir_tree(config["twms_gc_path"]) make_dir_tree(config["wmts_staging_location"]) make_dir_tree(config["twms_staging_location"]) # Create a continuous set of period files for each time interval test_dates = testutils.create_continuous_period_test_files( config["archive_location"], period_unit, period_length, 4, start_datetime, prefix=config["prefix"], suffix="_.idx", make_year_dirs=year_dir, ) # Create debug output message if DEBUG: print "\nTesting with {0} {1} periods".format(period_length, period_unit) print "Creating legacy subdaily files with dates: " for date in test_dates: print date.isoformat() # Run layer config command cmd = "oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}".format( self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config, ) run_command(cmd, ignore_warnings=True) # Check to see if proper period in GetCapabilities wmts_gc_file = os.path.join(config["wmts_gc_path"], "getCapabilities.xml") twms_gc_file = os.path.join(config["twms_gc_path"], "getCapabilities.xml") # Build search strings time_string = testutils.get_time_string(start_datetime, test_dates[-1], config) search_string = ( "<Value>" + time_string + "/PT{0}{1}</Value>".format(period_length, period_unit[0].upper()) ) search_result = find_string(wmts_gc_file, search_string) if DEBUG: print "\n" + "Searching for string in GetCapabilities: " + search_string # Cleanup rmtree(config["wmts_gc_path"]) rmtree(config["wmts_staging_location"]) rmtree(config["twms_staging_location"]) rmtree(config["archive_location"]) # Check result error = "{0} {1} continuous subdaily legacy period detection failed -- not found in WMTS GetCapabilities".format( period_length, period_unit ) self.assertTrue(search_result, error)
def test_continuous_zlevel_period_detection(self): """ Checks that the start and end periods of a z-level file are being correctly detected. """ if DEBUG: print "\nTESTING CONTINUOUS Z-LEVEL PERIOD DETECTION..." layer_config = os.path.join(self.testfiles_path, "conf/test_zindex_detect.xml") test_periods = (("minutes", 1), ("minutes", 5), ("hours", 1)) start_datetime = datetime.datetime(2014, 6, 1, 12, 0, 0) start_datestring = str(start_datetime.year) + str(start_datetime.timetuple().tm_yday).zfill(3) config = get_layer_config(layer_config, self.archive_config) for period_unit, period_length in test_periods: # Make archive location dir archive_location = os.path.join(config["archive_location"], str(start_datetime.year)) make_dir_tree(archive_location) # Make temp GC and archive directories and dummy MRF make_dir_tree(config["wmts_gc_path"]) make_dir_tree(config["twms_gc_path"]) make_dir_tree(config["wmts_staging_location"]) make_dir_tree(config["twms_staging_location"]) dummy_mrf = os.path.join(archive_location, config["prefix"] + start_datestring + "_.idx") open(dummy_mrf, "a").close() # Create a ZDB file for seeding with the dates we're looking for zdb_path = os.path.join(archive_location, config["prefix"] + start_datestring + "_.zdb") conn = sqlite3.connect(zdb_path) # Create ZINDEX table, generate test dates, and populate ZDB file conn.execute("CREATE TABLE ZINDEX(z INTEGER PRIMARY KEY AUTOINCREMENT, key_str TEXT);") test_dates = testutils.create_continuous_period_test_files( config["archive_location"], period_unit, period_length, 5, start_datetime, prefix=config["prefix"], no_files=True, ) # Create debug output message if DEBUG: print "Testing with {0} {1} periods".format(period_length, period_unit) print "Creating ZDB with dates: " for date in test_dates: print date.isoformat() # Populate the dates in the ZDB for i, date in enumerate(test_dates): z_key = date.strftime("%Y%m%d%H%M%S") sql = "INSERT INTO ZINDEX(z, key_str) VALUES ({0}, {1})".format(i, z_key) conn.execute(sql) conn.commit() # Close ZDB and run layer config conn.close() cmd = "oe_configure_layer -l {0} -z -e -a {1} -c {2} -p {3} -m {4}".format( self.testfiles_path, self.archive_config, layer_config, self.projection_config, self.tilematrixset_config, ) run_command(cmd, ignore_warnings=True) # Check to see if proper period in GetCapabilities wmts_gc_file = os.path.join(config["wmts_gc_path"], "getCapabilities.xml") twms_gc_file = os.path.join(config["twms_gc_path"], "getCapabilities.xml") # Build GC search string search_string = "<Value>" + test_dates[0].isoformat() + "Z/" + test_dates[-1].isoformat() + "Z</Value>" if DEBUG: print "\n" + "Searching for string in GetCapabilities: " + search_string # Check to see if string exists in the GC files error = "{0} {1} continuous z-level period detection failed -- not found in WMTS GetCapabilities".format( period_length, period_unit ) check_result = find_string(wmts_gc_file, search_string) # Cleanup conn.close() rmtree(config["wmts_gc_path"]) rmtree(config["wmts_staging_location"]) rmtree(config["twms_staging_location"]) rmtree(config["archive_location"]) # Check result self.assertTrue(check_result, error)