def _generate_input_land_cover(self, current_year, base_directory, urbansim_cache_directory, years, output_directory, convert_flt, convert_input): if current_year == years[0]: if not convert_input: return base_directory else: package_dir_path = package().get_package_path() command = os.path.join(package_dir_path, "tools", "lc_convert.py") status = os.system(command + ' %s -i "%s" -o "%s"' % ('input data', base_directory, self.temp_land_cover_dir)) assert(status == 0, "generate input failed") return self.temp_land_cover_dir previous_year = self._get_previous_year(current_year, years) if not convert_flt: logger.start_block("Copy data from %s to temp land cover folder" % urbansim_cache_directory) try: self._copy_invariants_to_temp_land_cover_dir(os.path.join(urbansim_cache_directory, str(previous_year))) finally: logger.end_block() return self.temp_land_cover_dir # package_dir_path = package().get_package_path() # command = os.path.join(package_dir_path, "tools", "lc_convert.py") flt_directory_in = os.path.join(output_directory, str(previous_year)) flt_directory_out = self.temp_land_cover_dir LCCMInputConvert()._convert_lccm_input(flt_directory_in, flt_directory_out) # status = os.system(command + ' %d -i "%s" -o "%s"' % (previous_year, flt_directory_in, flt_directory_out)) # assert(status == 0, "generate input failed") return self.temp_land_cover_dir
def setUp(self): package_dir_path = package().get_package_path() self.base_directory = os.path.join(package_dir_path, "data", "small_test_set_opus", "1991") zipped_cache = os.path.join(package_dir_path, "data", "sample_cache.zip") self.cache_dir = mkdtemp(prefix='tmp_biocomplexity') unzip_files_into_dir(zipped_cache, self.cache_dir) self.urbansim_cache_directory = os.path.join(self.cache_dir, "urbansim_cache", "data", "2006_02_14__11_43")
def _generate_output_flt(self, current_year, urbansim_cache_directory, output_directory, convert_flt): if not convert_flt: return package_dir_path = package().get_package_path() command = os.path.join(package_dir_path, "tools", "lc_convert_to_flt.py") flt_directory_in = os.path.join(urbansim_cache_directory, str(current_year)) flt_directory_out = os.path.join(output_directory, str(current_year)) status = os.system(sys.executable + ' ' + command + ' %d -i "%s" -o "%s"' % (current_year, flt_directory_in, flt_directory_out)) assert(status == 0, "generate output failed")
def __init__(self, **kargs): # Estimator.__init__(self, settings=None, run_land_price_model_before_estimation=False, **kargs) # <-- old __init__ # Estimator.__init__(self, config=None, save_estimation_results=True) # <-- new __init__ doesn't work, but not needed parent_dir_path = package().get_package_parent_path() package_path = OpusPackage().get_path_for_package("biocomplexity") self.storage = StorageFactory().get_storage("tab_storage", storage_location=os.path.join(package_path, "data")) ## 1. directory path of full (4 county spatial extent) dataset flt_directory = os.path.join(parent_dir_path, "biocomplexity", "data", "LCCM_4County") ## 2. select (uncomment) from one the following choices of directory pathes of subsetted sample input data/variables # flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "LCCM_small_test_set_opus") flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_estimation_all") # flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_estimation_all_orig") # flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_suburban_orig") # flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_urban") # flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_urban_orig") ## note - must rename lct-forusewith91sample.Float32 to lct.lf4 if doing 1991-1995 ## note - must rename lct-forusewith95sample.Float32 to lct.lf4 if doing 1995-1999 ## 3. select (uncomment) from one the following choices of land cover data (input data) date pairs (years) # years = [1991, 1995] # years = [1995, 1999] years = [1999, 2002] self.lc1 = LandCoverDataset( in_storage=StorageFactory().get_storage( "flt_storage", storage_location=os.path.join(flt_directory_est, str(years[0])) ), resources=Resources({"lowercase": 1}), ) self.lc2 = LandCoverDataset( in_storage=StorageFactory().get_storage( "flt_storage", storage_location=os.path.join(flt_directory_est, str(years[1])) ), resources=Resources({"lowercase": 1}), ) self.lc1_all = LandCoverDataset( in_storage=StorageFactory().get_storage( "flt_storage", storage_location=os.path.join(flt_directory, str(years[0])) ), resources=Resources({"lowercase": 1}), ) self.lc1_all.flush_dataset() self.lc2_all = LandCoverDataset( in_storage=StorageFactory().get_storage( "flt_storage", storage_location=os.path.join(flt_directory, str(years[1])) ), resources=Resources({"lowercase": 1}), ) self.lc2_all.flush_dataset()
def __init__(self, **kargs): # Estimator.__init__(self, settings=None, run_land_price_model_before_estimation=False, **kargs) # <-- old __init__ # Estimator.__init__(self, config=None, save_estimation_results=True) # <-- new __init__ doesn't work, but not needed parent_dir_path = package().get_package_parent_path() package_path = OpusPackage().get_path_for_package("biocomplexity") self.storage = StorageFactory().get_storage( 'tab_storage', storage_location=os.path.join(package_path, 'data')) ## 1. directory path of full (4 county spatial extent) dataset flt_directory = os.path.join(parent_dir_path, "biocomplexity", "data", "LCCM_4County") ## 2. select (uncomment) from one the following choices of directory pathes of subsetted sample input data/variables # flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "LCCM_small_test_set_opus") flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_estimation_all") # flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_estimation_all_orig") # flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_suburban_orig") # flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_urban") # flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_urban_orig") ## note - must rename lct-forusewith91sample.Float32 to lct.lf4 if doing 1991-1995 ## note - must rename lct-forusewith95sample.Float32 to lct.lf4 if doing 1995-1999 ## 3. select (uncomment) from one the following choices of land cover data (input data) date pairs (years) # years = [1991, 1995] years = [1995, 1999] # years = [1999, 2002] self.lc1 = LandCoverDataset(in_storage=StorageFactory().get_storage( "flt_storage", storage_location=os.path.join(flt_directory_est, str(years[0]))), resources=Resources({"lowercase": 1})) self.lc2 = LandCoverDataset(in_storage=StorageFactory().get_storage( "flt_storage", storage_location=os.path.join(flt_directory_est, str(years[1]))), resources=Resources({"lowercase": 1})) self.lc1_all = LandCoverDataset( in_storage=StorageFactory().get_storage( "flt_storage", storage_location=os.path.join(flt_directory, str(years[0]))), resources=Resources({"lowercase": 1})) self.lc1_all.flush_dataset() self.lc2_all = LandCoverDataset( in_storage=StorageFactory().get_storage( "flt_storage", storage_location=os.path.join(flt_directory, str(years[1]))), resources=Resources({"lowercase": 1})) self.lc2_all.flush_dataset()
#os.remove(file_name) content = fromfile(file_name, dtype=float32) # reads in existing file put(content.ravel(), flat_indices.astype(int32)[offset:], data[offset:]) # incoprorates existing data using offset (chunk) size else: content = self.nodata_values*ones(shape=(self.nrows, self.ncols)).astype(float32) put(content.ravel(), flat_indices.astype(int32), data) # <-- coerced flat_indices as integer32 type - changed on 26 may 09 by mm content.tofile(file_name) def _create_header(self, file_name): content = \ """ ncols %d nrows %d xllcorner %f yllcorner %f cellsize %d NODATA_value %d byteorder LSBFIRST """ % (self.ncols, self.nrows, self.xllcorner, self.yllcorner, self.cellsize, self.nodata_values) f = open(file_name, mode="w") f.write(content) f.close() if __name__ == "__main__": parent_dir_path = package().get_package_parent_path() flt_directory_in = os.path.join(parent_dir_path, "biocomplexity", "data", "LCCM_4County_converted", "1991") flt_directory_out = os.path.join(parent_dir_path, "biocomplexity", "data", "LCCM_4County", "2002") current_year = 1991 ConvertToFloat()._create_flt_file(current_year, flt_directory_in, flt_directory_out)
from biocomplexity.datasets.land_cover_dataset import LandCoverDataset from biocomplexity.examples.lccm_runner_sample import LccmConfiguration from biocomplexity.opus_package_info import package from opus_core.datasets.dataset import DatasetSubset from opus_core.storage_factory import StorageFactory from opus_core.logger import logger from numpy import where from optparse import OptionParser import os import shutil import sys if __name__ == "__main__": parent_dir_path = package().get_package_parent_path() # directory of the data # flt_directory_in = os.path.join(parent_dir_path, "biocomplexity", "data", "LCCM_small_test_set_opus", "1995") flt_directory_in = os.path.join(parent_dir_path, "biocomplexity", "data", "LCCM_4County", "2002") flt_directory_out = flt_directory_in # set here the type of the attributes valuetypes = {} test_flag = False if len(sys.argv) >= 2: parser = OptionParser() parser.add_option("-i", "--input", dest="input", action="store", type="string") parser.add_option("-o", "--output", dest="output", action="store", type="string") parser.add_option("-t", "--test", dest="test_flag", action="store_true", default="false")
def do_test_on_expected_data(self, input_variables_list, input_resources=None, element_atol=None, sum_atol=None): from biocomplexity.datasets.land_cover_dataset import LandCoverDataset from biocomplexity.tests.utils.land_cover_tests_utils import make_input_data import os package_dir_path = package().get_package_path() flt_directory = os.path.join(package_dir_path, "data", "small_test_set_opus", "1995") #flt_directory = r"C:\eclipse\LCCM_small_test_set_converted\1991" expected_lc = LandCoverDataset(in_storage = StorageFactory().get_storage('flt_storage', storage_location = flt_directory)) expected_lc.load_dataset() temp_dir = make_input_data(flt_directory, input_variables_list) try: lc = LandCoverDataset(in_storage = StorageFactory().get_storage( 'flt_storage', storage_location=temp_dir), out_storage = StorageFactory().get_storage('flt_storage', storage_location = r"c:/tmp")) lc.load_dataset() dataset_pool = DatasetPool( package_order=['biocomplexity'], storage=StorageFactory().get_storage('flt_storage', storage_location=temp_dir)) dataset_pool._add_dataset('land_cover', lc) lc.compute_variables(self.variable_name, resources=input_resources, dataset_pool=dataset_pool) #lc.write_dataset(attributes='*') lc_values = lc.get_attribute(self.variable_name) expected_values = expected_lc.get_attribute(self.variable_name) if sum_atol is None: sum_atol = 1e-8 if element_atol is None: element_atol = 1e-8 if (not ma.allclose(lc_values, expected_values, atol=element_atol)): logger.log_status("comparision using element-atol=%f, sum-atol=%f" % (element_atol, sum_atol)) logger.log_status(" computed expected"); logger.log_status("sum: ", lc_values.sum(), expected_values.sum()) logger.log_status("max: ", max(lc_values), max(expected_values)) logger.log_status("min: ", min(lc_values), min(expected_values)) c1 = 0 c2 = 0 for (i,j) in zip(lc_values, expected_values): if i != 0: c1 = c1 + 1 if j != 0: c2 = c2 + 1 logger.log_status("# non-zeros values: ", c1, c2) logger.log_status("max distance between 2 elements: %f" % self._max_distance_between_two_elements(lc_values,expected_values)) logger.log_status(lc_values, expected_values) count = 0 total = 0 for (i,j) in zip(lc_values, expected_values): if i != j: count = count + 1 total = total + 1 logger.log_status("# different elements = %d, over %d, with a %f percentage" \ % (count, total, count*1.0/total)) #self.assert_(ma.allclose(lc_values.sum(), expected_values.sum(), atol=sum_atol)) self.assert_(ma.allclose(lc_values, expected_values, atol=element_atol)) finally: if os.path.exists(temp_dir): rmtree(temp_dir)
def do_test_on_expected_data(self, input_variables_list, input_resources=None, element_atol=None, sum_atol=None): from biocomplexity.datasets.land_cover_dataset import LandCoverDataset from biocomplexity.tests.utils.land_cover_tests_utils import make_input_data import os package_dir_path = package().get_package_path() flt_directory = os.path.join(package_dir_path, "data", "small_test_set_opus", "1995") #flt_directory = r"C:\eclipse\LCCM_small_test_set_converted\1991" expected_lc = LandCoverDataset(in_storage=StorageFactory().get_storage( 'flt_storage', storage_location=flt_directory)) expected_lc.load_dataset() temp_dir = make_input_data(flt_directory, input_variables_list) try: lc = LandCoverDataset(in_storage=StorageFactory().get_storage( 'flt_storage', storage_location=temp_dir), out_storage=StorageFactory().get_storage( 'flt_storage', storage_location=r"c:/tmp")) lc.load_dataset() dataset_pool = DatasetPool(package_order=['biocomplexity'], storage=StorageFactory().get_storage( 'flt_storage', storage_location=temp_dir)) dataset_pool._add_dataset('land_cover', lc) lc.compute_variables(self.variable_name, resources=input_resources, dataset_pool=dataset_pool) #lc.write_dataset(attributes='*') lc_values = lc.get_attribute(self.variable_name) expected_values = expected_lc.get_attribute(self.variable_name) if sum_atol is None: sum_atol = 1e-8 if element_atol is None: element_atol = 1e-8 if (not ma.allclose(lc_values, expected_values, atol=element_atol)): logger.log_status( "comparision using element-atol=%f, sum-atol=%f" % (element_atol, sum_atol)) logger.log_status(" computed expected") logger.log_status("sum: ", lc_values.sum(), expected_values.sum()) logger.log_status("max: ", max(lc_values), max(expected_values)) logger.log_status("min: ", min(lc_values), min(expected_values)) c1 = 0 c2 = 0 for (i, j) in zip(lc_values, expected_values): if i != 0: c1 = c1 + 1 if j != 0: c2 = c2 + 1 logger.log_status("# non-zeros values: ", c1, c2) logger.log_status("max distance between 2 elements: %f" % self._max_distance_between_two_elements( lc_values, expected_values)) logger.log_status(lc_values, expected_values) count = 0 total = 0 for (i, j) in zip(lc_values, expected_values): if i != j: count = count + 1 total = total + 1 logger.log_status("# different elements = %d, over %d, with a %f percentage" \ % (count, total, count*1.0/total)) #self.assert_(ma.allclose(lc_values.sum(), expected_values.sum(), atol=sum_atol)) self.assert_( ma.allclose(lc_values, expected_values, atol=element_atol)) finally: if os.path.exists(temp_dir): rmtree(temp_dir)