def _generate_input_land_cover(self, current_year, base_directory, 
                         urbansim_cache_directory, years, output_directory,
                         convert_flt, convert_input):
        if current_year == years[0]:
            if not convert_input:
                return base_directory
            else:
                
                package_dir_path = package().get_package_path()
                command = os.path.join(package_dir_path, "tools", "lc_convert.py")
                status = os.system(command + ' %s -i "%s" -o "%s"' % ('input data',  base_directory, self.temp_land_cover_dir))
                assert(status == 0, "generate input failed")
                return self.temp_land_cover_dir
            
        previous_year = self._get_previous_year(current_year, years)
            
        if not convert_flt: 
            logger.start_block("Copy data from %s to temp land cover folder" % urbansim_cache_directory)
            try:
                self._copy_invariants_to_temp_land_cover_dir(os.path.join(urbansim_cache_directory, str(previous_year)))
            finally:
                logger.end_block()
            return self.temp_land_cover_dir
        
#        package_dir_path = package().get_package_path()
#        command = os.path.join(package_dir_path, "tools", "lc_convert.py")
        flt_directory_in = os.path.join(output_directory, str(previous_year))
        flt_directory_out = self.temp_land_cover_dir
        LCCMInputConvert()._convert_lccm_input(flt_directory_in, flt_directory_out)
#        status = os.system(command + ' %d -i "%s" -o "%s"' % (previous_year, flt_directory_in, flt_directory_out))
#        assert(status == 0, "generate input failed")        
        return self.temp_land_cover_dir
Ejemplo n.º 2
0
 def setUp(self):
     package_dir_path = package().get_package_path()
     self.base_directory = os.path.join(package_dir_path, "data",
                                   "small_test_set_opus", "1991")
     
     zipped_cache = os.path.join(package_dir_path, "data", "sample_cache.zip")
     self.cache_dir = mkdtemp(prefix='tmp_biocomplexity')
     unzip_files_into_dir(zipped_cache, self.cache_dir)        
     self.urbansim_cache_directory = os.path.join(self.cache_dir, "urbansim_cache", 
                                                  "data", "2006_02_14__11_43")
 def _generate_output_flt(self, current_year, urbansim_cache_directory,
                           output_directory, convert_flt):
     if not convert_flt:
         return
     
     package_dir_path = package().get_package_path()
     command = os.path.join(package_dir_path, "tools", "lc_convert_to_flt.py")        
     flt_directory_in = os.path.join(urbansim_cache_directory, str(current_year))
     flt_directory_out = os.path.join(output_directory, str(current_year))
     status = os.system(sys.executable + ' ' + command + ' %d -i "%s" -o "%s"' % (current_year, flt_directory_in, flt_directory_out))
     assert(status == 0, "generate output failed")
Ejemplo n.º 4
0
    def __init__(self, **kargs):
        #        Estimator.__init__(self, settings=None, run_land_price_model_before_estimation=False, **kargs) # <-- old __init__
        #        Estimator.__init__(self, config=None, save_estimation_results=True) # <-- new __init__ doesn't work, but not needed

        parent_dir_path = package().get_package_parent_path()
        package_path = OpusPackage().get_path_for_package("biocomplexity")
        self.storage = StorageFactory().get_storage("tab_storage", storage_location=os.path.join(package_path, "data"))

        ## 1. directory path of full (4 county spatial extent) dataset
        flt_directory = os.path.join(parent_dir_path, "biocomplexity", "data", "LCCM_4County")

        ## 2. select (uncomment) from one the following choices of directory pathes of subsetted sample input data/variables
        #        flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "LCCM_small_test_set_opus")
        flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_estimation_all")
        #        flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_estimation_all_orig")
        #        flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_suburban_orig")
        #        flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_urban")
        #        flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_urban_orig")

        ## note - must rename lct-forusewith91sample.Float32 to lct.lf4 if doing 1991-1995
        ## note - must rename lct-forusewith95sample.Float32 to lct.lf4 if doing 1995-1999

        ## 3. select (uncomment) from one the following choices of land cover data (input data) date pairs (years)
        #        years = [1991, 1995]
        #        years = [1995, 1999]
        years = [1999, 2002]

        self.lc1 = LandCoverDataset(
            in_storage=StorageFactory().get_storage(
                "flt_storage", storage_location=os.path.join(flt_directory_est, str(years[0]))
            ),
            resources=Resources({"lowercase": 1}),
        )
        self.lc2 = LandCoverDataset(
            in_storage=StorageFactory().get_storage(
                "flt_storage", storage_location=os.path.join(flt_directory_est, str(years[1]))
            ),
            resources=Resources({"lowercase": 1}),
        )

        self.lc1_all = LandCoverDataset(
            in_storage=StorageFactory().get_storage(
                "flt_storage", storage_location=os.path.join(flt_directory, str(years[0]))
            ),
            resources=Resources({"lowercase": 1}),
        )
        self.lc1_all.flush_dataset()
        self.lc2_all = LandCoverDataset(
            in_storage=StorageFactory().get_storage(
                "flt_storage", storage_location=os.path.join(flt_directory, str(years[1]))
            ),
            resources=Resources({"lowercase": 1}),
        )
        self.lc2_all.flush_dataset()
    def setUp(self):
        package_dir_path = package().get_package_path()
        self.base_directory = os.path.join(package_dir_path, "data",
                                           "small_test_set_opus", "1991")

        zipped_cache = os.path.join(package_dir_path, "data",
                                    "sample_cache.zip")
        self.cache_dir = mkdtemp(prefix='tmp_biocomplexity')
        unzip_files_into_dir(zipped_cache, self.cache_dir)
        self.urbansim_cache_directory = os.path.join(self.cache_dir,
                                                     "urbansim_cache", "data",
                                                     "2006_02_14__11_43")
Ejemplo n.º 6
0
    def __init__(self, **kargs):
        #        Estimator.__init__(self, settings=None, run_land_price_model_before_estimation=False, **kargs) # <-- old __init__
        #        Estimator.__init__(self, config=None, save_estimation_results=True) # <-- new __init__ doesn't work, but not needed

        parent_dir_path = package().get_package_parent_path()
        package_path = OpusPackage().get_path_for_package("biocomplexity")
        self.storage = StorageFactory().get_storage(
            'tab_storage', storage_location=os.path.join(package_path, 'data'))

        ## 1. directory path of full (4 county spatial extent) dataset
        flt_directory = os.path.join(parent_dir_path, "biocomplexity", "data",
                                     "LCCM_4County")

        ## 2. select (uncomment) from one the following choices of directory pathes of subsetted sample input data/variables
        #        flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "LCCM_small_test_set_opus")
        flt_directory_est = os.path.join(parent_dir_path, "biocomplexity",
                                         "data", "data_for_estimation_all")
        #        flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_estimation_all_orig")
        #        flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_suburban_orig")
        #        flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_urban")
        #        flt_directory_est = os.path.join(parent_dir_path, "biocomplexity", "data", "data_for_urban_orig")

        ## note - must rename lct-forusewith91sample.Float32 to lct.lf4 if doing 1991-1995
        ## note - must rename lct-forusewith95sample.Float32 to lct.lf4 if doing 1995-1999

        ## 3. select (uncomment) from one the following choices of land cover data (input data) date pairs (years)
        #        years = [1991, 1995]
        years = [1995, 1999]
        #        years = [1999, 2002]

        self.lc1 = LandCoverDataset(in_storage=StorageFactory().get_storage(
            "flt_storage",
            storage_location=os.path.join(flt_directory_est, str(years[0]))),
                                    resources=Resources({"lowercase": 1}))
        self.lc2 = LandCoverDataset(in_storage=StorageFactory().get_storage(
            "flt_storage",
            storage_location=os.path.join(flt_directory_est, str(years[1]))),
                                    resources=Resources({"lowercase": 1}))

        self.lc1_all = LandCoverDataset(
            in_storage=StorageFactory().get_storage(
                "flt_storage",
                storage_location=os.path.join(flt_directory, str(years[0]))),
            resources=Resources({"lowercase": 1}))
        self.lc1_all.flush_dataset()
        self.lc2_all = LandCoverDataset(
            in_storage=StorageFactory().get_storage(
                "flt_storage",
                storage_location=os.path.join(flt_directory, str(years[1]))),
            resources=Resources({"lowercase": 1}))
        self.lc2_all.flush_dataset()
Ejemplo n.º 7
0
            #os.remove(file_name)
            content = fromfile(file_name, dtype=float32) # reads in existing file
            put(content.ravel(), flat_indices.astype(int32)[offset:], data[offset:]) # incoprorates existing data using offset (chunk) size
        else:
            content = self.nodata_values*ones(shape=(self.nrows, self.ncols)).astype(float32)
            put(content.ravel(), flat_indices.astype(int32), data) # <-- coerced flat_indices as integer32 type - changed on 26 may 09 by mm
        content.tofile(file_name)

    def _create_header(self, file_name):
        content = \
    """    ncols         %d
    nrows         %d
    xllcorner     %f
    yllcorner     %f
    cellsize      %d
    NODATA_value  %d
    byteorder     LSBFIRST
    """ % (self.ncols, self.nrows, self.xllcorner, self.yllcorner, self.cellsize, self.nodata_values)
        f = open(file_name, mode="w")
        f.write(content)
        f.close()
        
if __name__ == "__main__":
    parent_dir_path = package().get_package_parent_path()
    flt_directory_in = os.path.join(parent_dir_path, "biocomplexity", "data", "LCCM_4County_converted", "1991")
    flt_directory_out = os.path.join(parent_dir_path, "biocomplexity", "data", "LCCM_4County", "2002")
    current_year = 1991
    ConvertToFloat()._create_flt_file(current_year, flt_directory_in, flt_directory_out)

    
    
Ejemplo n.º 8
0
from biocomplexity.datasets.land_cover_dataset import LandCoverDataset
from biocomplexity.examples.lccm_runner_sample import LccmConfiguration
from biocomplexity.opus_package_info import package
from opus_core.datasets.dataset import DatasetSubset
from opus_core.storage_factory import StorageFactory
from opus_core.logger import logger
from numpy import where
from optparse import OptionParser
import os

import shutil
import sys

if __name__ == "__main__":
    parent_dir_path = package().get_package_parent_path()
    
    # directory of the data 
#    flt_directory_in = os.path.join(parent_dir_path, "biocomplexity", "data", "LCCM_small_test_set_opus", "1995")
    flt_directory_in = os.path.join(parent_dir_path, "biocomplexity", "data", "LCCM_4County", "2002")
    flt_directory_out = flt_directory_in
    
    # set here the type of the attributes
    valuetypes = {}
    
    test_flag = False
    if len(sys.argv) >= 2:
        parser = OptionParser()
        parser.add_option("-i", "--input", dest="input", action="store", type="string")
        parser.add_option("-o", "--output", dest="output", action="store", type="string")
        parser.add_option("-t", "--test", dest="test_flag", action="store_true", default="false")
Ejemplo n.º 9
0
 def do_test_on_expected_data(self, input_variables_list, input_resources=None, 
                              element_atol=None, sum_atol=None):
     from biocomplexity.datasets.land_cover_dataset import LandCoverDataset
     from biocomplexity.tests.utils.land_cover_tests_utils import make_input_data            
     
     import os
     
     package_dir_path = package().get_package_path()
     flt_directory = os.path.join(package_dir_path, "data", "small_test_set_opus", "1995")
     #flt_directory = r"C:\eclipse\LCCM_small_test_set_converted\1991"
     expected_lc = LandCoverDataset(in_storage = StorageFactory().get_storage('flt_storage', storage_location = flt_directory))
     expected_lc.load_dataset()            
     
     temp_dir = make_input_data(flt_directory, input_variables_list)
     try:
         lc = LandCoverDataset(in_storage = StorageFactory().get_storage(
             'flt_storage', 
             storage_location=temp_dir),
             out_storage = StorageFactory().get_storage('flt_storage', storage_location = r"c:/tmp"))
         lc.load_dataset()
         
         dataset_pool = DatasetPool(
             package_order=['biocomplexity'],
             storage=StorageFactory().get_storage('flt_storage', storage_location=temp_dir))
         dataset_pool._add_dataset('land_cover', lc)
         lc.compute_variables(self.variable_name, resources=input_resources, 
                              dataset_pool=dataset_pool)
         
         #lc.write_dataset(attributes='*')
         
         lc_values = lc.get_attribute(self.variable_name)
         expected_values = expected_lc.get_attribute(self.variable_name)    
         
         if sum_atol is None: sum_atol = 1e-8
         if element_atol is None: element_atol = 1e-8
         
         if (not ma.allclose(lc_values, expected_values, atol=element_atol)):
             logger.log_status("comparision using element-atol=%f, sum-atol=%f" % (element_atol, sum_atol))
             logger.log_status("      computed      expected");
             logger.log_status("sum: ", lc_values.sum(), expected_values.sum())
             logger.log_status("max: ", max(lc_values), max(expected_values))
             logger.log_status("min: ", min(lc_values), min(expected_values))
             
             c1 = 0
             c2 = 0
             for (i,j) in zip(lc_values, expected_values):
                 if i != 0:
                     c1 = c1 + 1
                 if j != 0:
                     c2 = c2 + 1
                     
             logger.log_status("# non-zeros values: ", c1, c2)
             logger.log_status("max distance between 2 elements: %f" % 
                                         self._max_distance_between_two_elements(lc_values,expected_values))
             logger.log_status(lc_values, expected_values)
             count = 0
             total = 0
             for (i,j) in zip(lc_values, expected_values):
                 if i != j:
                     count = count + 1
                 total = total + 1
             logger.log_status("# different elements = %d, over %d, with a %f percentage" \
                             % (count, total, count*1.0/total))
         
         #self.assert_(ma.allclose(lc_values.sum(), expected_values.sum(), atol=sum_atol))
         self.assert_(ma.allclose(lc_values, expected_values, atol=element_atol))
     finally:
         if os.path.exists(temp_dir):
             rmtree(temp_dir)
Ejemplo n.º 10
0
    def do_test_on_expected_data(self,
                                 input_variables_list,
                                 input_resources=None,
                                 element_atol=None,
                                 sum_atol=None):
        from biocomplexity.datasets.land_cover_dataset import LandCoverDataset
        from biocomplexity.tests.utils.land_cover_tests_utils import make_input_data

        import os

        package_dir_path = package().get_package_path()
        flt_directory = os.path.join(package_dir_path, "data",
                                     "small_test_set_opus", "1995")
        #flt_directory = r"C:\eclipse\LCCM_small_test_set_converted\1991"
        expected_lc = LandCoverDataset(in_storage=StorageFactory().get_storage(
            'flt_storage', storage_location=flt_directory))
        expected_lc.load_dataset()

        temp_dir = make_input_data(flt_directory, input_variables_list)
        try:
            lc = LandCoverDataset(in_storage=StorageFactory().get_storage(
                'flt_storage', storage_location=temp_dir),
                                  out_storage=StorageFactory().get_storage(
                                      'flt_storage',
                                      storage_location=r"c:/tmp"))
            lc.load_dataset()

            dataset_pool = DatasetPool(package_order=['biocomplexity'],
                                       storage=StorageFactory().get_storage(
                                           'flt_storage',
                                           storage_location=temp_dir))
            dataset_pool._add_dataset('land_cover', lc)
            lc.compute_variables(self.variable_name,
                                 resources=input_resources,
                                 dataset_pool=dataset_pool)

            #lc.write_dataset(attributes='*')

            lc_values = lc.get_attribute(self.variable_name)
            expected_values = expected_lc.get_attribute(self.variable_name)

            if sum_atol is None: sum_atol = 1e-8
            if element_atol is None: element_atol = 1e-8

            if (not ma.allclose(lc_values, expected_values,
                                atol=element_atol)):
                logger.log_status(
                    "comparision using element-atol=%f, sum-atol=%f" %
                    (element_atol, sum_atol))
                logger.log_status("      computed      expected")
                logger.log_status("sum: ", lc_values.sum(),
                                  expected_values.sum())
                logger.log_status("max: ", max(lc_values),
                                  max(expected_values))
                logger.log_status("min: ", min(lc_values),
                                  min(expected_values))

                c1 = 0
                c2 = 0
                for (i, j) in zip(lc_values, expected_values):
                    if i != 0:
                        c1 = c1 + 1
                    if j != 0:
                        c2 = c2 + 1

                logger.log_status("# non-zeros values: ", c1, c2)
                logger.log_status("max distance between 2 elements: %f" %
                                  self._max_distance_between_two_elements(
                                      lc_values, expected_values))
                logger.log_status(lc_values, expected_values)
                count = 0
                total = 0
                for (i, j) in zip(lc_values, expected_values):
                    if i != j:
                        count = count + 1
                    total = total + 1
                logger.log_status("# different elements = %d, over %d, with a %f percentage" \
                                % (count, total, count*1.0/total))

            #self.assert_(ma.allclose(lc_values.sum(), expected_values.sum(), atol=sum_atol))
            self.assert_(
                ma.allclose(lc_values, expected_values, atol=element_atol))
        finally:
            if os.path.exists(temp_dir):
                rmtree(temp_dir)