def run(self, config, year, *args, **kwargs):
        """This is the main entry point.  It gets the appropriate configuration info from the 
        travel_model_configuration part of this config, and then copies the specified 
        UrbanSim data into files for travel mdel to read.  
        """
        cache_directory = config['cache_directory']
        simulation_state = SimulationState()
        simulation_state.set_cache_directory(cache_directory)
        simulation_state.set_current_time(year)
        attribute_cache = AttributeCache()
        dataset_pool = SessionConfiguration(new_instance=True,
                                            package_order=config['dataset_pool_configuration'].package_order,
                                            in_storage=attribute_cache).get_dataset_pool()

        #cache_storage = AttributeCache().get_flt_storage_for_year(year_for_base_year_cache)
        #datasets = DatasetFactory().create_datasets_from_flt(config.get('datasets_to_preload',{}),
                                                            #"urbansim",
                                                            #additional_arguments={'in_storage': attribute_cache})
        zone_set = dataset_pool.get_dataset('travel_zone')
        self.prepare_for_run(config['travel_model_configuration'], year)
        self.create_travel_model_input_file(config=config, 
                                            year=year, 
                                            zone_set=zone_set, 
                                            datasets=dataset_pool,
                                            *args, **kwargs)
Example #2
0
    def __init__(self, config):
        ss = SimulationState(new_instance=True)
        ss.set_current_time(config['base_year'])
        ss.set_cache_directory(config['cache_directory'])

        SessionConfiguration(new_instance=True,
                             package_order=config['dataset_pool_configuration'].package_order,
                             in_storage=AttributeCache())
        #if not os.path.exists(config['cache_directory']):  ## if cache exists, it will automatically skip
        cacher = CreateBaseyearCache()
        cache_dir = cacher.run(config)

        if 'estimation_database_configuration' in config:
            db_server = DatabaseServer(config['estimation_database_configuration'])
            db = db_server.get_database(config['estimation_database_configuration'].database_name)
            out_storage = StorageFactory().get_storage(
                'sql_storage', 
                storage_location = db)
        else:
            output_cache = os.path.join(config['cache_directory'], str(config['base_year']+1))
            out_storage = StorageFactory().get_storage('flt_storage', storage_location=output_cache)

        dataset_pool = SessionConfiguration().get_dataset_pool()
        households = dataset_pool.get_dataset("household")
        buildings = dataset_pool.get_dataset("building")
        zones = dataset_pool.get_dataset("zone")
        zone_ids = zones.get_id_attribute()
        capacity_attribute_name = "residential_units"  #_of_use_id_%s" % id
        capacity_variable_name = "%s=sanfrancisco.zone.aggregate_%s_from_building" % \
                                 (capacity_attribute_name, capacity_attribute_name)
        buildings.compute_variables("sanfrancisco.building.zone_id", dataset_pool=dataset_pool)
        zones.compute_variables(capacity_variable_name, dataset_pool=dataset_pool)

        building_zone_id = buildings.get_attribute('zone_id')
        
#        is_household_unplace = datasets['household'].get_attribute("building_id") <= 0
        is_household_unplaced = 1 #all households are unplaced
        household_building_id = zeros(households.size(), dtype='int32')-1 #datasets['household'].get_attribute("building_id")
        
        for zone_id in zone_ids:
            capacity = zones.get_attribute_by_id(capacity_attribute_name, zone_id)
            is_household_in_this_zone = (households.get_attribute('zone_id') == zone_id)
            is_unplaced_household_in_this_zone = is_household_in_this_zone * is_household_unplaced
            is_building_in_this_zone = (building_zone_id == zone_id)
#            if not is_household_in_this_zone.sum() <= capacity:
            if capacity == 0 or is_household_in_this_zone.sum()==0:
                print "WARNING: zone %s has %s households but only %s units" % (zone_id, is_household_in_this_zone.sum(), capacity)
                continue
                        
            prob = buildings.get_attribute(capacity_attribute_name) * is_building_in_this_zone / array(capacity, dtype=float64)

            r = random(sum(is_unplaced_household_in_this_zone))
            prob_cumsum = ncumsum(prob)
            index_to_bldg = searchsorted(prob_cumsum, r)

            household_building_id[where(is_unplaced_household_in_this_zone)] = buildings.get_attribute_by_index('building_id', index_to_bldg)

#        import pdb;pdb.set_trace()
        households.set_values_of_one_attribute('building_id', household_building_id)
        households.write_dataset(out_table_name='households', out_storage=out_storage)
    def __init__(self, config):
        if 'estimation_database_configuration' in config:
            db_server = DatabaseServer(config['estimation_database_configuration'])
            db = db_server.get_database(config['estimation_database_configuration'].database_name)
        
            out_storage = StorageFactory().build_storage_for_dataset(
                type='sql_storage', storage_location=db)
        else:
            out_storage = StorageFactory().get_storage(type='flt_storage',
                storage_location=os.path.join(config['cache_directory'], str(config['base_year']+1)))

        simulation_state = SimulationState()
        simulation_state.set_cache_directory(config['cache_directory'])
        simulation_state.set_current_time(config['base_year'])
        attribute_cache = AttributeCache()
        
        SessionConfiguration(new_instance=True,
                             package_order=config['dataset_pool_configuration'].package_order,
                             in_storage=attribute_cache)
        
        if not os.path.exists(os.path.join(config['cache_directory'], str(config['base_year']))):
            #raise RuntimeError, "datasets uncached; run prepare_estimation_data.py first"
            CacheScenarioDatabase().run(config, unroll_gridcells=False)

        for dataset_name in config['datasets_to_preload']:
            SessionConfiguration().get_dataset_from_pool(dataset_name)

        households = SessionConfiguration().get_dataset_from_pool("household")
        household_ids = households.get_id_attribute()
        workers = households.get_attribute("workers")
        
        hh_ids = []
        member_ids = []
        is_worker = []
        job_ids = []

        for i in range(households.size()):  
            if workers[i] > 0:
                hh_ids += [household_ids[i]] * workers[i]
                member_ids += range(1, workers[i]+1)
                is_worker += [1] * workers[i]
                job_ids += [-1] * workers[i]

        in_storage = StorageFactory().get_storage('dict_storage')
        
        persons_table_name = 'persons'
        in_storage.write_table(
                table_name=persons_table_name,
                table_data={
                    'person_id':arange(len(hh_ids))+1,
                    'household_id':array(hh_ids),
                    'member_id':array(member_ids),
                    'is_worker':array(is_worker),                    
                    'job_id':array(job_ids),
                    },
            )

        persons = PersonDataset(in_storage=in_storage, in_table_name=persons_table_name)
        persons.write_dataset(out_storage=out_storage, out_table_name=persons_table_name)
class RunSimulationFromMysql:
    def prepare_for_simulation(self, run_configuration, cache_directory=None):
        self.config = Resources(run_configuration)
        self.simulation_state = SimulationState(
            new_instance=True, base_cache_dir=cache_directory, start_time=self.config.get("base_year", 0)
        )

        ### TODO: Get rid of this! There is no good reason to be changing the
        ###       Configuration.
        if self.config["cache_directory"] is None:
            self.config["cache_directory"] = self.simulation_state.get_cache_directory()

        SessionConfiguration(
            new_instance=True,
            package_order=self.config["dataset_pool_configuration"].package_order,
            in_storage=AttributeCache(),
        )

        ForkProcess().fork_new_process(
            self.config["creating_baseyear_cache_configuration"].cache_scenario_database, self.config
        )

        # Create output database (normally done by run manager)
        if "estimation_database_configuration" in self.config:
            db_server = DatabaseServer(self.config["estimation_database_configuration"])
            if not db_server.has_database(self.config["estimation_database_configuration"].database_name):
                db_server.create_database(self.config["estimation_database_configuration"].database_name)

    def run_simulation(self, simulation_instance=None):
        logger.start_block("Simulation on database %s" % self.config["scenario_database_configuration"].database_name)
        try:
            if simulation_instance is None:
                simulation_instance = ModelSystem()
            simulation_instance.run(self.config)
            # simulation_instance.run_multiprocess(self.config, is_run_subset=True)
        finally:
            logger.end_block()
        logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())

    def cleanup(self, remove_cache, remove_output_database):
        """Remove all outputs of this simulation."""
        self.simulation_state.remove_singleton(delete_cache=remove_cache)
        # Remove SessionConfiguration singleton, if it exists
        Singleton().remove_singleton_for_class(SessionConfiguration)

        cache_dir = self.config["cache_directory"]
        if os.path.exists(cache_dir):
            rmtree(cache_dir)
        if remove_output_database and ("estimation_database_configuration" in self.config):
            db_server = DatabaseServer(self.config["estimation_database_configuration"])
            db_server.drop_database(self.config["estimation_database_configuration"].database_name)

    def prepare_and_run(self, run_configuration, simulation_instance=None, remove_cache=True):
        self.prepare_for_simulation(run_configuration)
        self.run_simulation(simulation_instance)
        self.cleanup(remove_cache)
Example #5
0
 def run(self, year, cache_directory=None):
     """The class is initialized with the appropriate configuration info from the 
     travel_model_configuration part of this config, and then copies the specified 
     UrbanSim data into files for daysim to read.
     The variables/expressions to export are defined in the node travel_model_configuration/urbansim_to_tm_variable_mapping
     of the configuration file.
     """
     if cache_directory is None:
         cache_directory = self.config['cache_directory']
     simulation_state = SimulationState()
     simulation_state.set_cache_directory(cache_directory)
     simulation_state.set_current_time(year)
     attribute_cache = AttributeCache()
     sc = SessionConfiguration(new_instance=True,
                               package_order=self.config['dataset_pool_configuration'].package_order,
                               in_storage=attribute_cache)
     dataset_pool = sc.get_dataset_pool()
     tm_config = self.config['travel_model_configuration']
     data_to_export = tm_config['urbansim_to_tm_variable_mapping']
     
     table_names = data_to_export.keys()
     variable_names = {}
     datasets = {}
     filenames = {}
     in_table_names = {}
     for table_name in table_names:
         filter = data_to_export[table_name].get('__filter__', None)
         if filter is not None:
             del data_to_export[table_name]['__filter__']
         out_table_name = data_to_export[table_name].get('__out_table_name__', None)
         if out_table_name is not None:
             del data_to_export[table_name]['__out_table_name__']
         else:
             out_table_name = table_name
         variables_to_export = map(lambda alias: "%s = %s" % (alias, data_to_export[table_name][alias]), data_to_export[table_name].keys())
         dataset_name = None            
         for var in variables_to_export:
             var_name = VariableName(var)
             if dataset_name is None:
                 dataset_name = var_name.get_dataset_name()
                 ds = dataset_pool.get_dataset(dataset_name)
                 
                 datasets[dataset_name] = ds
                 filenames[dataset_name] = out_table_name
                 in_table_names[dataset_name] = table_name
                 if dataset_name not in variable_names.keys():
                     variable_names[dataset_name] = []
             variable_names[dataset_name].append(var_name.get_alias())                
             ds.compute_variables([var_name], dataset_pool=dataset_pool)
         if filter is not None:
             filter_idx = where(ds.compute_variables(["__filter__ = %s" % filter], dataset_pool=dataset_pool)>0)[0]
             ds = DatasetSubset(ds, index = filter_idx)
             datasets[dataset_name] = ds
             
     return self._call_input_file_writer(year, datasets, in_table_names, filenames, variable_names, dataset_pool)
Example #6
0
 def test_doing_refinements_from_specified_refinement_dataset(self):
     self.prepare_cache()
     os.system("%(python)s %(script)s -c %(cache_directory)s -s %(start_year)s -e %(end_year)s --refinements-directory=%(refinement_directory)s" % 
               {'python': sys.executable, 'script': self.script, 'cache_directory': self.cache_dir,
                'start_year': 2021, 'end_year': 2022,
                'refinement_directory': os.path.join(self.cache_dir, '2000')}
               )
     
     simulation_state = SimulationState()
     
     ## test refinement for 2021
     simulation_state.set_current_time(2021)
     jobs = self.dataset_pool.get_dataset('job')
     buildings = self.dataset_pool.get_dataset('building')
     jobs13_raz3 = jobs.compute_variables('numpy.logical_and(job.sector_id==13, job.disaggregate(parcel.raz_id==3, intermediates=[building]))', 
                                               dataset_pool=self.dataset_pool)
     jobs13_raz4 = jobs.compute_variables('numpy.logical_and(job.sector_id==13, job.disaggregate(parcel.raz_id, intermediates=[building])==4)', 
                                               dataset_pool=self.dataset_pool)
     jobs13_raz5 = jobs.compute_variables('numpy.logical_and(job.sector_id==13, job.disaggregate(parcel.raz_id, intermediates=[building])==5 )', 
                                               dataset_pool=self.dataset_pool)
     jobs_raz5 = jobs.compute_variables('job.disaggregate(parcel.raz_id, intermediates=[building])==5', 
                                             dataset_pool=self.dataset_pool)
     
     #check results
     self.assertEqual(jobs13_raz3.sum(), 0)
     self.assertEqual(jobs13_raz4.sum(), 0)
     self.assertEqual(jobs13_raz5.sum() >= 5, True)
     self.assertEqual(jobs_raz5.sum(), 7)
     expected_nr_sqft = array([6, 0, 3, 6, 1, 6, 5, 0])
     ## was             array([6, 2, 3, 6, 1, 2, 5, 0]),
     self.assert_(allclose(buildings.get_attribute('non_residential_sqft'),  expected_nr_sqft))
     
     self.dataset_pool.remove_all_datasets()
     
     
     ## test refinement for 2022
     simulation_state.set_current_time(2022)
     hhs = self.dataset_pool.get_dataset('household')
     buildings = self.dataset_pool.get_dataset('building')
     
     hhs_raz6 = hhs.compute_variables('household.disaggregate(building.disaggregate(parcel.raz_id)==6)', 
                                           dataset_pool=self.dataset_pool)
     hhs_bldg = buildings.compute_variables('building.number_of_agents(household)', 
                                                 dataset_pool=self.dataset_pool)
     
     #check results
     self.assertEqual(hhs_raz6.sum(), 7)
     self.assert_(hhs_bldg.sum(),  7 )
     self.assert_((hhs_bldg!=0).sum(),  2)
     self.assert_(buildings.get_attribute('residential_units').sum(),  7)
     
     self.dataset_pool.remove_all_datasets()        
Example #7
0
 def run(self, year, skim_directory=None):
     """ It gets the appropriate values from the 
     travel_model_configuration part of this config, and then copies the specified 
     data into the specified travel_data variable names.  Results in
     a new travel_data cache for year+1.
     """
     cache_directory = self.config['cache_directory']
     simulation_state = SimulationState()
     simulation_state.set_current_time(year)
     simulation_state.set_cache_directory(cache_directory)
     
     year_config = self.config['travel_model_configuration'][year]
     self.write_travel_data(year, cache_directory)
    def _initialize_previous_years(self, attribute_cache, 
                                   base_year,
                                   creating_baseyear_cache_configuration):
        simulation_state = SimulationState()
        cache_directory = simulation_state.get_cache_directory()

        baseyear_cache_path = os.path.join(cache_directory, str(base_year))
        for table_name, year in creating_baseyear_cache_configuration.tables_to_copy_to_previous_years.iteritems():
            year_cache_path = os.path.join(cache_directory, str(year))
            dest_file_path = os.path.join(year_cache_path, table_name)
            if os.path.exists(dest_file_path):
                rmtree(dest_file_path)
            copytree(os.path.join(baseyear_cache_path, table_name),
                     dest_file_path)
def setup_environment(cache_directory, year, package_order, additional_datasets={}):
    gc.collect()
    ss = SimulationState(new_instance=True)
    ss.set_cache_directory(cache_directory)
    ss.set_current_time(year)
    ac = AttributeCache()
    storage = ac.get_flt_storage_for_year(year)
    sc = SessionConfiguration(new_instance=True,
                         package_order=package_order,
                         in_storage=ac)
    logger.log_status("Setup environment for year %s. Use cache directory %s." % (year, storage.get_storage_location()))
    dp = sc.get_dataset_pool()
    for name, ds in additional_datasets.iteritems():
        dp.replace_dataset(name, ds)
    return dp
    def prepare_for_simulation(self, run_configuration, cache_directory=None):
        self.config = Resources(run_configuration)
        self.simulation_state = SimulationState(
            new_instance=True, base_cache_dir=cache_directory, start_time=self.config.get("base_year", 0)
        )

        ### TODO: Get rid of this! There is no good reason to be changing the
        ###       Configuration.
        if self.config["cache_directory"] is None:
            self.config["cache_directory"] = self.simulation_state.get_cache_directory()

        SessionConfiguration(
            new_instance=True,
            package_order=self.config["dataset_pool_configuration"].package_order,
            in_storage=AttributeCache(),
        )

        ForkProcess().fork_new_process(
            self.config["creating_baseyear_cache_configuration"].cache_scenario_database, self.config
        )

        # Create output database (normally done by run manager)
        if "estimation_database_configuration" in self.config:
            db_server = DatabaseServer(self.config["estimation_database_configuration"])
            if not db_server.has_database(self.config["estimation_database_configuration"].database_name):
                db_server.create_database(self.config["estimation_database_configuration"].database_name)
    def setUp(self):
        self.start_year = 2001
        self.expected_sic_data = array([6,4,7,808,6])
        self.job_id = array([1,2,3,4,5])
        self.base_cache_dir = tempfile.mkdtemp(prefix='opus_tmp_test_dataset')
        self.simulation_state = SimulationState(low_memory_run=True, new_instance=True, base_cache_dir=self.base_cache_dir)
        self.dir = self.simulation_state.get_cache_directory()
        self.simulation_state.set_current_time(self.start_year)
        
        if not os.path.exists(self.dir):
            os.makedirs(self.dir)

        self.in_storage = StorageFactory().get_storage('dict_storage')
        self.in_storage.write_table(
            table_name='jobs',
            table_data={
                'grid_id':array([10,20,30,40,50]),
                'job_id':self.job_id,
                },
            )
            
        self.out_storage = StorageFactory().get_storage('dict_storage')
    
        self.job_set_resources = ResourceFactory().get_resources_for_dataset(
            'job', 
            in_storage = self.in_storage, 
            out_storage = self.out_storage,
            in_table_name_pair = ('jobs',None),
            out_table_name_pair = ('jobs_exported',None),
            attributes_pair = (None,AttributeType.PRIMARY),
            id_name_pair = ('job_id','job_id'), 
            nchunks_pair = (1,1), 
            debug_pair = (1,None)
            )
Example #12
0
    def setUp(self):
        self.config = TestCacheConfiguration()

        self.simulation_state = SimulationState(new_instance=True)
        SessionConfiguration(self.config, new_instance=True, 
                             package_order=['urbansim', 'opus_core'],
                             in_storage=AttributeCache()) 

        self.base_year = self.config['base_year']
        creating_baseyear_cache_configuration = self.config['creating_baseyear_cache_configuration']
        
        self.simulation_state.set_current_time(self.base_year)

        cache_directory = self.simulation_state.get_cache_directory()
        copytree(os.path.join(creating_baseyear_cache_configuration.baseyear_cache.existing_cache_to_copy, 
                              str(self.base_year)),
                 os.path.join(cache_directory, str(self.base_year)))
        cacher = CacheScenarioDatabase()
        cacher.prepare_data_before_baseyear(cache_directory, self.base_year, creating_baseyear_cache_configuration)
        
        self.config['cache_directory'] = cache_directory
        
        cache_storage = AttributeCache().get_flt_storage_for_year(self.base_year)
        cache_directory = self.simulation_state.get_cache_directory()
        flt_directory = os.path.join(cache_directory, str(self.base_year))
        self.gridcell = DatasetFactory().get_dataset('gridcell', 
            package='urbansim',
            subdir='datasets',
            arguments={'in_storage':StorageFactory().get_storage('flt_storage', storage_location=flt_directory)}
            )
Example #13
0
 def run(self, year):
     """Like its parent, but report files have different format and there are no banks.
     Zones are assumed to have no gaps.
     """
     cache_directory = self.config['cache_directory']
     simulation_state = SimulationState()
     simulation_state.set_current_time(year)
     simulation_state.set_cache_directory(cache_directory)
     
     year_config = self.config['travel_model_configuration'][year]
     
     bank_path = os.path.sep.join([self.get_emme2_base_dir()] + self.config['travel_model_configuration'][year]['bank'])
     for path, variable_dict in year_config['matrix_variable_map'].iteritems():
         path_name = os.path.sep.join([bank_path] + path.split('.'))
         self.get_needed_matrices_from_emme4(year, 
                                             year_config['cache_directory'],
                                             path_name, variable_dict)
    def run(self, config, show_output = False):
        logger.log_status("Caching large SQL tables to: " + config['cache_directory'])
        self.show_output = show_output
        
        #import pydevd;pydevd.settrace()
        
        server_configuration = config['scenario_database_configuration']
        
        scenario_database_manager = ScenarioDatabaseManager(
            server_configuration = server_configuration, 
            base_scenario_database_name = server_configuration.database_name                                                         
        )
        
        self.database_server = DatabaseServer(server_configuration)
        
        database_to_table_mapping = scenario_database_manager.get_database_to_table_mapping()
        
        self.tables_to_cache = config['creating_baseyear_cache_configuration'].tables_to_cache
                
        simulation_state = SimulationState()
        if 'low_memory_run' in config:
            simulation_state.set_low_memory_run(config['low_memory_run'])
        simulation_state.set_cache_directory(config['cache_directory'])
        simulation_state.set_current_time(config['base_year'])
                  
        self.tables_cached = set()      
        for database_name, tables in database_to_table_mapping.items():
            self.cache_database_tables(config, database_name, tables)

        un_cached_tables = set(self.tables_to_cache) - self.tables_cached
        if un_cached_tables:
            logger.log_warning('The following requested tables were NOT cached:')
            for table_name in un_cached_tables:
                logger.log_warning('\t%s' % table_name)
 def setUp(self):
     self.start_year = 2001
     self.expected_sic_data = array([6,4,7,808,6])
     self.job_id = array([1,2,3,4,5])
     self.base_cache_dir = tempfile.mkdtemp(prefix='opus_tmp')
     self.simulation_state = SimulationState(low_memory_run=True, new_instance=True, base_cache_dir=self.base_cache_dir)
     self.dir = self.simulation_state.get_cache_directory()
     self.simulation_state.set_current_time(self.start_year)
     if not os.path.exists(self.dir):
         os.makedirs(self.dir)
Example #16
0
 def __init__(self, package_name, attribute_name, lag_offset, dataset_name, index_name):
     self.package_name = package_name
     self.attribute_name = attribute_name
     self.simulation_state = SimulationState()
     self.attribute_cache = AttributeCache()
     self.lag_offset = lag_offset
     self.dataset_name = dataset_name
     self.index_name = index_name
     self.lag_index_name = "%s_lag%d" % (index_name, lag_offset)
     Variable.__init__(self)
    def run(self, year):
        """This is the main entry point.  The class is initialized with the appropriate configuration info from the 
        travel_model_configuration part of this config, and then copies the specified 
        UrbanSim data into files for emme/2 to read.  
        If households and jobs do not have a primary attribute zone_id, the entry 'locations_to_disaggregate'
        in the travel_model_configuration should be a list of dataset names over which the zone_id 
        will be dissaggregated, ordered from higher to lower aggregation level, e.g. ['parcel', 'building']
        """
        cache_directory = self.config['cache_directory']
        simulation_state = SimulationState()
        simulation_state.set_cache_directory(cache_directory)
        simulation_state.set_current_time(year)
        attribute_cache = AttributeCache()
        sc = SessionConfiguration(new_instance=True,
                                  package_order=self.config['dataset_pool_configuration'].package_order,
                                  in_storage=attribute_cache)
        dataset_pool = sc.get_dataset_pool()

        hh_set = dataset_pool.get_dataset('household')
        zone_set = dataset_pool.get_dataset('zone')
        job_set = dataset_pool.get_dataset('job')
        locations_to_disaggregate = self.config['travel_model_configuration']['locations_to_disaggregate']
        len_locations_to_disaggregate = len(locations_to_disaggregate)
        if len_locations_to_disaggregate > 0:
            primary_location = locations_to_disaggregate[0]
            if len_locations_to_disaggregate > 1:
                intermediates_string = ", intermediates=["
                for i in range(1, len_locations_to_disaggregate):
                    intermediates_string = "%s%s, " % (intermediates_string, locations_to_disaggregate[i])
                intermediates_string = "%s]" % intermediates_string
            else:
                intermediates_string = ""
            hh_set.compute_variables(['%s = household.disaggregate(%s.%s %s)' % (zone_set.get_id_name()[0],
                                                                                 primary_location, zone_set.get_id_name()[0],
                                                                                 intermediates_string)], 
                                      dataset_pool=dataset_pool)
            job_set.compute_variables(['%s = job.disaggregate(%s.%s %s)' % (zone_set.get_id_name()[0],
                                                                            primary_location, zone_set.get_id_name()[0],
                                                                            intermediates_string)], 
                                       dataset_pool=dataset_pool)
        
        return self._call_input_file_writer(year, dataset_pool)
Example #18
0
class RunSimulation(object):
    def prepare_for_simulation(self, config, cache_directory=None):
        self.config = Resources(config)
        base_cache_dir = self.config['creating_baseyear_cache_configuration'].cache_directory_root
        
        self.simulation_state = SimulationState(new_instance=True, base_cache_dir=base_cache_dir,
                                                start_time=self.config.get('base_year', 0))

        ### TODO: Get rid of this! There is no good reason to be changing the 
        ###       Configuration.
        if self.config['cache_directory'] is None:
            self.config['cache_directory'] = self.simulation_state.get_cache_directory()

        SessionConfiguration(new_instance=True,
                             package_order=self.config['dataset_pool_configuration'].package_order,
                             in_storage=AttributeCache())
        
        if config['creating_baseyear_cache_configuration'].cache_from_database:
            ForkProcess().fork_new_process(self.config['creating_baseyear_cache_configuration'].cache_scenario_database, self.config)
        else:
            CacheFltData().run(self.config)

    def run_simulation(self, simulation_instance=None):
        if simulation_instance is None:
            simulation_instance = ModelSystem()
        simulation_instance.run(self.config)
        #simulation_instance.run_multiprocess(self.config, is_run_subset=True)
        logger.log_status("Data cache in %s" % self.simulation_state.get_cache_directory())
        
    def cleanup(self, remove_cache=True):
        """Remove all outputs of this simulation."""    
        self.simulation_state.remove_singleton(delete_cache=remove_cache)
        SessionConfiguration().remove_singleton()
        if remove_cache:
            cache_dir = self.config['cache_directory']
            if os.path.exists(cache_dir):
                rmtree(cache_dir)

    def prepare_and_run(self, run_configuration, simulation_instance=None, remove_cache=True):
        self.prepare_for_simulation(run_configuration)
        self.run_simulation(simulation_instance)
        self.cleanup(remove_cache)
Example #19
0
 def run(self, year):
     """Like its parent, but skims are stored locally in matrix_directory in hdf5 format.
     It is one file per year, called xxxx-travelmodel.h5, where xxxx is the year. 
     Each file has one group per bank, e.g. Bank1, which contains the matrices.
     Zones are assumed to have no gaps.
     """
     cache_directory = self.config['cache_directory']
     simulation_state = SimulationState()
     simulation_state.set_current_time(year)
     simulation_state.set_cache_directory(cache_directory)
     
     year_config = self.config['travel_model_configuration'][year]
     
     bank_path = os.path.sep.join([self.get_emme2_base_dir()] + self.config['travel_model_configuration'][year]['bank'])
     #bank_file = os.path.join(matrix_directory, "%s-travelmodel.h5" % bank_year)
     for path, variable_dict in year_config['matrix_variable_map'].iteritems():
         path_name = os.path.sep.join([bank_path] + path.split('.'))
         self.get_needed_matrices_from_emme4(year, 
                                             year_config['cache_directory'],
                                             path_name, variable_dict)
    def target_func(self, est_v, func=lambda x, y: np.sum(np.abs(x - y)), **kwargs):
        """ Target function."""

        simulation_state = SimulationState()
        simulation_state.set_current_time(self.base_year)
        simulation_state.set_cache_directory(self.cache_directory)
        attribute_cache = AttributeCache()
        dataset_pool = SessionConfiguration(
            new_instance=True, package_order=self.package_order, in_storage=attribute_cache
        ).get_dataset_pool()

        calib_datasets = {}
        for dataset_name, calib_attr in calib_datasets.iteritems():
            dataset = dataset_pool.get_dataset(dataset_name, dataset_arguments={"id_name": []})
            assert (
                subset is None
                or subset.get(dataset_name, None) is None
                or subset_patterns is None
                or subset_patterns.get(dataset_name, None) is None
            )
            if subset is not None and subset.get(dataset_name, None) is not None:
                subset_attr, subset_cond = subset.get(dataset_name)
                index = np.in1d(dataset[subset_attr], subset_cond)
            elif subset_patterns is not None and subset_patterns.get(dataset_name, None) is not None:
                subset_attr, subset_pattern = subset_patterns.get(dataset_name)
                index = array([True if re.search(subset_pattern, attr_v) else False for attr_v in dataset[subset_attr]])
            else:
                index = arange(dataset.size(), dtype="i")

            calib_datasets[dataset_name] = [dataset, calib_attr, index]

        prediction = self.update_prediction(est_v, simulation_state, dataset_pool, calib_datasets, **kwargs)
        ## allow keys in target not appearing in prediction
        ## assuming their values to be 0
        ### every key in target should appear in prediction
        # assert np.all( np.in1d(self.target.keys(), prediction.keys()) )
        target = np.array(self.target.values())
        predct = np.array([prediction[k] if prediction.has_key(k) else 0 for k in self.target.keys()])
        results = func(predct, target)

        return results
Example #21
0
def import_travel_model_data(config, year):

    cache_directory = config['cache_directory']
    simulation_state = SimulationState()
    simulation_state.set_current_time(year)
    simulation_state.set_cache_directory(cache_directory)
    out_store = AttributeCache().get_flt_storage_for_year(year+1)
    out_store_loc = out_store.get_storage_location()

    tm_config = config['travel_model_configuration']
    data_to_import = tm_config['tm_to_urbansim_variable_mapping'] 
    base_dir = mtc_common.tm_get_base_dir(config)
    data_dir = tm_config[year]['data_dir']

    for dataset_name, skim_file in data_to_import.iteritems():
        skim_file = os.path.join(base_dir, data_dir, skim_file)
        data = read_csv(skim_file, header=0)
        
        with block("Caching {} to {}".format(dataset_name, out_store_loc)):
            logger.log_status("Source file {}".format(skim_file))
            opus_ds = to_opus_dataset(data, out_store, dataset_name)
    def __init__(
        self,
        model,
        year,
        scenario_name=None,
        model_group=None,
        configuration=None,
        xml_configuration=None,
        cache_directory=None,
    ):
        self.model_group = model_group
        self.explored_model = model

        if configuration is None:
            if xml_configuration is None:
                raise StandardError, "Either dictionary based or XML based configuration must be given."
            config = xml_configuration.get_run_configuration(scenario_name)
        else:
            config = Configuration(configuration)

        self.scenario_models = config["models"]
        if config.get("models_in_year", None) is not None and config["models_in_year"].get(year, None) is not None:
            del config["models_in_year"][year]
        if model is not None:
            dependent_models = config["models_configuration"][model]["controller"].get("dependencies", [])
            config["models"] = dependent_models
            if model_group is None:
                config["models"] = config["models"] + [{model: ["run"]}]
            else:
                config["models"] = config["models"] + [{model: {"group_members": [{model_group: ["run"]}]}}]
        else:
            config["models"] = []

        config["years"] = [year, year]
        config["datasets_to_cache_after_each_model"] = []
        config["flush_variables"] = False

        self.config = Resources(config)
        self.xml_configuration = xml_configuration

        if cache_directory is None:
            cache_directory = config["creating_baseyear_cache_configuration"].baseyear_cache.existing_cache_to_copy
        self.simulation_state = SimulationState(
            new_instance=True, base_cache_dir=cache_directory, start_time=config.get("base_year", 0)
        )
        self.config["cache_directory"] = cache_directory

        SessionConfiguration(
            new_instance=True,
            package_order=self.config["dataset_pool_configuration"].package_order,
            in_storage=AttributeCache(),
        )
Example #23
0
 def run(self, year):
     """
     Copy skims stored in hdf5 format into the UrbanSim cache.
     
     Should run after psrc_parcel.emme.models.run_export_skims which creates the skims hdf5 file.
     It creates a travel_model dataset with each skim being an attribute of it. 
     Zones are assumed to have no gaps.
     
     Arguments:
     year -- year of the urbansim run. Used to extract the TM year from the bank configuration.
     
     Configuration entries (in travel_model_configuration) used:
     matrix_variable_map -- dictionary of bank names and corresponding skim names.
             Bank names are the path where (back-)slashes are replaced by dots, e.g. skims.auto.am.
             A value for each of such bank name is a dictionary with keys being skim names and 
             values being the desired urbansim attribute name. E.g.
             {'skims.nonmotorized.am':
                   {'abketm': 'am_bike_to_work_travel_time',
                    'awlktm': 'am_walk_time_in_minutes'
                   }
             }
     matrix_h5_directory -- path to the hdf5 file called xxxx-travelmodel.h5  
             where xxxx is replaced by the TM year (default is the Emme base directory), 
             which contains the skims as n x n matrices.
     """
     cache_directory = self.config['cache_directory']
     simulation_state = SimulationState()
     simulation_state.set_current_time(year)
     simulation_state.set_cache_directory(cache_directory)
     
     tmconfig = self.config['travel_model_configuration']
     year_config = tmconfig[year]
     matrix_directory = tmconfig.get('matrix_h5_directory', self.get_emme2_base_dir())        
     bank_year = tmconfig[year]['bank'][0]
     bank_file = os.path.join(matrix_directory, "%s-travelmodel.h5" % bank_year)
     for path, variable_dict in year_config['matrix_variable_map'].iteritems():
         self.get_needed_matrices_from_emme4(year, 
                                             year_config['cache_directory'],
                                             path, variable_dict, bank_file=bank_file)
Example #24
0
 def test_doing_other_refinements(self):
     self.prepare_cache()
     os.system("%(python)s  %(script)s -c %(cache_directory)s -s %(start_year)s -e %(end_year)s" % 
               {'python': sys.executable, 'script': self.script, 'cache_directory':self.cache_dir,
                'start_year': 2023, 'end_year': 2027 }
               )        
     
     
     simulation_state = SimulationState()
             
     ## test refinement for 2023
     simulation_state.set_current_time(2023)
     hhs = self.dataset_pool.get_dataset('household')
     
     hhs_p5 = hhs.compute_variables('household.persons>5')
     
     #check results
     self.assert_(hhs.size(),  2)
     self.assertEqual(hhs_p5.sum(), 0)
     
     self.dataset_pool.remove_all_datasets()
     
     ## test refinement for 2027
     simulation_state.set_current_time(2027)
     hhs = self.dataset_pool.get_dataset('household')
     buildings = self.dataset_pool.get_dataset('building')
     persons = self.dataset_pool.get_dataset('person')
     
     
     hhs_raz6 = hhs.compute_variables('household.disaggregate(building.disaggregate(parcel.raz_id)==6)', 
                                           dataset_pool=self.dataset_pool)
     hhs_bldg = buildings.compute_variables('building.number_of_agents(household)', 
                                                 dataset_pool=self.dataset_pool)
     
     #check results
     self.assertEqual(hhs_raz6.sum(), 3)
     self.assert_(hhs_bldg.sum(),  3 )
     self.assert_((hhs_bldg!=0).sum(),  2)
     self.assert_(allclose(persons.get_attribute('job_id'), array([-1,  -1, -1, -1,  3,  4,  7])))
Example #25
0
    def run(self, base_directory, urbansim_cache_directory, years):
        """ run the simulation
                base_directory: directory contains all years folder of lccm.
                urbansim_cache_directory: directory contains all years folder of urbansim cache.
                years: lists of year to run."""
        model = LandCoverChangeModel(self.possible_lcts, submodel_string=self.lct_attribute, 
                                     choice_attribute_name= self.lct_attribute, debuglevel=4)
        coefficients = Coefficients()
        storage = StorageFactory().get_storage('tab_storage', 
            storage_location=os.path.join(self.package_path, 'data'))
        coefficients.load(in_storage=storage, in_table_name="land_cover_change_model_coefficients")
        specification = EquationSpecification(in_storage=storage)
        specification.load(in_table_name="land_cover_change_model_specification")
        specification.set_variable_prefix("biocomplexity.land_cover.")
        constants = Constants()
        simulation_state = SimulationState()
        simulation_state.set_cache_directory(urbansim_cache_directory)
        attribute_cache = AttributeCache()
        index = arange(100000)
        for year in years:
            simulation_state.set_current_time(year)
            #land_cover_path = os.path.join(base_directory, str(year))
            land_cover_path = base_directory
            land_covers = LandCoverDataset(in_storage=StorageFactory().get_storage('flt_storage', storage_location=land_cover_path),
                                       out_storage=StorageFactory().get_storage('flt_storage', storage_location=land_cover_path),
                                       debuglevel=4)
            land_covers.subset_by_index(index)
            #land_covers.load_dataset()
            gridcells = GridcellDataset(in_storage=attribute_cache, debuglevel=4)

            agents_index = None
            model.run(specification, coefficients, land_covers, data_objects={"gridcell":gridcells,
                          "constants":constants, "flush_variables":True},
                          chunk_specification = {'nchunks':1}
                          )
            land_covers.flush_dataset()
            del gridcells
            del land_covers
Example #26
0
    def run(self, table_names, out_storage=None, table_name_pattern=None, cache_directory=None, year=None, **kwargs):
        """
        export specified tables to database

        table_name_pattern: For example '{table_name}_{scenario_name}_{year}'
        """
        if not hasattr(self, "out_storage"):
            if out_storage is None:
                raise ValueError, "Either out_storage argument needs to be specified or " + "prepare_for_run called before run method to create a valid out_storage."
            else:
                self.out_storage = out_storage
        sim_state = SimulationState()
        if sim_state.get_current_time() == 0:
            sim_state.set_current_time(9999)
        if cache_directory is None:
            cache_directory = sim_state.get_cache_directory()

        attr_cache = AttributeCache(cache_directory=cache_directory)
        if year is None:
            years = attr_cache._get_sorted_list_of_years()
        else:
            assert isinstance(year, int)
            years = [year]

        for table_name in table_names:
            kwargs["table_name"] = table_name
            for year in years:
                kwargs["year"] = year
                out_table_name = table_name_pattern.format(**kwargs)
                in_storage = attr_cache.get_flt_storage_for_year(year)
                # cache_path = os.path.join(cache_directory, str(year))
                # in_storage = flt_storage(storage_location=cache_path)
                # TODO drop_table(table_name) if table_name exists
                ExportStorage().export_dataset(
                    table_name, in_storage=in_storage, out_storage=self.out_storage, out_dataset_name=out_table_name
                )
        self.post_run(kwargs["scenario_name"], years)
 def run(self, year, matrix_directory=None):
     """This is the main entry point.  It gets the appropriate values from the 
     travel_model_configuration part of this config, and then copies the specified 
     emme/2 matrices into the specified travel_data variable names.  Results in
     a new travel_data cache for year+1.
     If matrix_directory is not None, it is assumed the matrices files are already created 
     in the given directory.
     """
     cache_directory = self.config['cache_directory']
     simulation_state = SimulationState()
     simulation_state.set_current_time(year)
     simulation_state.set_cache_directory(cache_directory)
     
     year_config = self.config['travel_model_configuration'][year]
     matrices_created = False
     if matrix_directory is not None:
         matrices_created = True    
     reports = self.config['travel_model_configuration'].get('reports_to_copy', [])
     
     for x in 1,2,3:
         if matrix_directory is None:
             bank_dir = self.get_emme2_dir(year, "bank%i" % x)
         else:
             bank_dir = os.path.join(matrix_directory, "bank%i" % x)
         if "bank%i" % x in year_config['matrix_variable_map']:
             self.get_needed_matrices_from_emme2(year, 
                                             year_config['cache_directory'],
                                             bank_dir,
                                             year_config['matrix_variable_map']["bank%i" % x],
                                                 matrices_created)
             for report in reports:
                 self.copy_report_to_cache(report, year, year_config['cache_directory'], bank_dir)
         if "bank%i" % x in year_config.get('node_matrix_variable_map', {}):
             node_variable_map = year_config['node_matrix_variable_map']["bank%i" % x]
             if len(node_variable_map.keys()) > 0:
                 self.get_needed_node_matrices_from_emme2(year, year_config['cache_directory'], bank_dir, node_variable_map)
Example #28
0
    def __init__(self, config=None, save_estimation_results=False):
        if 'cache_directory' not in config or config['cache_directory'] is None:
            raise KeyError("The cache directory must be specified in the "
                "given configuration, giving the filesystem path to the cache "
                "directory containing the data with which to estimate. Please "
                "check that your configuration contains the 'cache_directory' "
                "entry and that it is not None.")

        self.simulation_state = SimulationState(new_instance=True)
        self.simulation_state.set_cache_directory(config['cache_directory'])

        SessionConfiguration(new_instance=True,
                             package_order=config['dataset_pool_configuration'].package_order,
                             in_storage=AttributeCache())
        self.config = Resources(config)
        self.save_estimation_results = save_estimation_results
        self.debuglevel = self.config.get("debuglevel", 4)
        self.model_system = ModelSystem()
        self.agents_index_for_prediction = None
        
        models = self.config.get('models',[])

        self.model_name = None
        if "model_name" in config.keys():
            self.model_name = config["model_name"]
        else:
            for model in models:
                if isinstance(model, dict):
                    model_name = model.keys()[0]
                    if (model[model_name] == "estimate") or (isinstance(model[model_name], list)
                        and ("estimate" in model[model_name])):
                            self.model_name = model_name
                            break
        estimate_config_changes = self.config.get('config_changes_for_estimation', {}).get('estimate_config', {})
        if len(estimate_config_changes) > 0:
            change = Resources({'models_configuration': {self.model_name: {'controller': {'init': {'arguments': {}}}}}})
            estimate_config_str = self.config['models_configuration'].get(self.model_name, {}).get('controller', {}).get('init', {}).get('arguments', {}).get('estimate_config', '{}')
            estimate_config = Resources({})
            try:
                estimate_config = eval(estimate_config_str)
            except:
                pass
 
            estimate_config.merge(estimate_config_changes)
            self.config.merge(change)
            self.config['models_configuration'][self.model_name]['controller']['init']['arguments']['estimate_config'] = 'Resources(%s)' % estimate_config
    def test_my_inputs(self):
        tester = VariableTester(__file__,
                                package_order=['urbansim_parcel', 'urbansim'],
                                test_data={
                                    'building': {
                                        'building_id':
                                        array([1, 2, 3, 4]),
                                        'year_built':
                                        array([1995, 1800, 2006, 200])
                                    },
                                    'urbansim_constant': {
                                        'absolute_min_year': array([1800])
                                    }
                                })

        SimulationState().set_current_time(2005)
        should_be = array([True, True, True, False])
        tester.test_is_equal_for_variable_defined_by_this_module(
            self, should_be)
    def test_err_duplicate_data(self):
        weather_exogenous_override = {
            'id': array([1, 4]),
            'year': array([1980, 1980]),
            self.base_id: array([1, 1]),  # Both 1980, both base_table_id 1.
            self.exogenous_attribute1: array([40, 70]),
            self.exogenous_attribute2: array([700, 1000]),
        }
        self.storage.write_table(table_name=self.weather_exogenous_table_name,
                                 table_data=weather_exogenous_override)

        base_dataset = Dataset(in_storage=self.storage,
                               in_table_name=self.base_table_name,
                               id_name=self.base_id)
        ExogenousAspectForDataset().apply(base_dataset)

        SimulationState().set_current_time(1980)
        self.assertRaises(AttributeError, base_dataset.get_attribute,
                          self.exogenous_attribute1)
Example #31
0
    def test_delete_computed_tables_if_nothing_to_delete(self):
        year = 1980
        tables_keep = ['test_table1', 'test_table2']
        tables_delete = []
        for table in tables_keep + tables_delete:
            os.makedirs(os.path.join(self.temp_dir, str(year), table))
            open(os.path.join(self.temp_dir, str(year), table, 'attr1.i16'),
                 'w').close()
            open(os.path.join(self.temp_dir, str(year), table, 'attr2.i32'),
                 'w').close()
        SimulationState().set_current_time(year)
        #check if all tables exist before the test
        for table in tables_keep + tables_delete:
            self.assert_(os.path.exists(self.get_table_path(year, table)))
        self.storage.delete_computed_tables()

        #check if the right tables exist
        for table in tables_keep:
            self.assert_(os.path.exists(self.get_table_path(year, table)))
            def _write_data_to_year(self, data, cache_dir, year):
                """Writes this data to this year of the cache.  Returns dataset.
                """
                # Import in unit test, so that circular dependency is avoided.
                from opus_core.datasets.dataset import Dataset

                SimulationState().set_cache_directory(cache_dir)

                storage = dict_storage()
                storage.write_table(
                    table_name=self._table_name,
                    table_data=data,
                )

                ds = Dataset(id_name=self._id_name,
                             in_storage=storage,
                             in_table_name=self._table_name)
                ds.load_dataset()
                self._write_dataset_to_cache(ds, cache_dir, year)
    def prepare_for_simulation(self, config, cache_directory=None):
        self.config = Resources(config)
        base_cache_dir = self.config['creating_baseyear_cache_configuration'].cache_directory_root
        
        self.simulation_state = SimulationState(new_instance=True, base_cache_dir=base_cache_dir)

        ### TODO: Get rid of this! There is no good reason to be changing the 
        ###       Configuration.
        if self.config['cache_directory'] is None:
            self.config['cache_directory'] = self.simulation_state.get_cache_directory()

        SessionConfiguration(new_instance=True,
                             package_order=self.config['dataset_pool_configuration'].package_order,
                             in_storage=AttributeCache())
        
        if config['creating_baseyear_cache_configuration'].cache_from_database:
            ForkProcess().fork_new_process(self.config['creating_baseyear_cache_configuration'].cache_scenario_database, self.config)
        else:
            CacheFltData().run(self.config)
Example #34
0
 def _get_sorted_list_of_years(self):
     """Returns a sorted list (descending order) of the current and prior years 
     having directories in the cache directory.
     """
     from os import listdir
     current_year = SimulationState().get_current_time()
     dirs = flt_storage(
         self.get_storage_location()).listdir_in_base_directory()
     years = []
     for dir_name in dirs:
         try:
             year = int(dir_name)
             if (year <= current_year):
                 years.append(year)
         except:
             pass
     years.sort()
     years.reverse()
     return years
    def setUp(self):
        self.base_table_name = 'base_table'
        self.base_id = 'base_table_id'
        self.exogenous_attribute1 = 'some_attribute'
        self.exogenous_attribute2 = 'some_other_attribute'

        base_table = {
            self.base_id: array([2, 1]),
        }

        self.weather_exogenous_table_name = 'weather_exogenous'
        weather_exogenous = {
            'id': array([1, 2, 3, 4, 5, 6]),
            'year': array([1980, 1981, 1982, 1980, 1981, 1982]),
            self.base_id: array([1, 1, 1, 2, 2, 2]),
            self.exogenous_attribute1: array([40, 50, 600, 70, 80, 900]),
            self.exogenous_attribute2: array([700, 800, 90, 1000, 1100, 120]),
        }

        SimulationState().set_current_time(1980)
        self.expected_exogenous_attribute_1980_1 = array([70, 40])
        self.expected_exogenous_attribute_1980_2 = array([1000, 700])
        self.expected_exogenous_attribute_1981_1 = array([80, 50])
        self.expected_exogenous_attribute_1981_2 = array([1100, 800])
        self.expected_exogenous_attribute_1982_1 = array([900, 600])
        self.expected_exogenous_attribute_1982_2 = array([120, 90])

        self.exogenous_relationships_table_name = 'exogenous_relationships'
        exogenous_relationships = {
            'exogenous_id': array([1]),
            'base_table': array([self.base_table_name]),
            'exogenous_table': array([self.weather_exogenous_table_name]),
        }

        self.storage = dict_storage()

        for table_name, table_values in [
            (self.base_table_name, base_table),
            (self.weather_exogenous_table_name, weather_exogenous),
            (self.exogenous_relationships_table_name, exogenous_relationships)
        ]:
            self.storage.write_table(table_name=table_name,
                                     table_data=table_values)
Example #36
0
    def target_func(self,
                    est_v,
                    func=lambda x, y: np.sum(np.abs(x - y)),
                    **kwargs):
        ''' Target function.'''

        simulation_state = SimulationState()
        simulation_state.set_current_time(self.base_year)
        simulation_state.set_cache_directory(self.cache_directory)
        attribute_cache = AttributeCache()
        dataset_pool = SessionConfiguration(
            new_instance=True,
            package_order=self.package_order,
            in_storage=attribute_cache).get_dataset_pool()

        calib_datasets = {}
        for dataset_name, calib_attr in calib_datasets.iteritems():
            dataset = dataset_pool.get_dataset(
                dataset_name, dataset_arguments={'id_name': []})
            assert subset is None or subset.get(dataset_name, None) is None or \
                   subset_patterns is None or subset_patterns.get(dataset_name, None) is None
            if subset is not None and subset.get(dataset_name,
                                                 None) is not None:
                subset_attr, subset_cond = subset.get(dataset_name)
                index = np.in1d(dataset[subset_attr], subset_cond)
            elif subset_patterns is not None and subset_patterns.get(
                    dataset_name, None) is not None:
                subset_attr, subset_pattern = subset_patterns.get(dataset_name)
                index = array([
                    True if re.search(subset_pattern, attr_v) else False
                    for attr_v in dataset[subset_attr]
                ])
            else:
                index = arange(dataset.size(), dtype='i')

            calib_datasets[dataset_name] = [dataset, calib_attr, index]

        prediction = self.update_prediction(est_v, simulation_state,
                                            dataset_pool, calib_datasets,
                                            **kwargs)
        ## allow keys in target not appearing in prediction
        ## assuming their values to be 0
        ### every key in target should appear in prediction
        #assert np.all( np.in1d(self.target.keys(), prediction.keys()) )
        target = np.array(self.target.values())
        predct = np.array([prediction[k] if prediction.has_key(k) else 0 \
                           for k in self.target.keys() ])
        results = func(predct, target)

        return results
Example #37
0
    def run(self, year):
        """This is the main entry point.  The class is initialized with the appropriate configuration info from the 
        travel_model_configuration part of this config, and then copies the specified 
        UrbanSim data into files for emme/2 to read.  
        If households and jobs do not have a primary attribute zone_id, the entry 'locations_to_disaggregate'
        in the travel_model_configuration should be a list of dataset names over which the zone_id 
        will be dissaggregated, ordered from higher to lower aggregation level, e.g. ['parcel', 'building']
        """
        cache_directory = self.config['cache_directory']
        simulation_state = SimulationState()
        simulation_state.set_cache_directory(cache_directory)
        simulation_state.set_current_time(year)
        attribute_cache = AttributeCache()
        sc = SessionConfiguration(
            new_instance=True,
            package_order=self.config['dataset_pool_configuration'].
            package_order,
            in_storage=attribute_cache)
        dataset_pool = sc.get_dataset_pool()

        hh_set = dataset_pool.get_dataset('household')
        zone_set = dataset_pool.get_dataset('zone')
        job_set = dataset_pool.get_dataset('job')
        locations_to_disaggregate = self.config['travel_model_configuration'][
            'locations_to_disaggregate']
        len_locations_to_disaggregate = len(locations_to_disaggregate)
        if len_locations_to_disaggregate > 0:
            primary_location = locations_to_disaggregate[0]
            if len_locations_to_disaggregate > 1:
                intermediates_string = ", intermediates=["
                for i in range(1, len_locations_to_disaggregate):
                    intermediates_string = "%s%s, " % (
                        intermediates_string, locations_to_disaggregate[i])
                intermediates_string = "%s]" % intermediates_string
            else:
                intermediates_string = ""
            hh_set.compute_variables([
                '%s = household.disaggregate(%s.%s %s)' %
                (zone_set.get_id_name()[0], primary_location,
                 zone_set.get_id_name()[0], intermediates_string)
            ],
                                     dataset_pool=dataset_pool)
            job_set.compute_variables([
                '%s = job.disaggregate(%s.%s %s)' %
                (zone_set.get_id_name()[0], primary_location,
                 zone_set.get_id_name()[0], intermediates_string)
            ],
                                      dataset_pool=dataset_pool)

        return self._call_input_file_writer(year, dataset_pool)
Example #38
0
    def run_in_one_process(
            self,
            resources,
            run_in_background=False,
            class_path='opus_core.model_coordinators.model_system'):
        resources = Resources(resources)
        if resources['cache_directory'] is not None:
            cache_directory = resources['cache_directory']
        else:
            cache_directory = SimulationState().get_cache_directory()

        ### TODO: Get rid of this! There is no good reason to be changing the
        ###       Configuration.
        resources['cache_directory'] = cache_directory

        self._fork_new_process('%s' % class_path,
                               resources,
                               delete_temp_dir=False,
                               run_in_background=run_in_background)
        self._notify_stopped()
def opusRun(progressCB, logCB, params):

    params_dict = {}
    for key, val in params.iteritems():
        params_dict[str(key)] = str(val)

    opus_data_directory = params_dict['opus_data_directory']
    opus_data_year = params_dict['opus_data_year']
    database_name = params_dict['database_name']
    table_name = params_dict['table_name']
    database_server_connection = params_dict['database_server_connection']

    dbs_config = DatabaseServerConfiguration(
        database_configuration=database_server_connection)
    server = DatabaseServer(database_server_configuration=dbs_config)
    opusdb = server.get_database(database_name=database_name)

    input_storage = sql_storage(storage_location=opusdb)

    attribute_cache = AttributeCache(cache_directory=opus_data_directory)
    output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)
    SimulationState().set_current_time(opus_data_year)
    SessionConfiguration(new_instance=True,
                         package_order=[],
                         in_storage=AttributeCache())

    if table_name == 'ALL':
        logCB('caching all tables...\n')
        lst = input_storage.get_table_names()
        for i in lst:
            ExportStorage().export_dataset(
                dataset_name=i,
                in_storage=input_storage,
                out_storage=output_storage,
            )
    else:
        logCB("Exporting table '%s' to year %s of cache located at %s...\n" %
              (table_name, opus_data_year, opus_data_directory))
        ExportStorage().export_dataset(dataset_name=table_name,
                                       in_storage=input_storage,
                                       out_storage=output_storage)
    def compute(self, dataset_pool):
        ds = self.get_dataset()
        current_year = SimulationState().get_current_time()
        velocity = ds.get_attribute("_velocity")
        index = current_year - ds.get_attribute("_start_year")

        def get_one_velocity(velocity_string, idx):
            a = array(eval(velocity_string))
            if idx >= a.size:
                return 0
            if idx == 0:
                return a[idx]
            else:
                return a[idx] - a[idx - 1]

        percent_of_development_this_year = array(
            map(lambda vel, i: get_one_velocity(vel, i), velocity, index))
        ds.touch_attribute(
            "_start_year"
        )  # in order to always recompute (because the simulation year can change)
        return percent_of_development_this_year
Example #41
0
    def test_simple_lag_variable2(self):
        test_data = {
            1000: {
                'tests': {
                    'id': array([1, 2, 3, 4]),
                    'attr1': array([10, 20, 30, 40]),
                },
            },
            1001: {
                'tests': {
                    'id': array([1, 2, 3, 5]),
                    'attr1': array([111, 222, 333, 555]),
                },
            },
        }
        cache_creator = CreateTestAttributeCache()
        cache_creator.create_attribute_cache_with_data(self._temp_dir,
                                                       test_data)

        SimulationState().set_current_time(1001)

        attribute_cache = AttributeCache()
        SessionConfiguration(new_instance=True,
                             package_order=['opus_core'],
                             in_storage=attribute_cache)

        ds = Dataset(in_storage=attribute_cache,
                     in_table_name='tests',
                     id_name=['id'],
                     dataset_name='tests')

        ds.compute_variables(['opus_core.tests.attr1'])
        self.assert_(
            ma.allequal(ds.get_attribute('attr1'), array([111, 222, 333,
                                                          555])))

        ds.compute_variables(['opus_core.tests.attr1_lag1'])
        self.assert_(
            ma.allequal(ds.get_attribute('attr1_lag1'),
                        array([10, 20, 30, 555])))
Example #42
0
    def run(self, config, show_output=False):
        logger.log_status("Caching large SQL tables to: " +
                          config['cache_directory'])
        self.show_output = show_output

        #import pydevd;pydevd.settrace()

        server_configuration = config['scenario_database_configuration']

        scenario_database_manager = ScenarioDatabaseManager(
            server_configuration=server_configuration,
            base_scenario_database_name=server_configuration.database_name)

        self.database_server = DatabaseServer(server_configuration)

        database_to_table_mapping = scenario_database_manager.get_database_to_table_mapping(
        )

        self.tables_to_cache = config[
            'creating_baseyear_cache_configuration'].tables_to_cache

        simulation_state = SimulationState()
        if 'low_memory_run' in config:
            simulation_state.set_low_memory_run(config['low_memory_run'])
        simulation_state.set_cache_directory(config['cache_directory'])
        simulation_state.set_current_time(config['base_year'])

        self.tables_cached = set()
        for database_name, tables in database_to_table_mapping.items():
            self.cache_database_tables(config, database_name, tables)

        un_cached_tables = set(self.tables_to_cache) - self.tables_cached
        if un_cached_tables:
            logger.log_warning(
                'The following requested tables were NOT cached:')
            for table_name in un_cached_tables:
                logger.log_warning('\t%s' % table_name)
    def test_my_inputs(self):
        SimulationState().set_current_time(2005)
        tester = VariableTester(
            __file__,
            package_order=['urbansim'],
            test_data={
                "gridcell": { 
                    "grid_id": array([1,2,3])
                    },
                "building": {
                    "building_id": array([1,2,3,4,5,6,7]),
                    "grid_id":     array([1,    1,     2,  3, 1,  2,  1]),
                    "year_built":  array([1995, 2000, 2005, 0, 10, 0, 2005])
                    },
                'urbansim_constant':{
                    "absolute_min_year": array([1800]),
                }
            }
        )

        should_be = array([True, True, False]) 
        tester.test_is_equal_for_variable_defined_by_this_module(self, should_be)
Example #44
0
    def run_multiprocess(self, resources):
        resources = Resources(resources)
        profiler_name = resources.get("profile_filename", None)
        if resources['cache_directory'] is not None:
            cache_directory = resources['cache_directory']
        else:
            cache_directory = SimulationState().get_cache_directory()

        ### TODO: Get rid of this! There is absolutely no good reason to be
        ###       changing the Configuration!
        resources['cache_directory'] = cache_directory

        log_file = os.path.join(cache_directory, 'run_multiprocess.log')
        logger.enable_file_logging(log_file)

        start_year = resources["years"][0]
        end_year = resources["years"][-1]
        nyears = end_year - start_year + 1
        root_seed = resources.get("seed", NO_SEED)
        if resources.get('_seed_dictionary_', None) is not None:
            # This is added by the RunManager to ensure reproducibility including restarted runs
            seed_dict = resources.get('_seed_dictionary_')
            seed_array = array(
                map(lambda year: seed_dict[year],
                    range(start_year, end_year + 1)))
        else:
            seed(root_seed)
            seed_array = randint(1, 2**30, nyears)
        logger.log_status("Running simulation for years %d thru %d" %
                          (start_year, end_year))
        logger.log_status("Simulation root seed: %s" % root_seed)

        self._run_each_year_as_separate_process(start_year, end_year,
                                                seed_array, resources)

        if profiler_name is not None:  # insert original value
            resources["profile_filename"] = profiler_name
        logger.log_status("Done running simulation for years %d thru %d" %
                          (start_year, end_year))
    def test_correct_land_value(self):
        #TODO: need to remove this when fixed LP correction only working when year >= 2002
        from opus_core.simulation_state import SimulationState
        SimulationState().set_current_time(2002)
        
        storage = StorageFactory().get_storage('dict_storage')

        gridcell_set_table_name = 'gridcell_set'        
        storage.write_table(
            table_name=gridcell_set_table_name,
            table_data={
                "percent_residential_within_walking_distance":array([30, 0, 90, 100]),
                "gridcell_year_built":array([2002, 1968, 1880, 1921]),
                "fraction_residential_land":array([0.5, 0.1, 0.3, 0.9]),
                "residential_land_value":array([0, 0, 0, 0]),                          
                "residential_land_value_lag1":array([15000, 0, 7500, 0]),
                "nonresidential_land_value":array([0, 0, 0, 0]),  
                "nonresidential_land_value_lag1":array([15000, 0, 17500, 0]),                                                                    
                "development_type_id":array(  [2, 1, 1, 1]),
                "development_type_id_lag2":array(  [1, 1, 1, 1]),
                "grid_id": array([1,2,3,4])
                }
            )

        gridcell_set = GridcellDataset(in_storage=storage, in_table_name=gridcell_set_table_name)

        specification = EquationSpecification(variables=(
            "percent_residential_within_walking_distance", 
            "gridcell_year_built", "constant"), 
            coefficients=("PRWWD", "YB", "constant"))
        coefficients = Coefficients(names=("constant", "PRWWD", "YB"), values=(10.0, -0.0025, 0.0001))
        lp = LandPriceModel(filter=None, debuglevel=3)
        lp.run(specification, coefficients, gridcell_set)
        correctmodel = CorrectLandValue()
        correctmodel.run(gridcell_set)
        result1 = gridcell_set.get_attribute("residential_land_value")
        result2 = gridcell_set.get_attribute("nonresidential_land_value")
        self.assertEqual(ma.allclose(result1, array([15000.0,  2681.723,  6367.914, 18708.617]), rtol=1e-3), True)
        self.assertEqual(ma.allclose(result2, array([15000.0,  24135.510, 14858.466,  2078.735]), rtol=1e-3), True)
Example #46
0
def opusRun(progressCB, logCB, params):

    params_dict = {}
    for key, val in params.iteritems():
        params_dict[str(key)] = str(val)

    esri_data_path = params_dict['esri_data_path']
    esri_table_name = params_dict['esri_table_name']
    opus_data_directory = params_dict['opus_data_directory']
    opus_data_year = params_dict['opus_data_year']

    input_storage = esri_storage(storage_location=esri_data_path)
    attribute_cache = AttributeCache(cache_directory=opus_data_directory)
    output_storage = attribute_cache.get_flt_storage_for_year(opus_data_year)

    SimulationState().set_current_time(opus_data_year)
    SessionConfiguration(new_instance=True,
                         package_order=[],
                         in_storage=AttributeCache())

    if esri_table_name == 'ALL':
        logCB("Sending all tables to OPUS storage...\n")
        lst = input_storage.get_table_names()
        for i in lst:
            ExportStorage().export_dataset(
                dataset_name=i,
                in_storage=input_storage,
                out_storage=output_storage,
            )

    else:
        logCB("Exporting table '%s' to OPUS storage located at %s...\n" %
              (esri_table_name, opus_data_directory))
        ExportStorage().export_dataset(
            dataset_name=esri_table_name,
            in_storage=input_storage,
            out_storage=output_storage,
        )
        logCB("Finished exporting table '%s'\n" % (esri_table_name))
    def test_my_inputs2(self):
        tester = VariableTester(__file__,
                                package_order=['urbansim_parcel', 'urbansim'],
                                test_data={
                                    'development_project_proposal': {
                                        "proposal_id":
                                        array([1, 2, 3, 4, 5]),
                                        "start_year":
                                        array([2005, 2007, 2005, 2006, 2006])
                                    },
                                    'development_project_proposal_component': {
                                        "proposal_component_id":
                                        arange(8) + 1,
                                        "proposal_id":
                                        array([3, 3, 5, 2, 5, 1, 3, 1]),
                                        "component_id":
                                        array([3, 1, 4, 2, 4, 1, 3, 4]),
                                        "annual_construction_schedule":
                                        array([
                                            "[0, 50, 100]", "[100]",
                                            "[25, 50, 75, 100]",
                                            "[0, 50, 100]", "[0, 50, 100]",
                                            "[25, 50, 75, 100]",
                                            "[25, 50, 75, 100]", "[100]"
                                        ])
                                    },
                                    'development_template_component': {
                                        "component_id": array([1, 2, 3, 4]),
                                        "velocity_function_id":
                                        array([2, 1, 1, 3])
                                    },
                                })
        SimulationState().set_current_time(2007)
        should_be = array([50, 0, 25, 0, 50, 25, 25, 0])

        tester.test_is_equal_for_variable_defined_by_this_module(
            self, should_be)
Example #48
0
    def test_gridcell_unrolling(self):
        """Checks that the unrolling of the gridcells by CacheScenarioDatabase worked correctly.
        """
        cache_directory = SimulationState().get_cache_directory()
        gridcells = SessionConfiguration().get_dataset_from_pool('gridcell')
        development_event_history = SessionConfiguration(
        ).get_dataset_from_pool('development_event_history')
        unroller = UnrollGridcells()
        unroller.unroll_gridcells_to_cache(gridcells,
                                           development_event_history,
                                           cache_directory, self.base_year)

        self.assertEqual(self.temp_dir, os.path.split(cache_directory)[0])

        gridcell = {}
        for year in [1976, 1977, 1979, 1980]:
            #current_year = SimulationState().get_current_time()
            #SimulationState().set_current_time(year)
            #gridcell[year] = SessionConfiguration().get_dataset_from_pool('gridcell')
            #SimulationState().set_current_time(current_year)
            flt_directory = os.path.join(cache_directory, str(year))
            gridcell[year] = DatasetFactory().get_dataset(
                'gridcell',
                package='urbansim',
                subdir='datasets',
                arguments={
                    'in_storage':
                    StorageFactory().get_storage(
                        'flt_storage', storage_location=flt_directory)
                })
        diff = gridcell[1980].get_attribute('residential_units') - gridcell[
            1979].get_attribute('residential_units')
        self.assertEqual(1, sum(diff))
        diff = gridcell[1977].get_attribute('commercial_sqft') - gridcell[
            1976].get_attribute('commercial_sqft')
        self.assertEqual(2255 + 199 + 332 + 2785, sum(diff))
Example #49
0
    def run(self, prefix='', suffix='', dataset_pool=None, alt_location=None, target_location=None): 
        """
        prefix, suffix - 'YEAR' is a magic word in prefix/suffix which will be replaced with 
        the current run year number, for example,
           - suffix='_YEAR' will swap 'zoning_2020' to 'zoning' in simulation year 2020, 
                                 and, if existing, 'zoning' to 'zoning_ORIGINAL' 
        by default, only search for datasets in the cache directory for current year
           - alt_location: alternative location to search for datasets of specified patterns,
                           for example, base_year_cache

           - target_location: where to put the datasets to be renamed
        """
        current_year = SimulationState().get_current_time()
        self.prefix = re.sub('YEAR', str(current_year), prefix) 
        self.suffix = re.sub('YEAR', str(current_year), suffix)
        
        if target_location is not None:
            target_location = target_location.get_storage_location()

        locations = []
        if dataset_pool is not None:
            storage = dataset_pool.get_storage().get_flt_storage_for_year(current_year)
            location = storage.get_storage_location()
            locations.append(location)
            if target_location is None:
                target_location = location

        if alt_location is not None:
            location = alt_location.get_storage_location()
            locations.append(location)
            if target_location is None:
                target_location = location

        for pattern in [os.path.join(location, '%s*%s' % (prefix, suffix)) 
                        for location in locations]:
            self._rename_datasets(pattern, target_location)
Example #50
0
    def test_my_inputs(self):
        SimulationState().set_current_time(2005)
        tester = VariableTester(
            __file__,
            package_order=['urbansim'],
            test_data={
                "gridcell": {
                    "grid_id": array([1, 2, 3])
                },
                "building": {
                    "building_id": array([1, 2, 3, 4, 5, 6, 7]),
                    "grid_id": array([1, 1, 2, 3, 1, 2, 1]),
                    "year_built": array([1995, 2000, 2005, 0, 10, 0, 2005]),
                    "is_building_type_commercial": array([0, 1, 0, 0, 0, 1, 1])
                },
                'urbansim_constant': {
                    "absolute_min_year": array([1800]),
                }
            })

        should_be = array([True, False, False])
        instance_name = "urbansim.gridcell.has_commercial_buildings_with_year_built"
        tester.test_is_equal_for_family_variable(self, should_be,
                                                 instance_name)
Example #51
0
    def test_compute_unloads_from_memory(self):

        storage = StorageFactory().get_storage('dict_storage')

        storage.write_table(table_name='tests',
                            table_data={
                                'a_dependent_variable': array([1, 5, 10]),
                                'id': array([1, 3, 4])
                            })

        dataset = Dataset(in_storage=storage,
                          in_table_name='tests',
                          id_name='id',
                          dataset_name='tests')

        SessionConfiguration(in_storage=storage)["flush_variables"] = True
        dataset.get_attribute("a_dependent_variable")
        self.assert_(
            "a_dependent_variable" in dataset.get_attributes_in_memory())
        dataset.compute_variables("opus_core.tests.a_test_variable")
        self.assert_(
            "a_dependent_variable" not in dataset.get_attributes_in_memory())
        self.assert_("a_test_variable" in dataset.get_attributes_in_memory())
        SimulationState().remove_singleton(delete_cache=True)
Example #52
0
def setup_environment(cache_directory,
                      year,
                      package_order,
                      additional_datasets={}):
    gc.collect()
    ss = SimulationState(new_instance=True)
    ss.set_cache_directory(cache_directory)
    ss.set_current_time(year)
    ac = AttributeCache()
    storage = ac.get_flt_storage_for_year(year)
    sc = SessionConfiguration(new_instance=True,
                              package_order=package_order,
                              in_storage=ac)
    logger.log_status(
        "Setup environment for year %s. Use cache directory %s." %
        (year, storage.get_storage_location()))
    dp = sc.get_dataset_pool()
    for name, ds in additional_datasets.iteritems():
        dp.replace_dataset(name, ds)
    return dp
Example #53
0
    def run(self, year, matrix_directory=None):
        """This is the main entry point.  It gets the appropriate values from the 
        travel_model_configuration part of this config, and then copies the specified 
        emme/2 matrices into the specified travel_data variable names.  Results in
        a new travel_data cache for year+1.
        If matrix_directory is not None, it is assumed the matrices files are already created 
        in the given directory.
        """
        cache_directory = self.config['cache_directory']
        simulation_state = SimulationState()
        simulation_state.set_current_time(year)
        simulation_state.set_cache_directory(cache_directory)

        year_config = self.config['travel_model_configuration'][year]
        matrices_created = False
        if matrix_directory is not None:
            matrices_created = True
        reports = self.config['travel_model_configuration'].get(
            'reports_to_copy', [])

        for x in 1, 2, 3:
            if matrix_directory is None:
                bank_dir = self.get_emme2_dir(year, "bank%i" % x)
            else:
                bank_dir = os.path.join(matrix_directory, "bank%i" % x)
            if "bank%i" % x in year_config['matrix_variable_map']:
                self.get_needed_matrices_from_emme2(
                    year, year_config['cache_directory'], bank_dir,
                    year_config['matrix_variable_map']["bank%i" % x],
                    matrices_created)
                for report in reports:
                    self.copy_report_to_cache(report, year,
                                              year_config['cache_directory'],
                                              bank_dir)
            if "bank%i" % x in year_config.get('node_matrix_variable_map', {}):
                node_variable_map = year_config['node_matrix_variable_map'][
                    "bank%i" % x]
                if len(node_variable_map.keys()) > 0:
                    self.get_needed_node_matrices_from_emme2(
                        year, year_config['cache_directory'], bank_dir,
                        node_variable_map)
Example #54
0
    def _compute_variable_for_prior_year(self,
                                         dataset,
                                         full_name,
                                         time,
                                         resources=None):
        """Create a new dataset for this variable, compute the variable, and then return
        the values for this variable."""
        calling_dataset_pool = SessionConfiguration().get_dataset_pool()
        calling_time = SimulationState().get_current_time()
        SimulationState().set_current_time(time)
        # Do not flush any variables when computing dependencies for a lag variable.
        prior_flush_state = SimulationState().get_flush_datasets()
        SimulationState().set_flush_datasets(False)
        try:
            # Get an empty dataset pool with same search paths.
            my_dataset_pool = DatasetPool(
                package_order=calling_dataset_pool.get_package_order(),
                storage=AttributeCache())

            try:
                ds = dataset.empty_dataset_like_me(in_storage=AttributeCache())
            except FileNotFoundError:
                ## necessary when a dataset is not cached, but created on-the-fly, e.g submarket
                ds = my_dataset_pool.get_dataset(dataset.dataset_name)

            # Don't pass any datasets via resources, since they may be from a different time.
            my_resources = Resources(resources)
            for key in my_resources:
                if isinstance(key, Dataset):
                    del my_resources[key]

            ds.compute_variables(full_name,
                                 my_dataset_pool,
                                 resources=my_resources)
            values = ds.get_attribute(full_name)
            return values
        finally:
            SimulationState().set_current_time(calling_time)
            SimulationState().set_flush_datasets(prior_flush_state)
    def run(self, config, year, *args, **kwargs):
        """This is the main entry point.  It gets the appropriate values from the 
        travel_model_configuration part of this config, and then copies the specified 
        data into the specified travel_data variable names.  Results in
        a new travel_data cache for year+1.
        """
        cache_directory = config['cache_directory']
        simulation_state = SimulationState()
        simulation_state.set_current_time(year)
        simulation_state.set_cache_directory(cache_directory)
        
        logger.start_block('Getting data from travel model')
        next_year = year + 1
        flt_dir_for_next_year = os.path.join(cache_directory, str(next_year))
        if not os.path.exists(flt_dir_for_next_year):
            os.mkdir(flt_dir_for_next_year)
        attribute_cache = AttributeCache()        
        dataset_pool = SessionConfiguration(new_instance=True,
                                            package_order=config['dataset_pool_configuration'].package_order,
                                            in_storage=attribute_cache).get_dataset_pool()
        zone_set = dataset_pool.get_dataset('zone')

#        zone_set = ZoneDataset(in_storage_location=flt_dir_for_this_year, 
#                               in_storage_type='flt_storage', 
#                               in_table_name='zones')
        zone_set.load_dataset()
        self.prepare_for_run(config['travel_model_configuration'], year)
        travel_data_set = self.get_travel_data_from_travel_model(config, year, zone_set, 
                                                                 *args, **kwargs)
        logger.end_block()
        
        logger.start_block('Writing travel data to cache')
        out_storage = StorageFactory().get_storage('flt_storage', storage_location = flt_dir_for_next_year)
        #out_storage = flt_storage(Resources(data={"storage_location":flt_dir_for_next_year}))
        travel_data_set.write_dataset(attributes=travel_data_set.get_known_attribute_names(), 
                                      out_storage=out_storage, 
                                      out_table_name='travel_data')
        logger.end_block()
    def run(self, config, year, *args, **kwargs):
        """This is the main entry point.  It gets the appropriate configuration info from the 
        travel_model_configuration part of this config, and then copies the specified 
        UrbanSim data into files for travel mdel to read.  
        """
        cache_directory = config['cache_directory']
        simulation_state = SimulationState()
        simulation_state.set_cache_directory(cache_directory)
        simulation_state.set_current_time(year)
        attribute_cache = AttributeCache()
        dataset_pool = SessionConfiguration(
            new_instance=True,
            package_order=config['dataset_pool_configuration'].package_order,
            in_storage=attribute_cache).get_dataset_pool()

        #cache_storage = AttributeCache().get_flt_storage_for_year(year_for_base_year_cache)
        #datasets = DatasetFactory().create_datasets_from_flt(config.get('datasets_to_preload',{}),
        #"urbansim",
        #additional_arguments={'in_storage': attribute_cache})
        zone_set = dataset_pool.get_dataset('zone')
        self.prepare_for_run(config['travel_model_configuration'], year)
        self.create_travel_model_input_file(config, year, zone_set,
                                            dataset_pool, *args, **kwargs)
    tab_directory = options.tab_directory
    attribute_cache_directory = options.attribute_cache_directory
    table_name = options.table_name
    cache_year = options.cache_year

    if (tab_directory is None or attribute_cache_directory is None
            or table_name is None or cache_year is None):

        parser.print_help()
        sys.exit(1)

    input_storage = tab_storage(storage_location=tab_directory)

    attribute_cache = AttributeCache(cache_directory=attribute_cache_directory)
    output_storage = attribute_cache.get_flt_storage_for_year(cache_year)
    SimulationState().set_current_time(cache_year)
    SessionConfiguration(new_instance=True,
                         package_order=[],
                         in_storage=AttributeCache())

    logger.start_block(
        "Exporting table '%s' to year %s of cache located at %s..." %
        (table_name, cache_year, attribute_cache_directory))
    try:
        ExportStorage().export_dataset(
            dataset_name=table_name,
            in_storage=input_storage,
            out_storage=output_storage,
        )
    finally:
        logger.end_block()
Example #58
0
    def run(self,
            specification,
            coefficients,
            dataset,
            index=None,
            chunk_specification=None,
            data_objects=None,
            run_config=None,
            debuglevel=0):
        """ For info on the arguments see RegressionModel.
        """
        outcome_attribute_short = self.outcome_attribute.get_alias()
        if data_objects is not None:
            self.dataset_pool.add_datasets_if_not_included(data_objects)
        if self.filter_attribute <> None:
            res = Resources({"debug": debuglevel})
            index = dataset.get_filtered_index(self.filter_attribute,
                                               threshold=0,
                                               index=index,
                                               dataset_pool=self.dataset_pool,
                                               resources=res)

        current_year = SimulationState().get_current_time()
        current_month = int(re.search('\d+$', outcome_attribute_short).group())
        # date in YYYYMM format, matching to the id_name field of weather dataset
        date = int("%d%02d" % (current_year, current_month))
        date = array([date] * dataset.size())

        if "date" in dataset.get_known_attribute_names():
            dataset.set_values_of_one_attribute("date", date)
        else:
            dataset.add_primary_attribute(date, "date")

        water_demand = RegressionModel.run(self,
                                           specification,
                                           coefficients,
                                           dataset,
                                           index,
                                           chunk_specification,
                                           run_config=run_config,
                                           debuglevel=debuglevel)
        if (water_demand == None) or (water_demand.size <= 0):
            return water_demand

        if index == None:
            index = arange(dataset.size())

        if re.search("^ln_", outcome_attribute_short):
            # if the outcome attr. name starts with 'ln_' the results will be exponentiated.
            outcome_attribute_name = outcome_attribute_short[
                3:len(outcome_attribute_short)]
            water_demand = exp(water_demand)
        else:
            outcome_attribute_name = outcome_attribute_short

        if outcome_attribute_name in dataset.get_known_attribute_names():
            dataset.set_values_of_one_attribute(outcome_attribute_name,
                                                water_demand, index)
        else:
            results = zeros(dataset.size(), dtype=water_demand.dtype)
            results[index] = water_demand
            dataset.add_primary_attribute(results, outcome_attribute_name)

        return water_demand
Example #59
0
 def write_table(self, table_name, table_data, mode=Storage.OVERWRITE):
     year = SimulationState().get_current_time()
     storage = flt_storage(
         os.path.join(self.get_storage_location(), '%s' % year))
     return storage.write_table(table_name, table_data, mode)