예제 #1
0
    def init_run(self, create_baseyear_cache=True):
        ''' init run, get run_id & cache_directory. '''
        ##avoid invoking start_run from cmd line -
        option_group = StartRunOptionGroup()
        option_group.parser.set_defaults(xml_configuration=self.xml_config,
                                         scenario_name=self.scenario)
        #run_id, cache_directory = start_run(option_group)

        options, args = option_group.parse()
        self.run_manager = RunManager(
            option_group.get_services_database_configuration(options))

        resources = XMLConfiguration(self.xml_config).get_run_configuration(
            self.scenario)
        insert_auto_generated_cache_directory_if_needed(resources)
        cache_directory = resources['cache_directory']
        self.run_manager.setup_new_run(cache_directory, resources)
        run_id, cache_directory = self.run_manager.run_id, self.run_manager.get_current_cache_directory(
        )
        self.run_manager.add_row_to_history(run_id, resources, "done")

        if create_baseyear_cache:
            self.run_manager.create_baseyear_cache(resources)

        ## good for testing
        #run_id = 275
        #cache_directory = '/home/lmwang/opus/data/paris_zone/runs/run_275.2012_05_26_00_20'
        assert run_id is not None
        assert cache_directory is not None
        return run_id, cache_directory
예제 #2
0
def main(option_group=None, args=None):
    if option_group is None:
        option_group = RestartRunOptionGroup()
    parser = option_group.parser
    if args is None:
        options, args = option_group.parse()
    else: 
        options, _args = option_group.parse()
    run_manager = RunManager(option_group.get_services_database_configuration(options))
    run_as_multiprocess = not options.run_as_single_process
    
    if len(args) < 2:
        parser.print_help()
    else:
        run_id, year = (int(args[0]), int(args[1]))
        end_year = int(options.end_year) if options.end_year is not None else None
        run_manager.restart_run(run_id, 
                                year,
                                options.project_name,
                                end_year=end_year,
                                skip_urbansim=options.skip_urbansim,
                                create_baseyear_cache_if_not_exists=options.create_baseyear_cache_if_not_exists,
                                skip_cache_cleanup=options.skip_cache_cleanup,
                                run_as_multiprocess=run_as_multiprocess
                                )
예제 #3
0
    def test_run(self):
 
        # The paths work as follows: opus_matsim.__path__ is the path of the opus_matsim python module.  So we can use that
        # as anchor ...
        config_location = os.path.join(opus_matsim.__path__[0], 'tests')
        print "location: ", config_location
        run_config = XMLConfiguration( os.path.join(config_location,"test_config.xml")).get_run_configuration("Test")
        
        run_config['creating_baseyear_cache_configuration'].cache_directory_root = self.temp_dir
        run_config['creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy = \
            os.path.join(opus_matsim.__path__[0], 'tests', 'testdata', 'base_year_data')

        # insert_auto_generated_cache_directory... does things I don't understand.  Need to do the following to obtain consistent
        # behavior independent from the file root:
        run_config['cache_directory'] = None
        
        insert_auto_generated_cache_directory_if_needed(run_config)
        run_manager = RunManager(ServicesDatabaseConfiguration())
    
        run_manager.setup_new_run(cache_directory = run_config['cache_directory'],
                                  configuration = run_config)
        
        run_manager.run_run(run_config, run_as_multiprocess = True )
        

        self.assert_(True)
        
        self.cleanup_test_run()
예제 #4
0
def main(option_group=None, args=None):
    if option_group is None:
        option_group = RestartRunOptionGroup()
    parser = option_group.parser
    if args is None:
        options, args = option_group.parse()
    else:
        options, _args = option_group.parse()
    run_manager = RunManager(
        option_group.get_services_database_configuration(options))
    run_as_multiprocess = not options.run_as_single_process

    if len(args) < 2:
        parser.print_help()
    else:
        run_id, year = (int(args[0]), int(args[1]))
        end_year = int(
            options.end_year) if options.end_year is not None else None
        run_manager.restart_run(run_id,
                                year,
                                options.project_name,
                                end_year=end_year,
                                skip_urbansim=options.skip_urbansim,
                                create_baseyear_cache_if_not_exists=options.
                                create_baseyear_cache_if_not_exists,
                                skip_cache_cleanup=options.skip_cache_cleanup,
                                run_as_multiprocess=run_as_multiprocess)
예제 #5
0
 def test_run(self):
     print "Entering test run"
     
     run_manager = RunManager(ServicesDatabaseConfiguration())
     run_manager.setup_new_run(cache_directory = self.config['cache_directory'],configuration = self.config)
     
     run_manager.run_run(self.config, run_as_multiprocess = True )
     
     print "Leaving test run"
예제 #6
0
def prepare_run_manager(option_group=None):
    if option_group is None:
        option_group = StartRunOptionGroup()
    parser = option_group.parser
    options, args = option_group.parse()

    run_manager = RunManager(
        option_group.get_services_database_configuration(options))

    if options.pickled_resource_file is not None:
        f = file(options.pickled_resource_file, 'r')
        try:
            config = pickle.load(f)
        finally:
            f.close()
    elif options.configuration_path is not None:
        opus_path = options.configuration_path
        try:
            config = get_config_from_opus_path(opus_path)
        except ImportError:
            # TODO: Once all fully-specified configurations are stored as classes,
            #       get rid of this use.
            import_stmt = 'from %s import run_configuration as config' % opus_path
            exec(import_stmt)
        insert_auto_generated_cache_directory_if_needed(config)
    elif options.xml_configuration is not None:
        if options.scenario_name is None:
            parser.print_help()
            sys.exit(1)
        config = XMLConfiguration(
            options.xml_configuration).get_run_configuration(
                options.scenario_name)
        insert_auto_generated_cache_directory_if_needed(config)
    else:
        parser.print_help()
        sys.exit(1)

    if options.existing_cache_to_copy is not None:
        config[
            'creating_baseyear_cache_configuration'].cache_from_database = False
        config[
            'creating_baseyear_cache_configuration'].baseyear_cache = BaseyearCacheConfiguration(
                existing_cache_to_copy=options.existing_cache_to_copy, )
        if options.years_to_cache is not None:
            config[
                'creating_baseyear_cache_configuration'].baseyear_cache.years_to_cache = eval(
                    options.years_to_cache)

    if options.profile_filename is not None:
        config["profile_filename"] = options.profile_filename

    run_manager.setup_new_run(cache_directory=config['cache_directory'],
                              configuration=config)

    return options, config, run_manager
예제 #7
0
    def run(self):
        
        logger.start_block()
        insert_auto_generated_cache_directory_if_needed(self.config)
         
        run_manager = RunManager(ServicesDatabaseConfiguration())
        run_manager.setup_new_run(cache_directory = self.config['cache_directory'],configuration = self.config)
        
        run_manager.run_run(self.config, run_as_multiprocess = True )

        logger.end_block()
예제 #8
0
    def on_buttonBox_accepted(self):
        path = str(self.lePath.text())
        if not os.path.exists(path):
            msg = 'Cannot import, %s does not exist' % path
            logger.log_warning(msg)
            MessageBox.warning(mainwindow=self, text=msg, detailed_text='')
        else:
            cache_directory = path
            years = []

            for dir in os.listdir(cache_directory):
                if len(dir) == 4 and dir.isdigit():
                    years.append(int(dir))
            if years == []:
                msg = 'Cannot import, %s has no run data' % path
                logger.log_warning(msg)
                MessageBox.warning(mainwindow=self, text=msg, detailed_text='')

            else:
                start_year = min(years)
                end_year = max(years)
                project_name = os.environ['OPUSPROJECTNAME']
                run_name = os.path.basename(path)

                server_config = ServicesDatabaseConfiguration()
                run_manager = RunManager(server_config)

                run_id = run_manager._get_new_run_id()
                resources = {
                    'cache_directory': cache_directory,
                    'description': '',
                    'years': (start_year, end_year),
                    'project_name': project_name
                }

                try:
                    run_manager.add_row_to_history(run_id=run_id,
                                                   resources=resources,
                                                   status='done',
                                                   run_name=run_name)
                    update_available_runs(self.project)
                    logger.log_status(
                        'Added run %s of project %s to run_activity table' %
                        (run_name, project_name))
                except:
                    errorInfo = formatExceptionInfo()
                    logger.log_error(errorInfo)
                    MessageBox.error(
                        mainwindow=self,
                        text=
                        'Could not add run %s of project %s to run_activity table'
                        % (run_name, project_name),
                        detailed_text=errorInfo)
        self.close()
 def test_simulation(self):
     services_db = ServicesDatabaseConfiguration( database_name = 'services',                         
                                                  database_configuration = 'services_database_server' )
     run_manager = RunManager(services_db)
     run_as_multiprocess = True
     for scenario_name in ['san_antonio_baseline_test']:
         config = self.xml_config.get_run_configuration(scenario_name)
         insert_auto_generated_cache_directory_if_needed(config)
         run_manager.setup_new_run(cache_directory = config['cache_directory'],
                                   configuration = config)
         run_manager.run_run(config, run_as_multiprocess = run_as_multiprocess)
    def on_buttonBox_accepted(self):
        path = str(self.lePath.text())
        if not os.path.exists(path):
            msg = 'Cannot import, %s does not exist' % path
            logger.log_warning(msg)
            MessageBox.warning(mainwindow = self,
                            text = msg,
                            detailed_text = '')
        else:
            cache_directory = path
            years = []

            for dir in os.listdir(cache_directory):
                if len(dir) == 4 and dir.isdigit():
                    years.append(int(dir))
            if years == []:
                msg = 'Cannot import, %s has no run data'%path
                logger.log_warning(msg)
                MessageBox.warning(mainwindow = self,
                                text = msg,
                                detailed_text = '')

            else:
                start_year = min(years)
                end_year = max(years)
                project_name = os.environ['OPUSPROJECTNAME']
                run_name = os.path.basename(path)

                server_config = ServicesDatabaseConfiguration()
                run_manager = RunManager(server_config)

                run_id = run_manager._get_new_run_id()
                resources = {
                     'cache_directory': cache_directory,
                     'description': '',
                     'years': (start_year, end_year),
                     'project_name': project_name
                }

                try:
                    run_manager.add_row_to_history(run_id = run_id,
                                                   resources = resources,
                                                   status = 'done',
                                                   run_name = run_name)
                    update_available_runs(self.project)
                    logger.log_status('Added run %s of project %s to run_activity table'%(run_name, project_name))
                except:
                    errorInfo = formatExceptionInfo()
                    logger.log_error(errorInfo)
                    MessageBox.error(mainwindow = self,
                                    text = 'Could not add run %s of project %s to run_activity table'%(run_name, project_name),
                                    detailed_text = errorInfo)
        self.close()
 def test_simulation(self):
     eugene_dir = __import__('eugene').__path__[0]
     xml_config = XMLConfiguration(os.path.join(eugene_dir, 'configs', 'eugene_gridcell.xml'))
     option_group = StartRunOptionGroup()
     parser = option_group.parser
     # simulate 0 command line arguments by passing in []
     (options, _) = parser.parse_args([])
     run_manager = RunManager(option_group.get_services_database_configuration(options))
     run_section = xml_config.get_run_configuration('Eugene_baseline')
     insert_auto_generated_cache_directory_if_needed(run_section)
     run_manager.setup_new_run(cache_directory = run_section['cache_directory'],
                               configuration = run_section)
     run_manager.run_run(run_section)
예제 #12
0
def prepare_run_manager(option_group=None):
    if option_group is None:
        option_group = StartRunOptionGroup()
    parser = option_group.parser
    options, args = option_group.parse()

    run_manager = RunManager(option_group.get_services_database_configuration(options))
    
    if options.pickled_resource_file is not None:
        f = file(options.pickled_resource_file, 'r')
        try:
            config = pickle.load(f)
        finally:
            f.close()
    elif options.configuration_path is not None:
        opus_path = options.configuration_path
        try:
            config = get_config_from_opus_path(opus_path)
        except ImportError:
            # TODO: Once all fully-specified configurations are stored as classes,
            #       get rid of this use.
            import_stmt = 'from %s import run_configuration as config' % opus_path
            exec(import_stmt)
        insert_auto_generated_cache_directory_if_needed(config)
    elif options.xml_configuration is not None:
        if options.scenario_name is None:
            parser.print_help()
            sys.exit(1)
        config = XMLConfiguration(options.xml_configuration).get_run_configuration(options.scenario_name)
        insert_auto_generated_cache_directory_if_needed(config)
    else:
        parser.print_help()
        sys.exit(1)
        
    if options.existing_cache_to_copy is not None:
        config['creating_baseyear_cache_configuration'].cache_from_database = False
        config['creating_baseyear_cache_configuration'].baseyear_cache = BaseyearCacheConfiguration(
            existing_cache_to_copy = options.existing_cache_to_copy,
            )
        if options.years_to_cache is not None:
            config['creating_baseyear_cache_configuration'].baseyear_cache.years_to_cache = eval(options.years_to_cache)

    if options.profile_filename is not None:
        config["profile_filename"] = options.profile_filename
 
    run_manager.setup_new_run(cache_directory = config['cache_directory'],
                              configuration = config)

    return options, config, run_manager
 def test_simulation(self):
     # check that the simulation proceeds without crashing
     # open the configuration for seattle_parcel.xml
     seattle_parcel_dir = __import__('seattle_parcel').__path__[0]
     xml_config = XMLConfiguration(os.path.join(seattle_parcel_dir, 'configs', 'seattle_parcel.xml'))
     option_group = StartRunOptionGroup()
     parser = option_group.parser
     # simulate 0 command line arguments by passing in []
     (options, _) = parser.parse_args([])
     run_manager = RunManager(option_group.get_services_database_configuration(options))
     run_section = xml_config.get_run_configuration('Seattle_baseline')
     insert_auto_generated_cache_directory_if_needed(run_section)
     run_manager.setup_new_run(cache_directory = run_section['cache_directory'],
                               configuration = run_section)
     run_manager.run_run(run_section)
예제 #14
0
    def init_run(self, create_baseyear_cache=True):
        """ init run, get run_id & cache_directory. """
        ##avoid invoking start_run from cmd line -
        option_group = StartRunOptionGroup()
        option_group.parser.set_defaults(xml_configuration=self.xml_config, scenario_name=self.scenario)
        # run_id, cache_directory = start_run(option_group)

        options, args = option_group.parse()
        self.run_manager = RunManager(option_group.get_services_database_configuration(options))

        resources = XMLConfiguration(self.xml_config).get_run_configuration(self.scenario)
        insert_auto_generated_cache_directory_if_needed(resources)
        cache_directory = resources["cache_directory"]
        self.run_manager.setup_new_run(cache_directory, resources)
        run_id, cache_directory = self.run_manager.run_id, self.run_manager.get_current_cache_directory()
        self.run_manager.add_row_to_history(run_id, resources, "done")

        if create_baseyear_cache:
            self.run_manager.create_baseyear_cache(resources)

        ## good for testing
        # run_id = 275
        # cache_directory = '/home/lmwang/opus/data/paris_zone/runs/run_275.2012_05_26_00_20'
        assert run_id is not None
        assert cache_directory is not None
        return run_id, cache_directory
예제 #15
0
def get_run_manager():
    '''
    Get an instance of a valid run manager
    @return: a run manager instance (RunManager)
    '''
    config = ServicesDatabaseConfiguration()
    run_manager = RunManager(config)
    return run_manager
예제 #16
0
 def get_run_manager(self):
     """in case the connection to services timeout, reconnect
     """
     try:
         self._run_manager.services_db.table_exists('run_activity')
     except:  #connection has gone away, re-create run_manager
         self._run_manager = RunManager(self.services_db_config)
     return self._run_manager
예제 #17
0
    def update_prediction(self, est_v, simulation_state, dataset_pool,
                          calib_datasets, *args, **kwargs):
        option_group = RestartRunOptionGroup()
        option_group.parser.set_defaults(
            project_name=self.project_name,
            skip_cache_cleanup=self.skip_cache_cleanup)

        options, args = option_group.parse()
        if self.run_manager is None:
            self.run_manager = RunManager(
                option_group.get_services_database_configuration(options))

        if lock != None: lock.acquire()
        ## query runs available for re-use
        runs_done = self.run_manager.get_run_info(run_ids=self.run_ids,
                                                  status='done')
        create_baseyear_cache = False
        import pdb
        pdb.set_trace()
        if len(runs_done
               ) == 0:  ##there is no re-usable run directory, init a new run
            run_id, cache_directory = self.init_run(
                create_baseyear_cache=False)
            self.run_ids.append(run_id)
            create_baseyear_cache = True
            logger.log_status('Initializing new run with id ' + str(run_id))
        else:
            run_id = runs_done[0].run_id  ##take the first 'done' run_id
            cache_directory = self.run_manager.get_cache_directory(run_id)
            logger.log_status('Using old run with id ' + str(run_id))
        resources = self.run_manager.get_resources_for_run_id_from_history(
            run_id, filter_by_status=False)
        self.run_manager.add_row_to_history(run_id, resources, "taken")
        if lock != None: lock.release()

        if create_baseyear_cache:
            self.run_manager.create_baseyear_cache(resources)

        self.update_parameters(est_v, cache_directory, simulation_state,
                               dataset_pool, calib_datasets, *args, **kwargs)
        restart_run(option_group=option_group, args=[run_id, self.start_year])

        prediction = self.summarize_prediction(cache_directory,
                                               simulation_state, dataset_pool,
                                               calib_datasets)
        return prediction
    def test_restart_simple_run(self):
        _do_run_simple_test_run(self, self.temp_dir, self.config, end_year=1983)
        runs_manager = RunManager(self.config)
        config = _get_run_config(temp_dir=self.temp_dir)
        runs_manager.update_environment_variables(run_resources=config)

        run_activity = runs_manager.services_db.get_table("run_activity")
        s = select([func.max(run_activity.c.run_id)])
        run_id = runs_manager.services_db.execute(s).fetchone()[0]

        s = select([run_activity.c.status], whereclause=run_activity.c.run_id == run_id)
        status = runs_manager.services_db.execute(s).fetchone()[0]

        expected = "done"
        self.assertEqual(status, expected)

        runs_manager.restart_run(run_id, restart_year=1981, project_name="eugene_gridcell", skip_urbansim=False)

        s = select([run_activity.c.status], whereclause=run_activity.c.run_id == run_id)
        status = runs_manager.services_db.execute(s).fetchone()[0]

        expected = "done"
        self.assertEqual(status, expected)

        # Restaring without running urbansim should not re-run that year.
        # TODO: test that no models are run this time.
        runs_manager.restart_run(run_id, restart_year=1982, project_name="eugene_gridcell", skip_urbansim=True)
        s = select([run_activity.c.status], whereclause=run_activity.c.run_id == run_id)
        status = runs_manager.services_db.execute(s).fetchone()[0]

        expected = "done"
        self.assertEqual(status, expected)

        self.cleanup_test_run()
예제 #19
0
def main(option_group=None, args=None):

    if option_group is None:
        option_group = restart_run.RestartRunOptionGroup()
    parser = option_group.parser
    if args is None:
        options, args = option_group.parse()
    else: 
        options, _args = option_group.parse()
    run_manager = RunManager(option_group.get_services_database_configuration(options))
    run_as_multiprocess = not options.run_as_single_process

    if len(args) < 1:
        parser.print_help()
        sys.exit(1)
    run_id = int(args[0])
    year = None
    if len(args) > 1:
        year = int(args[1])

    # Get configuration from DB and ensure the run directory is mounted.  Note
    # that we pass a dummy value for restart_year because we might not know it
    # yet.  But when we actually restart the run, we will pass the correct
    # year.
    run_resources = run_manager.create_run_resources_from_history(run_id=run_id,
                                                                  restart_year=2010)
    hudson_common.mount_cache_dir(run_resources)

    cache_dir = run_resources['cache_directory']
    if not year:
        # guess the year based on how the cache dir is populated
        years = map(lambda y : int(os.path.basename(y)),
                    glob.glob(os.path.join(cache_dir, "2*")))
        year = max(years)

    end_year = int(options.end_year) if options.end_year is not None else None
    run_manager.restart_run(run_id, 
                            year,
                            options.project_name,
                            end_year=end_year,
                            skip_urbansim=options.skip_urbansim,
                            create_baseyear_cache_if_not_exists=options.create_baseyear_cache_if_not_exists,
                            skip_cache_cleanup=options.skip_cache_cleanup,
                            run_as_multiprocess=run_as_multiprocess
                            )
def _do_run_simple_test_run(caller, temp_dir, config, end_year=None):
    """Runs model system with a single model (for speed).
    Sets the .resources property of the caller before starting the run.
    """
    runs_manager = RunManager(config)

    run_configuration = SubsetConfiguration()
    run_configuration['creating_baseyear_cache_configuration'].cache_directory_root = temp_dir
    run_configuration['models'] = ['land_price_model']
    if end_year is not None:
        run_configuration['years'] = (run_configuration['years'][0], end_year)
    
    SessionConfiguration(new_instance=True,
                         package_order=run_configuration['dataset_pool_configuration'].package_order,
                         in_storage=AttributeCache())
    insert_auto_generated_cache_directory_if_needed(run_configuration)
    caller.resources = run_configuration
    runs_manager.setup_new_run(cache_directory = run_configuration['cache_directory'],
                               configuration = run_configuration)
    runs_manager.run_run(run_configuration)
예제 #21
0
    def testName(self):
        print "entering test_run"
        
        logger.log_status('Preparing MATsim test run ...')
        # unzip MATSim files
        matsim_zip = ExtractZipFile(self.matsim_source, self.destination)
        matsim_zip.extract()
        matsim_extracted_files = os.path.join(self.destination, 'MATSimTestClasses') # location of unziped MATSim files
        # unzip base_year_cache
        base_year_data_zip = ExtractZipFile(self.base_year_data_source, self.destination)
        base_year_data_zip.extract()
        base_year_data_extracted_files = os.path.join(self.destination, 'base_year_data') # location of unziped base_year_cache
        
                
        # updating location of base_year_data
        self.run_config['creating_baseyear_cache_configuration'].cache_directory_root = self.destination
        self.run_config['creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy = base_year_data_extracted_files
        self.run_config['cache_directory'] = base_year_data_extracted_files
        self.run_config.add('matsim_files', matsim_extracted_files)
        self.run_config.add('matsim_config', self.matsim_config_full)
        self.run_config.add('root', self.destination)
        
        insert_auto_generated_cache_directory_if_needed(self.run_config)
        run_manager = RunManager(ServicesDatabaseConfiguration())
    
        run_manager.setup_new_run(cache_directory = self.run_config['cache_directory'],
                                  configuration = self.run_config)

        logger.log_status('Strating UrbanSim run ... ')
        run_manager.run_run(self.run_config, run_as_multiprocess = True )
        # after the UrbanSim run the travel data sets schould be equal
        # self.assertTrue( self.compare_travel_data_sets() )
        logger.log_status('... UrbanSim run finished.')
        
        print "leaving test_run"
예제 #22
0
def _do_run_simple_test_run(caller, temp_dir, config, end_year=None):
    """Runs model system with a single model (for speed).
    Sets the .resources property of the caller before starting the run.
    """

    runs_manager = RunManager(config)

    run_configuration = _get_run_config(temp_dir=temp_dir)

    insert_auto_generated_cache_directory_if_needed(run_configuration)
    run_configuration[
        'creating_baseyear_cache_configuration'].cache_directory_root = temp_dir
    run_configuration['models'] = ['land_price_model']
    if end_year is not None:
        run_configuration['years'] = (run_configuration['years'][0], end_year)

    SessionConfiguration(
        new_instance=True,
        package_order=run_configuration['dataset_pool_configuration'].
        package_order,
        in_storage=AttributeCache())
    insert_auto_generated_cache_directory_if_needed(run_configuration)
    caller.resources = run_configuration
    runs_manager.setup_new_run(
        cache_directory=run_configuration['cache_directory'],
        configuration=run_configuration)
    runs_manager.run_run(run_configuration)
예제 #23
0
    def test_run(self):

        # The paths work as follows: opus_matsim.__path__ is the path of the opus_matsim python module.  So we can use that
        # as anchor ...
        config_location = os.path.join(opus_matsim.__path__[0], 'tests')
        print "location: ", config_location
        run_config = XMLConfiguration(
            os.path.join(config_location,
                         "test_config.xml")).get_run_configuration("Test")

        run_config[
            'creating_baseyear_cache_configuration'].cache_directory_root = self.temp_dir
        run_config['creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy = \
            os.path.join(opus_matsim.__path__[0], 'tests', 'testdata', 'base_year_data')

        # insert_auto_generated_cache_directory... does things I don't understand.  Need to do the following to obtain consistent
        # behavior independent from the file root:
        run_config['cache_directory'] = None

        insert_auto_generated_cache_directory_if_needed(run_config)
        run_manager = RunManager(ServicesDatabaseConfiguration())

        run_manager.setup_new_run(
            cache_directory=run_config['cache_directory'],
            configuration=run_config)

        run_manager.run_run(run_config, run_as_multiprocess=True)

        self.assert_(True)

        self.cleanup_test_run()
    def testName(self):
        print "entering test_run"

        logger.log_status("Preparing MATsim test run ...")
        # unzip MATSim files
        matsim_zip = ExtractZipFile(self.matsim_source, self.destination)
        matsim_zip.extract()
        matsim_extracted_files = os.path.join(self.destination, "MATSimTestClasses")  # location of unziped MATSim files
        # unzip base_year_cache
        base_year_data_zip = ExtractZipFile(self.base_year_data_source, self.destination)
        base_year_data_zip.extract()
        base_year_data_extracted_files = os.path.join(
            self.destination, "base_year_data"
        )  # location of unziped base_year_cache

        # updating location of base_year_data
        self.run_config["creating_baseyear_cache_configuration"].cache_directory_root = self.destination
        self.run_config[
            "creating_baseyear_cache_configuration"
        ].baseyear_cache.existing_cache_to_copy = base_year_data_extracted_files
        self.run_config["cache_directory"] = base_year_data_extracted_files
        self.run_config.add("matsim_files", matsim_extracted_files)
        self.run_config.add("matsim_config", self.matsim_config_full)
        self.run_config.add("root", self.destination)

        insert_auto_generated_cache_directory_if_needed(self.run_config)
        run_manager = RunManager(ServicesDatabaseConfiguration())

        run_manager.setup_new_run(cache_directory=self.run_config["cache_directory"], configuration=self.run_config)

        logger.log_status("Strating UrbanSim run ... ")
        run_manager.run_run(self.run_config, run_as_multiprocess=True)
        # after the UrbanSim run the travel data sets schould be equal
        # self.assertTrue( self.compare_travel_data_sets() )
        logger.log_status("... UrbanSim run finished.")

        print "leaving test_run"
예제 #25
0
 def test_run(self):
     print "Entering test run"
     
     run_manager = RunManager(ServicesDatabaseConfiguration())
     run_manager.setup_new_run(cache_directory = self.config['cache_directory'],configuration = self.config)
     
     run_manager.run_run(self.config, run_as_multiprocess = True )
     
     print "Leaving test run"
예제 #26
0
    def test_restart_simple_run(self):
        _do_run_simple_test_run(self,
                                self.temp_dir,
                                self.config,
                                end_year=1983)
        runs_manager = RunManager(self.config)
        config = _get_run_config(temp_dir=self.temp_dir)
        runs_manager.update_environment_variables(run_resources=config)

        run_activity = runs_manager.services_db.get_table('run_activity')
        s = select([func.max(run_activity.c.run_id)])
        run_id = runs_manager.services_db.execute(s).fetchone()[0]

        s = select([run_activity.c.status],
                   whereclause=run_activity.c.run_id == run_id)
        status = runs_manager.services_db.execute(s).fetchone()[0]

        expected = 'done'
        self.assertEqual(status, expected)

        runs_manager.restart_run(run_id,
                                 restart_year=1981,
                                 project_name='eugene_gridcell',
                                 skip_urbansim=False)

        s = select([run_activity.c.status],
                   whereclause=run_activity.c.run_id == run_id)
        status = runs_manager.services_db.execute(s).fetchone()[0]

        expected = 'done'
        self.assertEqual(status, expected)

        # Restaring without running urbansim should not re-run that year.
        # TODO: test that no models are run this time.
        runs_manager.restart_run(run_id,
                                 restart_year=1982,
                                 project_name='eugene_gridcell',
                                 skip_urbansim=True)
        s = select([run_activity.c.status],
                   whereclause=run_activity.c.run_id == run_id)
        status = runs_manager.services_db.execute(s).fetchone()[0]

        expected = 'done'
        self.assertEqual(status, expected)

        self.cleanup_test_run()
예제 #27
0
class StartRunOptionGroup(object):
    """ Helper class to start model from an xml config file. 
    """

    logger.start_block("Starting UrbanSim")

    # get program arguments from the command line
    program_arguments = sys.argv[1:]

    # default parameters are:
    # --config=opus_matsim/sustain_city/configs/seattle_parcel_prescheduled_events.xml
    # --executable=Seattle_baseline
    parser = optparse.OptionParser()
    parser.add_option("-c",
                      "--config",
                      dest="config_file_name",
                      action="store",
                      type="string",
                      help="Name of file containing urbansim config")
    parser.add_option("-e",
                      "--executable",
                      dest="scenario_executable",
                      action="store",
                      type="string",
                      help="Model to execute")
    (options, args) = parser.parse_args()

    if options.config_file_name == None:
        logger.log_error("Missing path to the urbansim config file")
    if options.scenario_executable == None:
        logger.log_error("Missing name of executable scenario")

    config = XMLConfiguration(options.config_file_name).get_run_configuration(
        options.scenario_executable)

    insert_auto_generated_cache_directory_if_needed(config)

    run_manager = RunManager(ServicesDatabaseConfiguration())

    run_manager.setup_new_run(cache_directory=config['cache_directory'],
                              configuration=config)

    #try: #tnicolai
    #    import pydevd
    #    pydevd.settrace()
    #except: pass

    run_manager.run_run(config, run_as_multiprocess=True)
예제 #28
0
 def test_simulation(self):
     services_db = ServicesDatabaseConfiguration( database_name = 'services',                         
                                                  database_configuration = 'services_database_server' )
     run_manager = RunManager(services_db)
     run_as_multiprocess = True
     for scenario_name in ['psrc_baseline_test']:
         config = self.xml_config.get_run_configuration(scenario_name)
         insert_auto_generated_cache_directory_if_needed(config)
         run_manager.setup_new_run(cache_directory = config['cache_directory'],
                                   configuration = config)
         run_manager.run_run(config, run_as_multiprocess = run_as_multiprocess)
예제 #29
0
    def run(self, config, executable):
        #--config=opus_matsim/sustain_city/configs/seattle_parcel.xml --executable=Seattle_baseline
        config = XMLConfiguration(config).get_run_configuration(executable)
        
        insert_auto_generated_cache_directory_if_needed(config)
     
        run_manager = RunManager(ServicesDatabaseConfiguration())
        
        run_manager.setup_new_run(cache_directory = config['cache_directory'],configuration = config)

        run_manager.run_run(config, run_as_multiprocess = True )
예제 #30
0
class StartRunOptionGroup(object):
    """ Helper class to start model from an xml config file. 
    """

    config = XMLConfiguration("opus_matsim/configs/seattle_parcel.xml"
                              ).get_run_configuration("Seattle_baseline")
    #    config = XMLConfiguration("opus_matsim/configs/psrc_parcel.xml").get_run_configuration("PSRC_baseline")

    insert_auto_generated_cache_directory_if_needed(config)

    run_manager = RunManager(ServicesDatabaseConfiguration())

    run_manager.setup_new_run(cache_directory=config['cache_directory'],
                              configuration=config)

    #    run_manager.create_baseyear_cache(config)

    run_manager.run_run(config, run_as_multiprocess=True)
예제 #31
0
 def test_simulation(self):
     eugene_dir = __import__('eugene').__path__[0]
     xml_config = XMLConfiguration(
         os.path.join(eugene_dir, 'configs', 'eugene_gridcell.xml'))
     option_group = StartRunOptionGroup()
     parser = option_group.parser
     # simulate 0 command line arguments by passing in []
     (options, _) = parser.parse_args([])
     run_manager = RunManager(
         option_group.get_services_database_configuration(options))
     run_section = xml_config.get_run_configuration('Eugene_baseline')
     insert_auto_generated_cache_directory_if_needed(run_section)
     run_manager.setup_new_run(
         cache_directory=run_section['cache_directory'],
         configuration=run_section)
     run_manager.run_run(run_section)
 def test_restart_simple_run(self):
     _do_run_simple_test_run(self, self.temp_dir, self.config)
     runs_manager = RunManager(self.config)
     runs_manager.update_environment_variables(run_resources = SubsetConfiguration())
     
     run_activity = runs_manager.services_db.get_table('run_activity')
     s = select([func.max(run_activity.c.run_id)])
     run_id = runs_manager.services_db.execute(s).fetchone()[0]
     
     s = select([run_activity.c.status],
                whereclause = run_activity.c.run_id == run_id)
     status = runs_manager.services_db.execute(s).fetchone()[0]
                                                                
     expected = 'done'
     self.assertEqual(status, expected)
                     
     runs_manager.restart_run(run_id,
                              restart_year=2001,
                              project_name = SubsetConfiguration()['project_name'],
                              skip_urbansim=False)
     
     s = select([run_activity.c.status],
                whereclause = run_activity.c.run_id == run_id)
     status = runs_manager.services_db.execute(s).fetchone()[0]
                                                    
     expected = 'done'
     self.assertEqual(status, expected)
        
     # Restaring without running urbansim should not re-run that year.
     # TODO: test that no models are run this time.
     runs_manager.restart_run(run_id,
                              restart_year=2002,
                              project_name = SubsetConfiguration()['project_name'],
                              skip_urbansim=True)
     s = select([run_activity.c.status],
                whereclause = run_activity.c.run_id == run_id)
     status = runs_manager.services_db.execute(s).fetchone()[0]
                                                                
     expected = 'done'
     self.assertEqual(status, expected)
                 
     self.cleanup_test_run()
예제 #33
0
    def update_prediction(self, est_v, simulation_state, dataset_pool, calib_datasets, *args, **kwargs):
        option_group = RestartRunOptionGroup()
        option_group.parser.set_defaults(project_name=self.project_name, skip_cache_cleanup=self.skip_cache_cleanup)

        options, args = option_group.parse()
        if self.run_manager is None:
            self.run_manager = RunManager(option_group.get_services_database_configuration(options))

        if lock != None:
            lock.acquire()
        ## query runs available for re-use
        runs_done = self.run_manager.get_run_info(run_ids=self.run_ids, status="done")
        create_baseyear_cache = False
        import pdb

        pdb.set_trace()
        if len(runs_done) == 0:  ##there is no re-usable run directory, init a new run
            run_id, cache_directory = self.init_run(create_baseyear_cache=False)
            self.run_ids.append(run_id)
            create_baseyear_cache = True
            logger.log_status("Initializing new run with id " + str(run_id))
        else:
            run_id = runs_done[0].run_id  ##take the first 'done' run_id
            cache_directory = self.run_manager.get_cache_directory(run_id)
            logger.log_status("Using old run with id " + str(run_id))
        resources = self.run_manager.get_resources_for_run_id_from_history(run_id, filter_by_status=False)
        self.run_manager.add_row_to_history(run_id, resources, "taken")
        if lock != None:
            lock.release()

        if create_baseyear_cache:
            self.run_manager.create_baseyear_cache(resources)

        self.update_parameters(est_v, cache_directory, simulation_state, dataset_pool, calib_datasets, *args, **kwargs)
        restart_run(option_group=option_group, args=[run_id, self.start_year])

        prediction = self.summarize_prediction(cache_directory, simulation_state, dataset_pool, calib_datasets)
        return prediction
    def test_simulation(self):
        base_year_data_path = os.path.join(self.data_path, 'base_year_data')        
        if not os.path.exists(base_year_data_path):
            os.makedirs(base_year_data_path)

        ftp_url = os.environ["FTP_URL"]
        file_name = os.path.split(ftp_url)[1]
        ftp_user = os.environ["FTP_USERNAME"]
        ftp_password = os.environ["FTP_PASSWORD"]
        
        #stdout, stderr = Popen("ls -la %s" % base_year_data_path, shell=True).communicate()
        #stdout, stderr = Popen("echo '%s'" % (base_year_data_path), stdout=PIPE).communicate()
        #print stdout
        
        try:
            Popen( """
                        cd %s;
                        pwd;
                        ls -la;
                        echo wget --timestamping %s --ftp-user=%s --ftp-password=%s > /dev/null 2>&1;
                        rm -rf 2008;
                        unzip -o %s
                        """ % (base_year_data_path, ftp_url, ftp_user, ftp_password, file_name),
                        shell = True
                        ).communicate()
        except:
            print "Error when downloading and unzipping file from %s." % ftp_url
            raise

        services_db = ServicesDatabaseConfiguration( database_name = 'services',                         
                                                     database_configuration = 'services_database_server' )
        run_manager = RunManager(services_db)
        run_as_multiprocess = True
        xml_config = XMLConfiguration(os.path.join(self.opus_home, 'project_configs', 'washtenaw_parcel.xml'))
        for scenario_name in ['washtenaw_baseline_test']:
            config = xml_config.get_run_configuration(scenario_name)
            insert_auto_generated_cache_directory_if_needed(config)
#            base_year = config['base_year']
#            config['years_to_run'] = (base_year+1, base_year+2)
            run_manager.setup_new_run(cache_directory = config['cache_directory'],
                                      configuration = config)
            run_manager.run_run(config, run_as_multiprocess = run_as_multiprocess)
예제 #35
0
        except:  #connection has gone away, re-create run_manager
            self._run_manager = RunManager(self.services_db_config)
        return self._run_manager


if __name__ == "__main__":

    try:
        import wingdbstub
    except:
        pass
    option_group = OptionGroup()
    parser = option_group.parser
    (options, args) = parser.parse_args()
    services_db = option_group.get_services_database_configuration(options)
    run_manager = RunManager(services_db)
    if not run_manager.services_db:
        raise RuntimeError, "services database must exist; use --hostname argument to specify the database server containing services database."

    urbansim_server = options.urbansim_server or os.environ.get(
        'URBANSIMHOSTNAME', 'localhost')
    urbansim_user = options.runserver_username or os.environ.get(
        'URBANSIMUSERNAME', None)
    urbansim_password = os.environ.get('URBANSIMPASSWORD', None)

    travelmodel_server = options.travelmodel_server or os.environ.get(
        'TRAVELMODELHOSTNAME', 'localhost')
    travelmodel_user = options.runserver_username or os.environ.get(
        'TRAVELMODELUSERNAME', None)
    travelmodel_password = os.environ.get('TRAVELMODELPASSWORD', None)
    if not (options.configuration_path or options.run_id):
    def on_pbnStartModel_released(self):
        duplicate = False
        self.diagnostic_go_button.setEnabled(True)

        if self.running and not self.paused:
            # Take care of pausing a run
            success = self.runThread.pause()
            if success:
                self.paused = True
                self.timer.stop()
                self.pbnStartModel.setText(QString("Resume simulation run..."))
        elif self.running and self.paused:
            # Need to resume a paused run
            success = self.runThread.resume()
            if success:
                self.paused = False
                self.timer.start(1000)
                self.pbnStartModel.setText(QString("Pause simulation run..."))
        elif not self.running:
            run_name = str(self.leRunName.text())
            if run_name == '':
                run_name = None
            else:
                run_id = None
                run_nodes = get_available_run_nodes(self.project)
                for run_node in run_nodes:
                    existing_run_name = run_node.tag
                    if run_name == existing_run_name:
                        duplicate = True
                        r = run_node.get('run_id')
                        if r is not None:
                            run_id = int(r)
                        break
                if duplicate:
                    dlg_dup = OverwriteRunDialog(self)

                    if dlg_dup.exec_() == QDialog.Rejected:
                        return
                    delete_simulation_run(self.project, run_node.tag) # todo change to run_node.get('name')
            # Update the XML
            self.project.update_xml_config()
            self.updateConfigAndGuiForRun()

            # Fire up a new thread and run the model
            self.pbnStartModel.setText(QString("Pause simulation run..."))
            # References to the GUI elements for status for this run...
            self.progressBarTotal = self.runProgressBarTotal
            self.progressBarYear = self.runProgressBarYear
            self.progressBarModel = self.runProgressBarModel

            #self.pbnRemoveModel.setEnabled(False)
            #self.pbnStartModel.setEnabled(False)

            # Initializing values
            self.progressBarTotal.setValue(0)
            self.progressBarYear.setValue(0)
            self.progressBarModel.setValue(0)
            self.progressBarTotal.setRange(0,0)
            self.progressBarYear.setRange(0,0)
            self.progressBarModel.setRange(0,0)

            batch_name = str(self.cboOptionalIndicatorBatch.currentText())
            if batch_name == '(None)':
                batch_name = None

            self.runThread = RunModelThread(get_mainwindow_instance(),
                                            self,
                                            batch_name,
                                            run_name)

            if duplicate and run_id is not None:
                from opus_core.services.run_server.run_manager import RunManager as ServicesRunManager
                run_manager = ServicesRunManager(ServicesDatabaseConfiguration())
                run_manager.delete_everything_for_this_run(run_id = run_id)
                run_manager.close()


            # Use this signal from the thread if it is capable of producing its own status signal
            QObject.connect(self.runThread, SIGNAL("runFinished(PyQt_PyObject)"),
                            self.runFinishedFromThread)
            QObject.connect(self.runThread, SIGNAL("runError(PyQt_PyObject)"),
                            self.runErrorFromThread)
            # Use this timer to call a function in the thread to check status if the thread is unable
            # to produce its own signal above
            self.timer = QTimer()
            QObject.connect(self.timer, SIGNAL("timeout()"),
                            self.runStatusFromThread)
            self.timer.start(1000)
            self.running = True
            self.paused = False
            self.runThread.start()
        else:
            print "Unexpected state in the model run..."
예제 #37
0
class Calibration(object):
    """ Class to calibrate UrbanSim model coefficients.
    
    """

    def __init__(
        self,
        xml_config,
        scenario,
        calib_datasets,
        target_expression,
        target_file,
        subset=None,
        subset_patterns=None,
        skip_cache_cleanup=False,
        log_directory=None,
    ):
        """
        - xml_config: xml configuration file, for ex '/home/atschirhar/opus/project_configs/paris_zone.xml'
        - scenario: name of scenario to run for calibration, where models_to_run and simulation years are specified
        - calib_datasets: dictionary specifying dataset names and attributes to be calibrated, e.g.
                  {'establishment_location_choice_model_coefficients': 'estimate'}
        - target_expression: opus expression computing values from prediction to be compared with targets 
        - target_file: name of csv file providing targets 
        - subset: dictionary specifying the dataset to be calibrated,
                  {'etablishment_location_choice_model_coefficients': ['coefficient_name', ['paris_celcm, 'biotech_celcm']]}
          subset and subset_patterns can not be both specified for the same dataset
        - subset_patterns: dictionary specifying the dataset to be calibrated through a regular expression (re) pattern
                  {'etablishment_location_choice_model_coefficients': ['coefficient_name', '*_celcm']} 
          subset and subset_patterns can not be both specified for the same dataset

        """
        self.target_expression = target_expression
        self.target = self.read_target(target_file)

        self.run_manager = None
        self.xml_config = xml_config
        self.scenario = scenario
        self.skip_cache_cleanup = skip_cache_cleanup
        self.run_id, self.cache_directory = self.init_run()
        self.run_ids = [self.run_id]  # allow starting of multiple runs for parallel optimization
        self.log_directory = log_directory
        if self.log_directory is None:
            self.log_directory = self.cache_directory  # legacy

        log_file = os.path.join(self.log_directory, "calibration.log")
        logger.enable_file_logging(log_file)

        dict_config = XMLConfiguration(self.xml_config).get_run_configuration(self.scenario)
        ## get parameters from config
        self.base_year = dict_config["base_year"]
        self.start_year, self.end_year = dict_config["years"]
        self.project_name = dict_config["project_name"]
        self.package_order = dict_config["dataset_pool_configuration"].package_order

    @log_block("Start Calibration")
    def run(self, optimizer="lbfgsb", results_pickle_prefix="calib", optimizer_kwargs={}):
        """ Call specifized optimizer to calibrate
        
        Arguments:
            - optimizer: optimization method chosen (fmin_bfgs, simulated anneal etc.)
            - results_pickle_prefix: prefix of the pickle file name that will be saved after the simulation; if None, results is not saved
            
        Returns:
            - the results from the opimizater
            - a pickle dump of the results in the cache_directory, if results_pickle_prefix is specified
        
        """

        simulation_state = SimulationState()
        simulation_state.set_current_time(self.base_year)
        simulation_state.set_cache_directory(self.cache_directory)
        attribute_cache = AttributeCache()
        dataset_pool = SessionConfiguration(
            new_instance=True, package_order=self.package_order, in_storage=attribute_cache
        ).get_dataset_pool()

        calib_datasets = {}
        for dataset_name, calib_attr in calib_datasets.iteritems():
            dataset = dataset_pool.get_dataset(dataset_name, dataset_arguments={"id_name": []})
            assert (
                subset is None
                or subset.get(dataset_name, None) is None
                or subset_patterns is None
                or subset_patterns.get(dataset_name, None) is None
            )
            if subset is not None and subset.get(dataset_name, None) is not None:
                subset_attr, subset_cond = subset.get(dataset_name)
                index = np.in1d(dataset[subset_attr], subset_cond)
            elif subset_patterns is not None and subset_patterns.get(dataset_name, None) is not None:
                subset_attr, subset_pattern = subset_patterns.get(dataset_name)
                index = array([True if re.search(subset_pattern, attr_v) else False for attr_v in dataset[subset_attr]])
            else:
                index = arange(dataset.size(), dtype="i")

            calib_datasets[dataset_name] = [dataset, calib_attr, index]

        init_v = array([], dtype="f8")
        for dataset_name, calib in calib_datasets.iteritems():
            dataset, calib_attr, index = calib
            if type(calib_attr) == str:
                init_v = np.concatenate((init_v, dataset[calib_attr][index]))
            elif type(calib_attr) in (list, tuple):
                for attr in calib_attr:
                    init_v = np.concatenate((init_v, dataset[attr][index]))
            else:
                raise TypeError, "Unrecongized data type in calib_datasets"

        t0 = time.time()

        if is_parallelizable == True:
            set_parallel(True)

        print OKBLUE + "\noptimizer = {} (is_parallel = {})".format(optimizer, is_parallelizable) + ENDC
        print OKBLUE + "-------------------------------------------------------\n" + ENDC
        if optimizer == "bfgs":
            default_kwargs = {
                "fprime": None,
                "epsilon": 1e-08,
                "maxiter": None,
                "full_output": 1,
                "disp": 1,
                "retall": 0,
                "callback": None,
            }
            optimizer_func = fmin_bfgs
        elif optimizer == "lbfgsb":
            default_kwargs = {"fprime": None, "approx_grad": True, "bounds": None, "factr": 1e12, "iprint": 1}

            optimizer_func = fmin_l_bfgs_b
        elif optimizer == "anneal":
            default_kwargs = {
                "schedule": "fast",
                "full_output": 1,
                "T0": None,
                "Tf": 1e-12,
                "maxeval": None,
                "maxaccept": None,
                "maxiter": 400,
                "boltzmann": 1.0,
                "learn_rate": 0.5,
                "feps": 1e-06,
                "quench": 1.0,
                "m": 1.0,
                "n": 1.0,
                "lower": -1,
                "upper": 1,
                "dwell": 50,
                "disp": True,
            }

            optimizer_func = anneal
        elif optimizer == "panneal":
            default_kwargs = {
                "schedule": "fast",
                "full_output": 1,
                "T0": None,
                "Tf": 1e-12,
                "maxeval": None,
                "maxaccept": None,
                "maxiter": 400,
                "boltzmann": 1.0,
                "learn_rate": 0.5,
                "feps": 1e-06,
                "quench": 1.0,
                "m": 1.0,
                "n": 1.0,
                "lower": -1,
                "upper": 1,
                "dwell": 50,
                "disp": True,
                "cores": 24,
                "interv": 20,
            }

            optimizer_func = panneal
        else:
            raise ValueError, "Unrecognized optimizer {}".format(optimizer)

        default_kwargs.update(optimizer_kwargs)
        results = optimizer_func(self.target_func, copy(init_v), **default_kwargs)

        duration = time.time() - t0
        if results_pickle_prefix is not None:
            pickle_file = "{}_{}.pickle".format(results_pickle_prefix, optimizer)
            pickle_file = os.path.join(self.log_directory, pickle_file)
            pickle.dump(results, open(pickle_file, "wb"))

        if is_parallelizable == True:
            set_parallel(False)

        logger.log_status("init target_func: {}".format(self.target_func(init_v)))
        logger.log_status("end target_func: {}".format(results[:]))  # which one?
        logger.log_status("outputs from optimizer: {}".format(results))
        logger.log_status("Execution time: {}".format(duration))

    def init_run(self, create_baseyear_cache=True):
        """ init run, get run_id & cache_directory. """
        ##avoid invoking start_run from cmd line -
        option_group = StartRunOptionGroup()
        option_group.parser.set_defaults(xml_configuration=self.xml_config, scenario_name=self.scenario)
        # run_id, cache_directory = start_run(option_group)

        options, args = option_group.parse()
        self.run_manager = RunManager(option_group.get_services_database_configuration(options))

        resources = XMLConfiguration(self.xml_config).get_run_configuration(self.scenario)
        insert_auto_generated_cache_directory_if_needed(resources)
        cache_directory = resources["cache_directory"]
        self.run_manager.setup_new_run(cache_directory, resources)
        run_id, cache_directory = self.run_manager.run_id, self.run_manager.get_current_cache_directory()
        self.run_manager.add_row_to_history(run_id, resources, "done")

        if create_baseyear_cache:
            self.run_manager.create_baseyear_cache(resources)

        ## good for testing
        # run_id = 275
        # cache_directory = '/home/lmwang/opus/data/paris_zone/runs/run_275.2012_05_26_00_20'
        assert run_id is not None
        assert cache_directory is not None
        return run_id, cache_directory

    def update_parameters(
        self, est_v, cache_directory, simulation_state, dataset_pool, calib_datasets, *args, **kwargs
    ):
        i_est_v = 0
        current_year = simulation_state.get_current_time()
        simulation_state.set_current_time(self.base_year)
        simulation_state.set_cache_directory(cache_directory)

        for dataset_name, calib in calib_datasets.iteritems():
            dataset, calib_attr, index = calib
            if type(calib_attr) == str:
                dtype = dataset[calib_attr].dtype
                dataset[calib_attr][index] = (est_v[i_est_v : i_est_v + index.size]).astype(dtype)
                i_est_v += index.size
            elif type(calib_attr) in (list, tuple):
                for attr in calib_attr:
                    dtype = dataset[attr].dtype
                    dataset[attr][index] = (est_v[i_est_v : i_est_v + index.size]).astype(dtype)
                    i_est_v += index.size
            else:
                raise TypeError, "Unrecongized data type in calib_datasets"

            # dtype = dataset[calib_attr].dtype
            # dataset[calib_attr][index] = (est_v[i_est_v:i_est_v+index.size]).astype(dtype)
            # flush dataset
            dataset.flush_dataset()
            # i_est_v += index.size
        simulation_state.set_current_time(current_year)

    def update_prediction(self, est_v, simulation_state, dataset_pool, calib_datasets, *args, **kwargs):
        option_group = RestartRunOptionGroup()
        option_group.parser.set_defaults(project_name=self.project_name, skip_cache_cleanup=self.skip_cache_cleanup)

        options, args = option_group.parse()
        if self.run_manager is None:
            self.run_manager = RunManager(option_group.get_services_database_configuration(options))

        if lock != None:
            lock.acquire()
        ## query runs available for re-use
        runs_done = self.run_manager.get_run_info(run_ids=self.run_ids, status="done")
        create_baseyear_cache = False
        import pdb

        pdb.set_trace()
        if len(runs_done) == 0:  ##there is no re-usable run directory, init a new run
            run_id, cache_directory = self.init_run(create_baseyear_cache=False)
            self.run_ids.append(run_id)
            create_baseyear_cache = True
            logger.log_status("Initializing new run with id " + str(run_id))
        else:
            run_id = runs_done[0].run_id  ##take the first 'done' run_id
            cache_directory = self.run_manager.get_cache_directory(run_id)
            logger.log_status("Using old run with id " + str(run_id))
        resources = self.run_manager.get_resources_for_run_id_from_history(run_id, filter_by_status=False)
        self.run_manager.add_row_to_history(run_id, resources, "taken")
        if lock != None:
            lock.release()

        if create_baseyear_cache:
            self.run_manager.create_baseyear_cache(resources)

        self.update_parameters(est_v, cache_directory, simulation_state, dataset_pool, calib_datasets, *args, **kwargs)
        restart_run(option_group=option_group, args=[run_id, self.start_year])

        prediction = self.summarize_prediction(cache_directory, simulation_state, dataset_pool, calib_datasets)
        return prediction

    def summarize_prediction(self, cache_directory, simulation_state, dataset_pool, calib_datasets):
        dataset_name = VariableName(self.target_expression).get_dataset_name()
        current_year = simulation_state.get_current_time()
        simulation_state.set_current_time(self.end_year)
        simulation_state.set_cache_directory(cache_directory)
        # force reload
        dataset_pool.remove_all_datasets()
        dataset = dataset_pool[dataset_name]
        ids = dataset.get_id_attribute()
        results = dataset.compute_variables(self.target_expression, dataset_pool=dataset_pool)
        simulation_state.set_current_time(current_year)
        return dict(zip(ids, results))

    def read_target(self, target_file):
        ## read (& process) target numbers into a dictionary: {id:value}
        ## csv file with header
        ## id, target
        header = file(target_file, "r").readline().strip().split(",")
        contents = np.genfromtxt(target_file, delimiter=",", comments="#", skip_header=1)
        target = dict(zip(contents[:, 0], contents[:, 1]))

        return target

    def target_func(self, est_v, func=lambda x, y: np.sum(np.abs(x - y)), **kwargs):
        """ Target function."""

        simulation_state = SimulationState()
        simulation_state.set_current_time(self.base_year)
        simulation_state.set_cache_directory(self.cache_directory)
        attribute_cache = AttributeCache()
        dataset_pool = SessionConfiguration(
            new_instance=True, package_order=self.package_order, in_storage=attribute_cache
        ).get_dataset_pool()

        calib_datasets = {}
        for dataset_name, calib_attr in calib_datasets.iteritems():
            dataset = dataset_pool.get_dataset(dataset_name, dataset_arguments={"id_name": []})
            assert (
                subset is None
                or subset.get(dataset_name, None) is None
                or subset_patterns is None
                or subset_patterns.get(dataset_name, None) is None
            )
            if subset is not None and subset.get(dataset_name, None) is not None:
                subset_attr, subset_cond = subset.get(dataset_name)
                index = np.in1d(dataset[subset_attr], subset_cond)
            elif subset_patterns is not None and subset_patterns.get(dataset_name, None) is not None:
                subset_attr, subset_pattern = subset_patterns.get(dataset_name)
                index = array([True if re.search(subset_pattern, attr_v) else False for attr_v in dataset[subset_attr]])
            else:
                index = arange(dataset.size(), dtype="i")

            calib_datasets[dataset_name] = [dataset, calib_attr, index]

        prediction = self.update_prediction(est_v, simulation_state, dataset_pool, calib_datasets, **kwargs)
        ## allow keys in target not appearing in prediction
        ## assuming their values to be 0
        ### every key in target should appear in prediction
        # assert np.all( np.in1d(self.target.keys(), prediction.keys()) )
        target = np.array(self.target.values())
        predct = np.array([prediction[k] if prediction.has_key(k) else 0 for k in self.target.keys()])
        results = func(predct, target)

        return results
예제 #38
0
def add_runs_to_services_db_from_disk(projects = None):
        
    server_config = ServicesDatabaseConfiguration()
    
    if server_config.protocol == 'sqlite':
        

        
        datapath = paths.OPUS_DATA_PATH
        
            
        for project_name in os.listdir(datapath):
            if projects is not None and project_name not in projects: continue
            
            if not os.path.isdir(os.path.join(datapath, project_name)): continue
            os.environ['OPUSPROJECTNAME'] = project_name
            server = DatabaseServer(server_config)
            server.drop_database(database_name = 'run_activity')
            server.close()
            
            run_manager = RunManager(server_config)
            
            baseyear_directory = os.path.join(datapath, project_name, 'base_year_data')
            
            if os.path.exists(baseyear_directory):
                years = []
                if os.path.exists(baseyear_directory):
                    for dir in os.listdir(baseyear_directory):
                        if len(dir) == 4 and dir.isdigit():
                            years.append(int(dir))
                    start_year = min(years)
                    end_year = max(years)
                    run_name = 'base_year_data'
                    run_id = run_manager._get_new_run_id()
                    resources = {
                         'cache_directory': baseyear_directory,
                         'description': 'base year data',
                         'years': (start_year, end_year)
                    }
                    logger.log_status('Adding run %s of project %s to run_activity table'%(run_name, project_name))
                    run_manager.add_row_to_history(run_id = run_id, 
                                                   resources = resources, 
                                                   status = 'done', 
                                                   run_name = run_name)

            data_directory = os.path.join(datapath, project_name, 'runs')
            if not os.path.exists(data_directory): continue
            
            for run_name in os.listdir(data_directory):
                try:
                    cache_directory = os.path.join(data_directory,run_name)
                    years = []
                    if not os.path.isdir(cache_directory): continue
                    
                    for dir in os.listdir(cache_directory):
                        if len(dir) == 4 and dir.isdigit():
                            years.append(int(dir))
                    start_year = min(years)
                    end_year = max(years)
                    run_id = run_manager._get_new_run_id()
                    resources = {
                         'cache_directory': cache_directory,
                         'description': '',
                         'years': (start_year, end_year)
                    }
                    logger.log_status('Adding run %s of project %s to run_activity table'%(run_name, project_name))
                    run_manager.add_row_to_history(run_id = run_id, 
                                                   resources = resources, 
                                                   status = 'done', 
                                                   run_name = run_name)
                except: pass        
예제 #39
0
    def __init__(self):
        GenericOptionGroup.__init__(self, usage="python %prog [options] configuration",
            description="Delete results of a simulation run.")
        self.parser.add_option("--run-id", dest="run_id", default=None, 
                                action="store", 
                                help="The simulation run to delete.")
        self.parser.add_option("--years-to-delete", dest="years_to_delete", 
                                default=None, action="store", 
                                help="Python expression specifying list of years to delete from the simulation's cache.")

if __name__ == "__main__":
    try: import wingdbstub
    except: pass
    option_group = StartRunOptionGroup()
    parser = option_group.parser
    (options, args) = parser.parse_args()

    
    run_manager = RunManager(option_group.get_services_database_configuration(options))

    if options.run_id is None:
        parser.print_help()
    elif options.years_to_delete:
        years_to_delete = eval(options.years_to_delete)
        if not isinstance(years_to_delete, list):
            years_to_delete = [years_to_delete]
        run_manager.delete_year_dirs_in_cache(options.run_id, 
                                                 years_to_delete=years_to_delete)
    else:
        run_manager.delete_everything_for_this_run(options.run_id)
예제 #40
0
                                help="name of the scenario to run")
        self.parser.add_option("--directory-to-cache", dest="existing_cache_to_copy", default=None,
                                action="store",
                                help="Directory containing data to put in new cache.")
        self.parser.add_option("--years-to-cache", dest="years_to_cache",
                                default=None, action="store",
                                help="List of years of data to take from the directory-to-cache (default is all years).")

if __name__ == "__main__":
    try: import wingdbstub
    except: pass
    option_group = StartRunSetOptionGroup()
    parser = option_group.parser
    (options, args) = parser.parse_args()

    run_manager = RunManager(option_group.get_services_database_configuration(options))

    if options.pickled_resource_file is not None:
        f = file(options.pickled_resource_file, 'r')
        try:
            config = pickle.load(f)
        finally:
            f.close()

    elif options.configuration_path is not None:
        opus_path = options.configuration_path
        try:
            config = get_config_from_opus_path(opus_path)
        except ImportError:
            # TODO: Once all fully-specified configurations are stored as classes,
            #       get rid of this use.
예제 #41
0
        self.parser.add_option("--years-to-cache", dest="years_to_cache", default=None, 
                                help="List of years of data to take from the directory-to-cache (default is all years).")                   
        self.parser.add_option("--run-as-single-process", dest="run_as_single_process", default=False, 
                                help="Determines if multiple processes may be used.")
        self.parser.add_option("-p", "--profile", dest="profile_filename", default=None, 
                                help="Turn on code profiling. Output data are in python hotshot format.")
        

if __name__ == "__main__":
    #try: import wingdbstub
    #except: pass
    option_group = StartRunOptionGroup()
    parser = option_group.parser
    (options, args) = parser.parse_args()
    
    run_manager = RunManager(option_group.get_services_database_configuration(options))
    run_as_multiprocess = not options.run_as_single_process
    
    if options.pickled_resource_file is not None:
        f = file(options.pickled_resource_file, 'r')
        try:
            config = pickle.load(f)
        finally:
            f.close()
    elif options.configuration_path is not None:
        opus_path = options.configuration_path
        try:
            config = get_config_from_opus_path(opus_path)
        except ImportError:
            # TODO: Once all fully-specified configurations are stored as classes,
            #       get rid of this use.
    password = None
    if options.server_file is None:
        if options.server is None:
            hostname = raw_input('Hostname [%s]: ' %
                                 RemoteRunSet.default_hostname)
            if len(hostname) == 0:
                hostname = RemoteRun.default_hostname
        else:
            hostname = options.server
        if hostname <> 'localhost':
            if options.user is None:
                username = raw_input('Username [%s]: ' %
                                     RemoteRunSet.default_username)
                if len(username) == 0:
                    username = RemoteRun.default_username
                else:
                    username = options.user
            password = getpass.getpass('Password for %s@%s: ' %
                                       (username, hostname))

    #try: import wingdbstub
    #except: pass
    run_manager = RunManager(
        option_group.get_services_database_configuration(options))
    run = RemoteRunSet(options.server_file, hostname, username, password,
                       options.server, options.database_name,
                       options.skip_travel_model, options.skip_urbansim,
                       run_manager)
    run.run(options.start_year, options.end_year, options.configuration_path,
            options.run_id_file)
예제 #43
0
                               dest="force",
                               default=False,
                               action="store_true",
                               help="force to overwrite pre-existing run_id")


if __name__ == "__main__":
    try:
        import wingdbstub
    except:
        pass
    option_group = OptionGroup()
    parser = option_group.parser
    (options, args) = parser.parse_args()

    run_manager = RunManager(
        option_group.get_services_database_configuration(options))

    if options.configuration_path is not None:
        opus_path = options.configuration_path
        try:
            config = get_config_from_opus_path(opus_path)
        except ImportError:
            import_stmt = 'from %s import run_configuration as config' % opus_path
            exec(import_stmt)
    config['cache_directory'] = options.cache_directory

    results = run_manager.storage.GetResultsFromQuery(
        "SELECT * from run_activity WHERE run_id = %s " % options.run_id)

    if len(results) > 1 and not options.force:
        print "WARNING: run_id %s exists in run_activity. Use --force to override." % options.run_id
예제 #44
0
def main():
    option_group = ReportOptionGroup()
    parser = option_group.parser
    (options, args) = parser.parse_args()

    if not options.xml_configuration or \
           not options.years or \
           not options.output:
        print "ERROR: -x, -y, and -o are required options"
        sys.exit(1)

    # determine the run_id from the command line args
    if options.run_id:
        run_id = options.run_id
    elif options.cache_directory:
        run_id = hudson_common.run_id_from_cache_dir(options.cache_directory)
        if not run_id:
            print "Failed to parse run ID from cache directory name"
            sys.exit(1)
    else:
        # get most recent succesful run from DB
        conn = MySQLdb.connect('paris.urbansim.org', 'hudson', os.getenv('HUDSON_DBPASS'), 'services');
        c = conn.cursor()
        c.execute("SELECT max(date_time), run_id FROM run_activity where status='done'")
        results = c.fetchone()
        run_id = str(results[1])

    # Note the dummy restart_year.  Because we're not actually going to
    # restart the run, this will be ignored.
    run_manager = RunManager(option_group.get_services_database_configuration(options))
    run_resources = run_manager.create_run_resources_from_history(run_id=run_id,
                                                                  restart_year=2010)
    cache_directory = run_resources['cache_directory']
    scenario = run_resources['scenario_name']

    # The cache directory may not exist if this script is being run on a hudson
    # slave that did not perform the original run.  Ensure that we mount it if
    # nec.
    hudson_common.mount_cache_dir(run_resources)

    # Hudson appends "_hudson" to the scenario names.  Peel this off
    # if it's present.
    scenario = scenario.replace("_hudson", "")

    # prepare the report staging directory
    output_dir = os.path.join(options.output, scenario, "run_" + run_id)
    if os.path.exists(output_dir):
        shutil.rmtree(output_dir)
    os.makedirs(output_dir)
    f = open(os.path.join(output_dir, "README.txt"), 'w')

    years = eval(options.years)
    text = ""
    if os.getenv("HUDSON_DRAFT_RESULTS") == "true":
        text += "\n".join(textwrap.wrap(
"""NOTICE: This report contains unofficial draft model results that have not
been reviewed or approved by any agency.
""")) + "\n\n"
    text += "\n".join(textwrap.wrap(
"""This report has been auto-generated by urbansim for the following model run:
"""
)) + """
project: bay_area_parcel
scenario: %s
years: %d-%d
cached data: %s
""" % (scenario, years[0], years[-1],
       os.sep.join(cache_directory.split(os.sep)[-3:]))
    comment = os.getenv("HUDSON_COMMENT")
    if comment:
        text += "\n".join(textwrap.wrap("comment: " + comment)) + "\n"
    f.write(text.replace("\n", "\r\n"))
    f.close()

    # prepare the indicators
    shutil.rmtree(os.path.join(output_dir, "indicators"), True)
    for indicator_batch in ["county_indicators", "zone_data", "superdistrict_indicators", "regional_indicators_short","pda_indicators","abag_eir_area_permutation"]:
        urbansim.tools.make_indicators.run(options.xml_configuration,
                                           indicator_batch,
                                           None,
                                           cache_directory,
                                           options.years)

    my_location = os.path.split(__file__)[0]
    shp_path = os.path.join(os.getenv("OPUS_HOME"), "data", "bay_area_parcel", "shapefiles")
    
    ##county-level summary report
    pdf_county_script = os.path.join(my_location, "summarize_county_indicators_map.R")
    cmd = "Rscript %s %s %d %d %s %s %s %s" % (pdf_county_script,
                                   os.path.join(cache_directory, "indicators"),
                                   years[0], years[-1],shp_path,"bayarea_counties.shp", scenario, "county")
    print "Summarizing county indicators: " + cmd
    if os.system(cmd) != 0:
        print "WARNING: Failed to generate county indicators"
       
    ##superdistrict-level summary report --never mind the name of the script; is en route to being generalized   
    pdf_supdist_script = os.path.join(my_location, "summarize_superdistrict_indicators_map.R")
    cmd = "Rscript %s %s %d %d %s %s %s %s" % (pdf_supdist_script,
                                   os.path.join(cache_directory, "indicators"),
                                   years[0], years[-1],shp_path,"superdistricts.shp", scenario, "superdistrict")
    print "Summarizing superdistrict indicators: " + cmd
    if os.system(cmd) != 0:
        print "WARNING: Failed to generate superdistrict indicators"

    
    shutil.copytree(os.path.join(cache_directory, "indicators"),
                    os.path.join(output_dir, "indicators"),
                    ignore=shutil.ignore_patterns("*_stored_data*", "*.log"))

    # add the travel model EMFAC output to the web report
    config = XMLConfiguration(options.xml_configuration).get_run_configuration(scenario)
    travel_model = os.getenv("HUDSON_TRAVEL_MODEL")
    travel_model_for_R ="FALSE"
    if travel_model and travel_model.lower() == "full":
        print "Copying over travel model output"
        travel_model_for_R = "TRUE"
        tm_base_dir = mtc_common.tm_get_base_dir(config)
        tm_dir = os.path.join(tm_base_dir, "runs", cache_directory.split(os.sep)[-1])
        for f in glob.glob(os.path.join(tm_dir, '*', 'emfac', 'output', 'EMFAC2011-SG Summary*Group 1.*')):
            shutil.copy(f, output_dir)
		
    p = os.path.join(cache_directory, "mtc_data")
    if os.path.exists(p):
        shutil.copytree(p, os.path.join(output_dir, "mtc_data"))
    
    #Generate PDA comparison chart
    pda_script = os.path.join(my_location, "pda_compare.R")
    cmd = "Rscript %s %s %d %d %s %s" % (pda_script,
                                   os.path.join(cache_directory, "indicators"),
                                   years[0], years[-1], run_id, scenario)
    print "Creating pda comparison chart: " + cmd
    
    
    #Generate county x PDA facet chart
    pda_county_script = os.path.join(my_location, "growth_by_county_by_pda.R")
    cmd = "Rscript %s %s %d %d %s %s" % (pda_county_script,
                                   os.path.join(cache_directory, "indicators"),
                                   years[0], years[-1], run_id, scenario)
    print "Creating county by pda chart: " + cmd
    
    
    # topsheet--need the EMFAC outputs to be generated first, so this goes last
    #travel_model_for_R = "TRUE" if travel_model else "FALSE" 
    topsheet_script = os.path.join(my_location, "regional_indicators.R")
    cmd = "Rscript %s %s %d %d %s %s %s" % (topsheet_script,
                                   os.path.join(cache_directory, "indicators"),
                                   years[0], years[-1], run_id, travel_model_for_R, scenario)
    print "Creating topsheet: " + cmd
    if os.system(cmd) != 0:
        print "WARNING: Failed to generate indicators"
예제 #45
0
    def on_pbnStartModel_released(self):
        duplicate = False
        self.diagnostic_go_button.setEnabled(True)

        if self.running and not self.paused:
            # Take care of pausing a run
            success = self.runThread.pause()
            if success:
                self.paused = True
                self.timer.stop()
                self.pbnStartModel.setText(QString("Resume simulation run..."))
        elif self.running and self.paused:
            # Need to resume a paused run
            success = self.runThread.resume()
            if success:
                self.paused = False
                self.timer.start(1000)
                self.pbnStartModel.setText(QString("Pause simulation run..."))
        elif not self.running:
            run_name = str(self.leRunName.text())
            if run_name == '':
                run_name = None
            else:
                run_id = None
                run_nodes = get_available_run_nodes(self.project)
                for run_node in run_nodes:
                    existing_run_name = run_node.tag
                    if run_name == existing_run_name:
                        duplicate = True
                        r = run_node.get('run_id')
                        if r is not None:
                            run_id = int(r)
                        break
                if duplicate:
                    dlg_dup = OverwriteRunDialog(self)

                    if dlg_dup.exec_() == QDialog.Rejected:
                        return
                    delete_simulation_run(
                        self.project,
                        run_node.tag)  # todo change to run_node.get('name')
            # Update the XML
            self.project.update_xml_config()
            self.updateConfigAndGuiForRun()

            # Fire up a new thread and run the model
            self.pbnStartModel.setText(QString("Pause simulation run..."))
            # References to the GUI elements for status for this run...
            self.progressBarTotal = self.runProgressBarTotal
            self.progressBarYear = self.runProgressBarYear
            self.progressBarModel = self.runProgressBarModel

            #self.pbnRemoveModel.setEnabled(False)
            #self.pbnStartModel.setEnabled(False)

            # Initializing values
            self.progressBarTotal.setValue(0)
            self.progressBarYear.setValue(0)
            self.progressBarModel.setValue(0)
            self.progressBarTotal.setRange(0, 0)
            self.progressBarYear.setRange(0, 0)
            self.progressBarModel.setRange(0, 0)

            batch_name = str(self.cboOptionalIndicatorBatch.currentText())
            if batch_name == '(None)':
                batch_name = None

            self.runThread = RunModelThread(get_mainwindow_instance(), self,
                                            batch_name, run_name)

            if duplicate and run_id is not None:
                from opus_core.services.run_server.run_manager import RunManager as ServicesRunManager
                run_manager = ServicesRunManager(
                    ServicesDatabaseConfiguration())
                run_manager.delete_everything_for_this_run(run_id=run_id)
                run_manager.close()

            # Use this signal from the thread if it is capable of producing its own status signal
            QObject.connect(self.runThread,
                            SIGNAL("runFinished(PyQt_PyObject)"),
                            self.runFinishedFromThread)
            QObject.connect(self.runThread, SIGNAL("runError(PyQt_PyObject)"),
                            self.runErrorFromThread)
            # Use this timer to call a function in the thread to check status if the thread is unable
            # to produce its own signal above
            self.timer = QTimer()
            QObject.connect(self.timer, SIGNAL("timeout()"),
                            self.runStatusFromThread)
            self.timer.start(1000)
            self.running = True
            self.paused = False
            self.runThread.start()
        else:
            print "Unexpected state in the model run..."
예제 #46
0
            "--profile",
            dest="profile_filename",
            default=None,
            help=
            "Turn on code profiling. Output data are in python hotshot format."
        )


if __name__ == "__main__":
    #try: import wingdbstub
    #except: pass
    option_group = StartRunOptionGroup()
    parser = option_group.parser
    (options, args) = parser.parse_args()

    run_manager = RunManager(
        option_group.get_services_database_configuration(options))
    run_as_multiprocess = not options.run_as_single_process

    if options.pickled_resource_file is not None:
        f = file(options.pickled_resource_file, 'r')
        try:
            config = pickle.load(f)
        finally:
            f.close()
    elif options.configuration_path is not None:
        opus_path = options.configuration_path
        try:
            config = get_config_from_opus_path(opus_path)
        except ImportError:
            # TODO: Once all fully-specified configurations are stored as classes,
            #       get rid of this use.
예제 #47
0
                               dest="project_name",
                               default='',
                               help="The project name")


if __name__ == "__main__":
    option_group = OptionGroup()
    parser = option_group.parser
    (options, args) = parser.parse_args()

    try:
        run_id = int(args[0])
    except IndexError:
        parser.error("run_id must be provided.")
        parser.print_help()
        sys.exit(1)
    if len(args) == 2:
        pickle_file = args[1]
    else:
        pickle_file = "resources.pickle"

    run_manager = RunManager(
        option_group.get_services_database_configuration(options))
    if options.project_name:
        run_manager.update_environment_variables(
            run_resources={'project_name': options.project_name})

    resources = run_manager.get_resources_for_run_id_from_history(
        run_id=run_id)
    write_resources_to_file(pickle_file, resources)
예제 #48
0
            action="store",
            help=
            "List of years of data to take from the directory-to-cache (default is all years)."
        )


if __name__ == "__main__":
    try:
        import wingdbstub
    except:
        pass
    option_group = StartRunSetOptionGroup()
    parser = option_group.parser
    (options, args) = parser.parse_args()

    run_manager = RunManager(
        option_group.get_services_database_configuration(options))

    if options.pickled_resource_file is not None:
        f = file(options.pickled_resource_file, 'r')
        try:
            config = pickle.load(f)
        finally:
            f.close()

    elif options.configuration_path is not None:
        opus_path = options.configuration_path
        try:
            config = get_config_from_opus_path(opus_path)
        except ImportError:
            # TODO: Once all fully-specified configurations are stored as classes,
            #       get rid of this use.
class OptionGroup(GenericOptionGroup):
    def __init__(self):
        GenericOptionGroup.__init__(self, usage="python %prog [options] run_id [pickle_file]",
               description="dump resources.pickle from services db for the given run_id")
        self.parser.add_option("-p", "--project-name", dest="project_name", 
                                default='',help="The project name")
                                
if __name__ == "__main__":
    option_group = OptionGroup()
    parser = option_group.parser
    (options, args) = parser.parse_args()

    try:
        run_id = int(args[0])
    except IndexError:
        parser.error("run_id must be provided.")
        parser.print_help()
        sys.exit(1)
    if len(args) == 2: 
        pickle_file = args[1]
    else:
        pickle_file = "resources.pickle"

    run_manager = RunManager(option_group.get_services_database_configuration(options))
    if options.project_name:
        run_manager.update_environment_variables(run_resources={'project_name':options.project_name}) 

    resources = run_manager.get_resources_for_run_id_from_history(run_id=run_id)
    write_resources_to_file(pickle_file, resources)
                                 
예제 #50
0
        self.parser.add_option("-p", "--project-name", dest="project_name", 
                                default='',help="The name project name")
        self.parser.add_option("--skip-urbansim", dest="skip_urbansim", default=False, 
                                action="store_true", 
                                help="Skip running UrbanSim for the restart year.")
        self.parser.add_option("--create-baseyear-cache-if-not-exists", dest="create_baseyear_cache_if_not_exists", default=False, 
                                action="store_true",
                                help="Create baseyear cache if not already exists")
        self.parser.add_option("--skip-cache-cleanup", dest="skip_cache_cleanup", 
                                default=False, action="store_true", 
                                help="Skip removing year caches for this and future years.")
                                
if __name__ == "__main__":
    option_group = RestartRunOptionGroup()
    parser = option_group.parser
    (options, args) = parser.parse_args()

    run_manager = RunManager(option_group.get_services_database_configuration(options))

    if len(args) < 2:
        parser.print_help()
    else:
        run_id, year = (int(args[0]), int(args[1]))
        run_manager.restart_run(run_id, 
                                year,
                                options.project_name,
                                skip_urbansim=options.skip_urbansim,
                                create_baseyear_cache_if_not_exists=options.create_baseyear_cache_if_not_exists,
                                skip_cache_cleanup=options.skip_cache_cleanup)
                                  
예제 #51
0
def main():
    option_group = UrbanvisionOptionGroup()
    parser = option_group.parser
    (options, args) = parser.parse_args()

    if not options.run_id and not options.cache_directory:
        print "ERROR: either -r or -c is required"
        sys.exit(1)

    if options.run_id:
        run_id = options.run_id
    elif options.cache_directory:
        run_id = run_id_from_cache_dir(options.cache_directory)
        if not run_id:
            print "Failed to parse run ID from cache directory name"
            sys.exit(1)

    # Note the dummy restart_year.  Because we're not actually going to
    # restart the run, this will be ignored.
    run_manager = RunManager(option_group.get_services_database_configuration(options))
    run_resources = run_manager.create_run_resources_from_history(run_id=run_id,
                                                                  restart_year=2010)
    cache_directory = run_resources['cache_directory']
    mount_cache_dir(run_resources)
    scenario = run_resources['scenario_name']
    scenario = scenario.replace("_hudson", "")

    # Ensure that there's a suitable scenario abiding by the urbanvision
    # convention
    scenario_name = ("%s run %s" % (scenario, run_id)).replace("_", " ")
    passwd = os.environ['OPUS_DBPASS']
    conn_string = "host='paris.urbansim.org' dbname='bayarea' user='******' password='******'" % passwd
    conn = psycopg2.connect(conn_string)
    cursor = conn.cursor()
    s = "select id from scenario where name='{}'".format(scenario_name)
    cursor.execute(s)
    records = cursor.fetchall()
    if len(records) == 0:
        print "Creating new scenario '" + scenario_name + "'"
        s = "insert into scenario (name, parent) values ('{}', 1)".format(scenario_name)
        cursor.execute(s)
        conn.commit()
        s = "select id from scenario where name='{}'".format(scenario_name)
        cursor.execute(s)
        records = cursor.fetchall()
        assert(len(records) == 1)
        id = records[0][0]
    elif len(records) == 1:
        id = records[0][0]
    else:
        print "ERROR: Found more than one scenario named %s!" % scenario_name
        cursor.close()
        conn.close()
        sys.exit(1)

    cursor.close()
    conn.close()

    # Now that we have the scenario id, we can run the script that inserts the
    # records into the db
    psql = os.path.join(os.path.split(__file__)[0], "process_building_to_sql.py")
    cmd = "%s %s %s %d" % (sys.executable, psql, cache_directory, id)
    cmd += " | psql -h paris.urbansim.org -q -U urbanvision bayarea > /dev/null"
    print "Exporting buildings to db: " + cmd
    if os.system("export PGPASSWORD=$OPUS_DBPASS; " + cmd) != 0:
        print "ERROR: Failed to export buildings to urbanvision DB"
        sys.exit(1)
                               help="cache directory")
        self.parser.add_option("--run-id", dest="run_id", default=None, 
                               help="which run_id to update")
        self.parser.add_option("--force", dest="force", 
                               default=False, action="store_true", 
                               help="force to overwrite pre-existing run_id")
        

if __name__ == "__main__":
    try: import wingdbstub
    except: pass
    option_group = OptionGroup()
    parser = option_group.parser
    (options, args) = parser.parse_args()

    run_manager = RunManager(option_group.get_services_database_configuration(options))

    if options.configuration_path is not None:
        opus_path = options.configuration_path
        try:
            config = get_config_from_opus_path(opus_path)
        except ImportError:
            import_stmt = 'from %s import run_configuration as config' % opus_path
            exec(import_stmt)
    config['cache_directory'] = options.cache_directory
    
    results = run_manager.storage.GetResultsFromQuery("SELECT * from run_activity WHERE run_id = %s " % options.run_id)

    if len(results) > 1 and not options.force:
        print "WARNING: run_id %s exists in run_activity. Use --force to override." % options.run_id
        sys.exit()
예제 #53
0
class Calibration(object):
    ''' Class to calibrate UrbanSim model coefficients.
    
    '''
    def __init__(self,
                 xml_config,
                 scenario,
                 calib_datasets,
                 target_expression,
                 target_file,
                 subset=None,
                 subset_patterns=None,
                 skip_cache_cleanup=False,
                 log_directory=None):
        """
        - xml_config: xml configuration file, for ex '/home/atschirhar/opus/project_configs/paris_zone.xml'
        - scenario: name of scenario to run for calibration, where models_to_run and simulation years are specified
        - calib_datasets: dictionary specifying dataset names and attributes to be calibrated, e.g.
                  {'establishment_location_choice_model_coefficients': 'estimate'}
        - target_expression: opus expression computing values from prediction to be compared with targets 
        - target_file: name of csv file providing targets 
        - subset: dictionary specifying the dataset to be calibrated,
                  {'etablishment_location_choice_model_coefficients': ['coefficient_name', ['paris_celcm, 'biotech_celcm']]}
          subset and subset_patterns can not be both specified for the same dataset
        - subset_patterns: dictionary specifying the dataset to be calibrated through a regular expression (re) pattern
                  {'etablishment_location_choice_model_coefficients': ['coefficient_name', '*_celcm']} 
          subset and subset_patterns can not be both specified for the same dataset

        """
        self.target_expression = target_expression
        self.target = self.read_target(target_file)

        self.run_manager = None
        self.xml_config = xml_config
        self.scenario = scenario
        self.skip_cache_cleanup = skip_cache_cleanup
        self.run_id, self.cache_directory = self.init_run()
        self.run_ids = [
            self.run_id
        ]  #allow starting of multiple runs for parallel optimization
        self.log_directory = log_directory
        if self.log_directory is None:
            self.log_directory = self.cache_directory  #legacy

        log_file = os.path.join(self.log_directory, "calibration.log")
        logger.enable_file_logging(log_file)

        dict_config = XMLConfiguration(self.xml_config).get_run_configuration(
            self.scenario)
        ## get parameters from config
        self.base_year = dict_config['base_year']
        self.start_year, self.end_year = dict_config['years']
        self.project_name = dict_config['project_name']
        self.package_order = dict_config[
            'dataset_pool_configuration'].package_order

    @log_block("Start Calibration")
    def run(self,
            optimizer='lbfgsb',
            results_pickle_prefix="calib",
            optimizer_kwargs={}):
        ''' Call specifized optimizer to calibrate
        
        Arguments:
            - optimizer: optimization method chosen (fmin_bfgs, simulated anneal etc.)
            - results_pickle_prefix: prefix of the pickle file name that will be saved after the simulation; if None, results is not saved
            
        Returns:
            - the results from the opimizater
            - a pickle dump of the results in the cache_directory, if results_pickle_prefix is specified
        
        '''

        simulation_state = SimulationState()
        simulation_state.set_current_time(self.base_year)
        simulation_state.set_cache_directory(self.cache_directory)
        attribute_cache = AttributeCache()
        dataset_pool = SessionConfiguration(
            new_instance=True,
            package_order=self.package_order,
            in_storage=attribute_cache).get_dataset_pool()

        calib_datasets = {}
        for dataset_name, calib_attr in calib_datasets.iteritems():
            dataset = dataset_pool.get_dataset(
                dataset_name, dataset_arguments={'id_name': []})
            assert subset is None or subset.get(dataset_name, None) is None or \
                   subset_patterns is None or subset_patterns.get(dataset_name, None) is None
            if subset is not None and subset.get(dataset_name,
                                                 None) is not None:
                subset_attr, subset_cond = subset.get(dataset_name)
                index = np.in1d(dataset[subset_attr], subset_cond)
            elif subset_patterns is not None and subset_patterns.get(
                    dataset_name, None) is not None:
                subset_attr, subset_pattern = subset_patterns.get(dataset_name)
                index = array([
                    True if re.search(subset_pattern, attr_v) else False
                    for attr_v in dataset[subset_attr]
                ])
            else:
                index = arange(dataset.size(), dtype='i')

            calib_datasets[dataset_name] = [dataset, calib_attr, index]

        init_v = array([], dtype='f8')
        for dataset_name, calib in calib_datasets.iteritems():
            dataset, calib_attr, index = calib
            if type(calib_attr) == str:
                init_v = np.concatenate((init_v, dataset[calib_attr][index]))
            elif type(calib_attr) in (list, tuple):
                for attr in calib_attr:
                    init_v = np.concatenate((init_v, dataset[attr][index]))
            else:
                raise TypeError, "Unrecongized data type in calib_datasets"

        t0 = time.time()

        if is_parallelizable == True: set_parallel(True)

        print OKBLUE + "\noptimizer = {} (is_parallel = {})".format(
            optimizer, is_parallelizable) + ENDC
        print OKBLUE + "-------------------------------------------------------\n" + ENDC
        if optimizer == 'bfgs':
            default_kwargs = {
                'fprime': None,
                'epsilon': 1e-08,
                'maxiter': None,
                'full_output': 1,
                'disp': 1,
                'retall': 0,
                'callback': None
            }
            optimizer_func = fmin_bfgs
        elif optimizer == 'lbfgsb':
            default_kwargs = {
                'fprime': None,
                'approx_grad': True,
                'bounds': None,
                'factr': 1e12,
                'iprint': 1
            }

            optimizer_func = fmin_l_bfgs_b
        elif optimizer == 'anneal':
            default_kwargs = {
                'schedule': 'fast',
                'full_output': 1,
                'T0': None,
                'Tf': 1e-12,
                'maxeval': None,
                'maxaccept': None,
                'maxiter': 400,
                'boltzmann': 1.0,
                'learn_rate': 0.5,
                'feps': 1e-06,
                'quench': 1.0,
                'm': 1.0,
                'n': 1.0,
                'lower': -1,
                'upper': 1,
                'dwell': 50,
                'disp': True
            }

            optimizer_func = anneal
        elif optimizer == 'panneal':
            default_kwargs = {
                'schedule': 'fast',
                'full_output': 1,
                'T0': None,
                'Tf': 1e-12,
                'maxeval': None,
                'maxaccept': None,
                'maxiter': 400,
                'boltzmann': 1.0,
                'learn_rate': 0.5,
                'feps': 1e-06,
                'quench': 1.0,
                'm': 1.0,
                'n': 1.0,
                'lower': -1,
                'upper': 1,
                'dwell': 50,
                'disp': True,
                'cores': 24,
                'interv': 20
            }

            optimizer_func = panneal
        else:
            raise ValueError, "Unrecognized optimizer {}".format(optimizer)

        default_kwargs.update(optimizer_kwargs)
        results = optimizer_func(self.target_func, copy(init_v),
                                 **default_kwargs)

        duration = time.time() - t0
        if results_pickle_prefix is not None:
            pickle_file = "{}_{}.pickle".format(results_pickle_prefix,
                                                optimizer)
            pickle_file = os.path.join(self.log_directory, pickle_file)
            pickle.dump(results, open(pickle_file, "wb"))

        if is_parallelizable == True: set_parallel(False)

        logger.log_status('init target_func: {}'.format(
            self.target_func(init_v)))
        logger.log_status('end target_func: {}'.format(
            results[:]))  #which one?
        logger.log_status('outputs from optimizer: {}'.format(results))
        logger.log_status('Execution time: {}'.format(duration))

    def init_run(self, create_baseyear_cache=True):
        ''' init run, get run_id & cache_directory. '''
        ##avoid invoking start_run from cmd line -
        option_group = StartRunOptionGroup()
        option_group.parser.set_defaults(xml_configuration=self.xml_config,
                                         scenario_name=self.scenario)
        #run_id, cache_directory = start_run(option_group)

        options, args = option_group.parse()
        self.run_manager = RunManager(
            option_group.get_services_database_configuration(options))

        resources = XMLConfiguration(self.xml_config).get_run_configuration(
            self.scenario)
        insert_auto_generated_cache_directory_if_needed(resources)
        cache_directory = resources['cache_directory']
        self.run_manager.setup_new_run(cache_directory, resources)
        run_id, cache_directory = self.run_manager.run_id, self.run_manager.get_current_cache_directory(
        )
        self.run_manager.add_row_to_history(run_id, resources, "done")

        if create_baseyear_cache:
            self.run_manager.create_baseyear_cache(resources)

        ## good for testing
        #run_id = 275
        #cache_directory = '/home/lmwang/opus/data/paris_zone/runs/run_275.2012_05_26_00_20'
        assert run_id is not None
        assert cache_directory is not None
        return run_id, cache_directory

    def update_parameters(self, est_v, cache_directory, simulation_state,
                          dataset_pool, calib_datasets, *args, **kwargs):
        i_est_v = 0
        current_year = simulation_state.get_current_time()
        simulation_state.set_current_time(self.base_year)
        simulation_state.set_cache_directory(cache_directory)

        for dataset_name, calib in calib_datasets.iteritems():
            dataset, calib_attr, index = calib
            if type(calib_attr) == str:
                dtype = dataset[calib_attr].dtype
                dataset[calib_attr][index] = (est_v[i_est_v:i_est_v +
                                                    index.size]).astype(dtype)
                i_est_v += index.size
            elif type(calib_attr) in (list, tuple):
                for attr in calib_attr:
                    dtype = dataset[attr].dtype
                    dataset[attr][index] = (est_v[i_est_v:i_est_v +
                                                  index.size]).astype(dtype)
                    i_est_v += index.size
            else:
                raise TypeError, "Unrecongized data type in calib_datasets"

            #dtype = dataset[calib_attr].dtype
            #dataset[calib_attr][index] = (est_v[i_est_v:i_est_v+index.size]).astype(dtype)
            #flush dataset
            dataset.flush_dataset()
            #i_est_v += index.size
        simulation_state.set_current_time(current_year)

    def update_prediction(self, est_v, simulation_state, dataset_pool,
                          calib_datasets, *args, **kwargs):
        option_group = RestartRunOptionGroup()
        option_group.parser.set_defaults(
            project_name=self.project_name,
            skip_cache_cleanup=self.skip_cache_cleanup)

        options, args = option_group.parse()
        if self.run_manager is None:
            self.run_manager = RunManager(
                option_group.get_services_database_configuration(options))

        if lock != None: lock.acquire()
        ## query runs available for re-use
        runs_done = self.run_manager.get_run_info(run_ids=self.run_ids,
                                                  status='done')
        create_baseyear_cache = False
        import pdb
        pdb.set_trace()
        if len(runs_done
               ) == 0:  ##there is no re-usable run directory, init a new run
            run_id, cache_directory = self.init_run(
                create_baseyear_cache=False)
            self.run_ids.append(run_id)
            create_baseyear_cache = True
            logger.log_status('Initializing new run with id ' + str(run_id))
        else:
            run_id = runs_done[0].run_id  ##take the first 'done' run_id
            cache_directory = self.run_manager.get_cache_directory(run_id)
            logger.log_status('Using old run with id ' + str(run_id))
        resources = self.run_manager.get_resources_for_run_id_from_history(
            run_id, filter_by_status=False)
        self.run_manager.add_row_to_history(run_id, resources, "taken")
        if lock != None: lock.release()

        if create_baseyear_cache:
            self.run_manager.create_baseyear_cache(resources)

        self.update_parameters(est_v, cache_directory, simulation_state,
                               dataset_pool, calib_datasets, *args, **kwargs)
        restart_run(option_group=option_group, args=[run_id, self.start_year])

        prediction = self.summarize_prediction(cache_directory,
                                               simulation_state, dataset_pool,
                                               calib_datasets)
        return prediction

    def summarize_prediction(self, cache_directory, simulation_state,
                             dataset_pool, calib_datasets):
        dataset_name = VariableName(self.target_expression).get_dataset_name()
        current_year = simulation_state.get_current_time()
        simulation_state.set_current_time(self.end_year)
        simulation_state.set_cache_directory(cache_directory)
        #force reload
        dataset_pool.remove_all_datasets()
        dataset = dataset_pool[dataset_name]
        ids = dataset.get_id_attribute()
        results = dataset.compute_variables(self.target_expression,
                                            dataset_pool=dataset_pool)
        simulation_state.set_current_time(current_year)
        return dict(zip(ids, results))

    def read_target(self, target_file):
        ## read (& process) target numbers into a dictionary: {id:value}
        ## csv file with header
        ## id, target
        header = file(target_file, 'r').readline().strip().split(',')
        contents = np.genfromtxt(target_file,
                                 delimiter=",",
                                 comments='#',
                                 skip_header=1)
        target = dict(zip(contents[:, 0], contents[:, 1]))

        return target

    def target_func(self,
                    est_v,
                    func=lambda x, y: np.sum(np.abs(x - y)),
                    **kwargs):
        ''' Target function.'''

        simulation_state = SimulationState()
        simulation_state.set_current_time(self.base_year)
        simulation_state.set_cache_directory(self.cache_directory)
        attribute_cache = AttributeCache()
        dataset_pool = SessionConfiguration(
            new_instance=True,
            package_order=self.package_order,
            in_storage=attribute_cache).get_dataset_pool()

        calib_datasets = {}
        for dataset_name, calib_attr in calib_datasets.iteritems():
            dataset = dataset_pool.get_dataset(
                dataset_name, dataset_arguments={'id_name': []})
            assert subset is None or subset.get(dataset_name, None) is None or \
                   subset_patterns is None or subset_patterns.get(dataset_name, None) is None
            if subset is not None and subset.get(dataset_name,
                                                 None) is not None:
                subset_attr, subset_cond = subset.get(dataset_name)
                index = np.in1d(dataset[subset_attr], subset_cond)
            elif subset_patterns is not None and subset_patterns.get(
                    dataset_name, None) is not None:
                subset_attr, subset_pattern = subset_patterns.get(dataset_name)
                index = array([
                    True if re.search(subset_pattern, attr_v) else False
                    for attr_v in dataset[subset_attr]
                ])
            else:
                index = arange(dataset.size(), dtype='i')

            calib_datasets[dataset_name] = [dataset, calib_attr, index]

        prediction = self.update_prediction(est_v, simulation_state,
                                            dataset_pool, calib_datasets,
                                            **kwargs)
        ## allow keys in target not appearing in prediction
        ## assuming their values to be 0
        ### every key in target should appear in prediction
        #assert np.all( np.in1d(self.target.keys(), prediction.keys()) )
        target = np.array(self.target.values())
        predct = np.array([prediction[k] if prediction.has_key(k) else 0 \
                           for k in self.target.keys() ])
        results = func(predct, target)

        return results
예제 #54
0
            dest="years_to_delete",
            default=None,
            action="store",
            help=
            "Python expression specifying list of years to delete from the simulation's cache."
        )


if __name__ == "__main__":
    try:
        import wingdbstub
    except:
        pass
    option_group = StartRunOptionGroup()
    parser = option_group.parser
    (options, args) = parser.parse_args()

    run_manager = RunManager(
        option_group.get_services_database_configuration(options))

    if options.run_id is None:
        parser.print_help()
    elif options.years_to_delete:
        years_to_delete = eval(options.years_to_delete)
        if not isinstance(years_to_delete, list):
            years_to_delete = [years_to_delete]
        run_manager.delete_year_dirs_in_cache(options.run_id,
                                              years_to_delete=years_to_delete)
    else:
        run_manager.delete_everything_for_this_run(options.run_id)