def _read_xml_from_config(self, xml_config): ''' Extract XML information from the XML Configuration object ''' self.xml_config = xml_config self._root_node = xml_config.full_tree.getroot() if xml_config.inherited_tree is None: self._inherited_root = XMLConfiguration().full_tree.getroot() else: self._inherited_root = xml_config.inherited_tree # map id's to nodes for the inherited and the local nodes inherited_ids_to_nodes = dict( (node_identity_string(n), n) for n in self._inherited_root.getiterator()) local_ids_to_nodes = dict((node_identity_string(n), n) for n in self._root_node.getiterator() if not n.get('inherited')) self._shadowing_nodes = {} # join the local and inherited nodes on id-match for id_, node in local_ids_to_nodes.items(): if id_ in inherited_ids_to_nodes: self._shadowing_nodes[node] = inherited_ids_to_nodes[id_] if self.find('./general/project_name') is not None: self.name = self.find('./general/project_name').text else: self.name = 'unnamed_project' os.environ['OPUSPROJECTNAME'] = self.name self.dirty = False
def setUp(self): print "entering setUp" logger.log_status('Testing UrbanSim export and import functionallity for MATSim...') # get root path to test cases self.path = test_path.__path__[0] logger.log_status('Set root path for MATSim config file to: %s' % self.path) if not os.path.exists(self.path): raise StandardError("Root path doesn't exist: %s" % self.path) # get path to MATSim source files and base_year_cache self.matsim_source, self.base_year_data_source = self.get_source_files() self.destination = tempfile.mkdtemp(prefix='opus_tmp') #self.destination = '/Users/thomas/Desktop/x' # for debugging #if not os.path.exists(self.destination): # for debugging # os.mkdir(self.destination) # for debugging # load UrbanSim config to run MATSim test urbansim_config_location = os.path.join( self.path, 'configs', 'urbansim_config') #'/Users/thomas/Development/workspace/urbansim_trunk/opus_matsim/tests/test_config.xml' logger.log_status('Loading UrbanSim config: %s' % urbansim_config_location) #self.run_config = XMLConfiguration( urbansim_config_location ).get_run_configuration("Test") urbansim_config_name = "urbansim_config_for_matsim_run_test.xml" self.run_config = XMLConfiguration( os.path.join(urbansim_config_location, urbansim_config_name)).get_run_configuration("Test") # set destination for MATSim config file self.matsim_config_full = os.path.join( self.destination, "test_matsim_config.xml" ) print "leaving setUp"
def make_zone_dbfs(cache_directory): xmlconfig = XMLConfiguration(filename="sanfrancisco.xml", default_directory=r'C:\opus\project_configs', is_parent=False) runconfig = xmlconfig.get_run_configuration("sanfrancisco_baseline2009", merge_controllers=True) tm_config = runconfig['travel_model_configuration'] print tm_config['urbansim_to_tm_variable_mapping'] travel_model_years = [] for key in tm_config.iterkeys(): if isinstance(key,int) and tm_config[key].has_key('year_dir'): travel_model_years.append(key) travel_model_years.sort() zonedbfs_source_data = SourceData( cache_directory = cache_directory, run_description = "Run description is used for what?", years = travel_model_years, dataset_pool_configuration = DatasetPoolConfiguration( package_order=['sanfrancisco','urbansim','opus_core'], ), ) attrs = [] for key,val in tm_config['urbansim_to_tm_variable_mapping'].iteritems(): key = key.replace(".", "_") attrs.append("%s=%s" % (key,val)) attrs.extend([\ "pwac_bus=sanfrancisco.zone.bus_travel_time_weighted_access_by_population", "pwac_exp=sanfrancisco.zone.exp_travel_time_weighted_access_by_population", "pwac_lrt=sanfrancisco.zone.lrt_travel_time_weighted_access_by_population", "pwac_bart=sanfrancisco.zone.bart_travel_time_weighted_access_by_population", "pwac_hwy=sanfrancisco.zone.hwy_travel_time_weighted_access_by_population", "ewac_bus=sanfrancisco.zone.bus_travel_time_weighted_access_to_employment", "ewac_exp=sanfrancisco.zone.exp_travel_time_weighted_access_to_employment", "ewac_lrt=sanfrancisco.zone.lrt_travel_time_weighted_access_to_employment", "ewac_bart=sanfrancisco.zone.bart_travel_time_weighted_access_to_employment", "ewac_hwy=sanfrancisco.zone.hwy_travel_time_weighted_access_to_employment", "ttpw_bus=sanfrancisco.zone.bus_travel_time_to_751", "ttpw_exp=sanfrancisco.zone.exp_travel_time_to_751", "ttpw_lrt=sanfrancisco.zone.lrt_travel_time_to_751", "ttpw_bart=sanfrancisco.zone.bart_travel_time_to_751", "ttpw_hwy=sanfrancisco.zone.hwy_travel_time_to_751", "d2powell=sanfrancisco.zone.dist_travel_time_to_751" ]) zonedbf_indicators = [ DatasetTable( source_data = zonedbfs_source_data, dataset_name = 'zone', name = 'zone Indicators', output_type='dbf', attributes = attrs ) ] IndicatorFactory().create_indicators(indicators = zonedbf_indicators, display_error_box = False, show_results = False)
def make_zone_dbfs(cache_directory): xmlconfig = XMLConfiguration(filename="sanfrancisco.xml", default_directory=r'C:\opus\project_configs', is_parent=False) runconfig = xmlconfig.get_run_configuration("sanfrancisco_baseline2009", merge_controllers=True) tm_config = runconfig['travel_model_configuration'] print tm_config['urbansim_to_tm_variable_mapping'] travel_model_years = [] for key in tm_config.iterkeys(): if isinstance(key, int) and tm_config[key].has_key('year_dir'): travel_model_years.append(key) travel_model_years.sort() zonedbfs_source_data = SourceData( cache_directory=cache_directory, run_description="Run description is used for what?", years=travel_model_years, dataset_pool_configuration=DatasetPoolConfiguration( package_order=['sanfrancisco', 'urbansim', 'opus_core'], ), ) attrs = [] for key, val in tm_config['urbansim_to_tm_variable_mapping'].iteritems(): key = key.replace(".", "_") attrs.append("%s=%s" % (key, val)) attrs.extend([\ "pwac_bus=sanfrancisco.zone.bus_travel_time_weighted_access_by_population", "pwac_exp=sanfrancisco.zone.exp_travel_time_weighted_access_by_population", "pwac_lrt=sanfrancisco.zone.lrt_travel_time_weighted_access_by_population", "pwac_bart=sanfrancisco.zone.bart_travel_time_weighted_access_by_population", "pwac_hwy=sanfrancisco.zone.hwy_travel_time_weighted_access_by_population", "ewac_bus=sanfrancisco.zone.bus_travel_time_weighted_access_to_employment", "ewac_exp=sanfrancisco.zone.exp_travel_time_weighted_access_to_employment", "ewac_lrt=sanfrancisco.zone.lrt_travel_time_weighted_access_to_employment", "ewac_bart=sanfrancisco.zone.bart_travel_time_weighted_access_to_employment", "ewac_hwy=sanfrancisco.zone.hwy_travel_time_weighted_access_to_employment", "ttpw_bus=sanfrancisco.zone.bus_travel_time_to_751", "ttpw_exp=sanfrancisco.zone.exp_travel_time_to_751", "ttpw_lrt=sanfrancisco.zone.lrt_travel_time_to_751", "ttpw_bart=sanfrancisco.zone.bart_travel_time_to_751", "ttpw_hwy=sanfrancisco.zone.hwy_travel_time_to_751", "d2powell=sanfrancisco.zone.dist_travel_time_to_751" ]) zonedbf_indicators = [ DatasetTable(source_data=zonedbfs_source_data, dataset_name='zone', name='zone Indicators', output_type='dbf', attributes=attrs) ] IndicatorFactory().create_indicators(indicators=zonedbf_indicators, display_error_box=False, show_results=False)
def _read_xml_from_config(self, xml_config): ''' Extract XML information from the XML Configuration object ''' self.xml_config = xml_config self._root_node = xml_config.full_tree.getroot() if xml_config.inherited_tree is None: self._inherited_root = XMLConfiguration().full_tree.getroot() else: self._inherited_root = xml_config.inherited_tree self._init_shadowing_nodes() self._set_project_name()
def delete_or_update_node(self, node, new_node): ''' Delete or update a node from the XML DOM. If the node was shadowing an inherited node, the inherited node is (re-)inserted into the DOM (after merging with new_node in the case of an update) and returned. Calling delete_or_update_node on an inherited node has no effect. @node (Element) node to remove @new_node (Element) node to insert instead of the removed element; None to remove only @return the (re-inserted) node (Element) or None ''' # The three cases of deleting/updating a node: # The node is local (simplest case -- just remove it) # - if updating, simply add the new node # The node is inherited (no, wait, this is the simplest case -- do nothing) # The node is shadowing an inherited node (remove the node) # - if removing, reinsert the inherited node # - if updating, merge the new node with the inherited node and insert # helper function to clean out all child nodes from shadowing_nodes def clean_shadownodes(node): for child_node in node: clean_shadownodes(child_node) if node in self._shadowing_nodes: del self._shadowing_nodes[node] assert node not in self._shadowing_nodes parent = node.getparent() node_index = parent.index(node) inherited_node = None reinserted_node = new_node if node in self._shadowing_nodes: inherited_node = copy.deepcopy(self._shadowing_nodes[node]) if new_node is None: reinserted_node = inherited_node inherited_node = None elif node.get('inherited'): return else: pass clean_shadownodes(node) node_id = node_identity_string(node) parent.remove(node) if inherited_node is not None: assert new_node is not None XMLConfiguration._merge_nodes(inherited_node, new_node) if new_node is not None: self._add_shadowing_nodes(new_node, node_id) if reinserted_node is not None: parent.insert(node_index, reinserted_node) return reinserted_node
def test_simulation(self): eugene_dir = __import__('eugene').__path__[0] xml_config = XMLConfiguration(os.path.join(eugene_dir, 'configs', 'eugene_gridcell.xml')) option_group = StartRunOptionGroup() parser = option_group.parser # simulate 0 command line arguments by passing in [] (options, _) = parser.parse_args([]) run_manager = RunManager(option_group.get_services_database_configuration(options)) run_section = xml_config.get_run_configuration('Eugene_baseline') insert_auto_generated_cache_directory_if_needed(run_section) run_manager.setup_new_run(cache_directory = run_section['cache_directory'], configuration = run_section) run_manager.run_run(run_section)
def test_simulation(self): # check that the simulation proceeds without crashing # open the configuration for seattle_parcel.xml seattle_parcel_dir = __import__('seattle_parcel').__path__[0] xml_config = XMLConfiguration(os.path.join(seattle_parcel_dir, 'configs', 'seattle_parcel.xml')) option_group = StartRunOptionGroup() parser = option_group.parser # simulate 0 command line arguments by passing in [] (options, _) = parser.parse_args([]) run_manager = RunManager(option_group.get_services_database_configuration(options)) run_section = xml_config.get_run_configuration('Seattle_baseline') insert_auto_generated_cache_directory_if_needed(run_section) run_manager.setup_new_run(cache_directory = run_section['cache_directory'], configuration = run_section) run_manager.run_run(run_section)
def _read_xml_from_config(self, xml_config): ''' Extract XML information from the XML Configuration object ''' self.xml_config = xml_config self._root_node = xml_config.full_tree.getroot() if xml_config.inherited_tree is None: self._inherited_root = XMLConfiguration().full_tree.getroot() else: self._inherited_root = xml_config.inherited_tree # map id's to nodes for the inherited and the local nodes inherited_ids_to_nodes = dict((node_identity_string(n), n) for n in self._inherited_root.getiterator()) local_ids_to_nodes = dict((node_identity_string(n), n) for n in self._root_node.getiterator() if not n.get('inherited')) self._shadowing_nodes = {} # join the local and inherited nodes on id-match for id_, node in local_ids_to_nodes.items(): if id_ in inherited_ids_to_nodes: self._shadowing_nodes[node] = inherited_ids_to_nodes[id_] if self.find('./general/project_name') is not None: self.name = self.find('./general/project_name').text else: self.name = 'unnamed_project' os.environ['OPUSPROJECTNAME'] = self.name self.dirty = False
def init_run(self, create_baseyear_cache=True): ''' init run, get run_id & cache_directory. ''' ##avoid invoking start_run from cmd line - option_group = StartRunOptionGroup() option_group.parser.set_defaults(xml_configuration=self.xml_config, scenario_name=self.scenario) #run_id, cache_directory = start_run(option_group) options, args = option_group.parse() self.run_manager = RunManager( option_group.get_services_database_configuration(options)) resources = XMLConfiguration(self.xml_config).get_run_configuration( self.scenario) insert_auto_generated_cache_directory_if_needed(resources) cache_directory = resources['cache_directory'] self.run_manager.setup_new_run(cache_directory, resources) run_id, cache_directory = self.run_manager.run_id, self.run_manager.get_current_cache_directory( ) self.run_manager.add_row_to_history(run_id, resources, "done") if create_baseyear_cache: self.run_manager.create_baseyear_cache(resources) ## good for testing #run_id = 275 #cache_directory = '/home/lmwang/opus/data/paris_zone/runs/run_275.2012_05_26_00_20' assert run_id is not None assert cache_directory is not None return run_id, cache_directory
def test_run(self): # The paths work as follows: opus_matsim.__path__ is the path of the opus_matsim python module. So we can use that # as anchor ... config_location = os.path.join(opus_matsim.__path__[0], 'tests') print "location: ", config_location run_config = XMLConfiguration( os.path.join(config_location, "test_config.xml")).get_run_configuration("Test") run_config[ 'creating_baseyear_cache_configuration'].cache_directory_root = self.temp_dir run_config['creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy = \ os.path.join(opus_matsim.__path__[0], 'tests', 'testdata', 'base_year_data') # insert_auto_generated_cache_directory... does things I don't understand. Need to do the following to obtain consistent # behavior independent from the file root: run_config['cache_directory'] = None insert_auto_generated_cache_directory_if_needed(run_config) run_manager = RunManager(ServicesDatabaseConfiguration()) run_manager.setup_new_run( cache_directory=run_config['cache_directory'], configuration=run_config) run_manager.run_run(run_config, run_as_multiprocess=True) self.assert_(True) self.cleanup_test_run()
def setUp(self): print "entering setUp" logger.log_status("Testing UrbanSim export and import functionallity for MATSim...") # get root path to test cases self.path = test_path.__path__[0] logger.log_status("Set root path for MATSim config file to: %s" % self.path) if not os.path.exists(self.path): raise StandardError("Root path doesn't exist: %s" % self.path) # get path to MATSim source files and base_year_cache self.matsim_source, self.base_year_data_source = self.get_source_files() self.destination = tempfile.mkdtemp(prefix="opus_tmp") # self.destination = '/Users/thomas/Desktop/x' # for debugging # if not os.path.exists(self.destination): # for debugging # os.mkdir(self.destination) # for debugging # load UrbanSim config to run MATSim test urbansim_config_location = os.path.join( self.path, "configs", "urbansim_config" ) #'/Users/thomas/Development/workspace/urbansim_trunk/opus_matsim/tests/test_config.xml' logger.log_status("Loading UrbanSim config: %s" % urbansim_config_location) # self.run_config = XMLConfiguration( urbansim_config_location ).get_run_configuration("Test") urbansim_config_name = "urbansim_config_for_matsim_run_test.xml" self.run_config = XMLConfiguration( os.path.join(urbansim_config_location, urbansim_config_name) ).get_run_configuration("Test") # set destination for MATSim config file self.matsim_config_full = os.path.join(self.destination, "test_matsim_config.xml") print "leaving setUp"
def open(self, filename): ''' Load a project file from XML. @return: flag and message (tuple(boolean, String)) The flag is only True if the project loaded without problems. ''' # Always close the project before loading another one to avoid mixing # data if the load is only partly successful self.close() filename = str(filename) if not os.path.exists(filename): return ( False, "Tried to load project from file '%s', but that file does not exist" % filename) default_path = os.path.dirname(filename) filename = os.path.basename(filename) try: xml_config = XMLConfiguration(filename, default_path) self._read_xml_from_config(xml_config) self.filename = os.path.normpath( os.path.join(default_path, filename)) return (True, 'Project %s loaded OK' % filename) # Catch only the errors that XMLConfiguration is known to throw except (IOError, SyntaxError, ValueError, SyntaxError, XMLVersionException), ex: self.close() return (False, str(ex))
def main(): option_group = EstimationOptionGroup() parser = option_group.parser (options, args) = parser.parse_args() if options.model_name is None: raise StandardError, "Model name (argument -m) must be given." if (options.configuration_path is None) and (options.xml_configuration is None): raise StandardError, "Configuration path (argument -c) or XML configuration (argument -x) must be given." if (options.specification is None) and (options.xml_configuration is None): logger.log_warning( "No specification given (arguments -s or -x). Specification taken from the cache." ) if options.xml_configuration is not None: xconfig = XMLConfiguration(options.xml_configuration) else: xconfig = None if options.configuration_path is None: config = None else: config = get_config_from_opus_path(options.configuration_path) estimator = EstimationRunner(model=options.model_name, specification_module=options.specification, xml_configuration=xconfig, model_group=options.model_group, configuration=config, save_estimation_results=options.save_results) estimator.estimate() return estimator
def __init__(self, config_path=None, config_file_name=None, destination=None, testRun=True): ''' Constructor ''' print "Entering __init__" # load test urbansim config self.config_path = config_path if self.config_path == None: self.config_path = pyxb_test.__path__[0] self.config_name = config_file_name if self.config_name == None: self.config_name = 'test_urbansim_config.xml' self.matsim_config_location = destination if self.matsim_config_location == None: self.matsim_config_destination = os.path.join( os.environ['OPUS_HOME'], "opus_matsim", "matsim_config") if not self.test_path(self.matsim_config_destination): return False self.matsim_config_location = os.path.join( self.matsim_config_destination, "test_matsim_config.xml") self.config_location = os.path.join(self.config_path, self.config_name) print "Loding test config file: %s" % self.config_location self.config = XMLConfiguration( self.config_location).get_run_configuration("Test") # get travel model parameter from the opus dictionary self.travel_model_configuration = self.config[ 'travel_model_configuration'] # get years parameter self.years = self.config['years'] # gather all parameters for the MATSim config file # network parameter self.network_file = self.travel_model_configuration[ 'matsim_network_file'] self.network_file = os.path.join(os.environ['OPUS_HOME'], "opus_matsim", self.network_file) # controler parameter: self.first_iteration = self.travel_model_configuration[ 'first_iteration'] self.last_iteration = self.travel_model_configuration['last_iteration'] # planCalcScoreType self.activityType_0 = self.travel_model_configuration['activityType_0'] self.activityType_1 = self.travel_model_configuration['activityType_1'] # urbansim parameter self.year = self.years[0] self.samplingRate = self.travel_model_configuration['sampling_rate'] self.temp_directory = self.travel_model_configuration['temp_directory'] self.isTestRun = testRun self.opus_home = os.environ['OPUS_HOME'] self.firstRun = "FALSE" # create xml config file # return self.build_xml_config() print "Leaving __init__"
def test_simulation(self): eugene_dir = __import__('eugene').__path__[0] xml_config = XMLConfiguration( os.path.join(eugene_dir, 'configs', 'eugene_gridcell.xml')) option_group = StartRunOptionGroup() parser = option_group.parser # simulate 0 command line arguments by passing in [] (options, _) = parser.parse_args([]) run_manager = RunManager( option_group.get_services_database_configuration(options)) run_section = xml_config.get_run_configuration('Eugene_baseline') insert_auto_generated_cache_directory_if_needed(run_section) run_manager.setup_new_run( cache_directory=run_section['cache_directory'], configuration=run_section) run_manager.run_run(run_section)
def setUp(self): """ set up opus data path, a base_year_data cache directory needs to exists or be created through downloading and unzipping etc """ self.opus_home = os.environ["OPUS_HOME"] if os.environ.has_key('OPUS_DATA_PATH'): self.data_path = os.path.join(os.environ['OPUS_DATA_PATH'], 'psrc_parcel') else: self.data_path = os.path.join(self.opus_home, 'data', 'psrc_parcel') self.xml_config = XMLConfiguration(os.path.join(self.opus_home, 'project_configs', 'psrc_parcel_test.xml')) base_year_data_path = os.path.join(self.data_path, 'base_year_data') if not os.path.exists(base_year_data_path): os.makedirs(base_year_data_path)
def _get_run_config(temp_dir): config_path = os.path.join(temp_dir, 'testconfig.xml') f = open(config_path, 'w') f.write(config_template % temp_dir) f.close() run_configuration = XMLConfiguration(config_path).get_run_configuration( 'Eugene_baseline') return run_configuration
def prepare_run_manager(option_group=None): if option_group is None: option_group = StartRunOptionGroup() parser = option_group.parser options, args = option_group.parse() run_manager = RunManager( option_group.get_services_database_configuration(options)) if options.pickled_resource_file is not None: f = file(options.pickled_resource_file, 'r') try: config = pickle.load(f) finally: f.close() elif options.configuration_path is not None: opus_path = options.configuration_path try: config = get_config_from_opus_path(opus_path) except ImportError: # TODO: Once all fully-specified configurations are stored as classes, # get rid of this use. import_stmt = 'from %s import run_configuration as config' % opus_path exec(import_stmt) insert_auto_generated_cache_directory_if_needed(config) elif options.xml_configuration is not None: if options.scenario_name is None: parser.print_help() sys.exit(1) config = XMLConfiguration( options.xml_configuration).get_run_configuration( options.scenario_name) insert_auto_generated_cache_directory_if_needed(config) else: parser.print_help() sys.exit(1) if options.existing_cache_to_copy is not None: config[ 'creating_baseyear_cache_configuration'].cache_from_database = False config[ 'creating_baseyear_cache_configuration'].baseyear_cache = BaseyearCacheConfiguration( existing_cache_to_copy=options.existing_cache_to_copy, ) if options.years_to_cache is not None: config[ 'creating_baseyear_cache_configuration'].baseyear_cache.years_to_cache = eval( options.years_to_cache) if options.profile_filename is not None: config["profile_filename"] = options.profile_filename run_manager.setup_new_run(cache_directory=config['cache_directory'], configuration=config) return options, config, run_manager
def test_simulation(self): base_year_data_path = os.path.join(self.data_path, 'base_year_data') if not os.path.exists(base_year_data_path): os.makedirs(base_year_data_path) ftp_url = os.environ["FTP_URL"] file_name = os.path.split(ftp_url)[1] ftp_user = os.environ["FTP_USERNAME"] ftp_password = os.environ["FTP_PASSWORD"] #stdout, stderr = Popen("ls -la %s" % base_year_data_path, shell=True).communicate() #stdout, stderr = Popen("echo '%s'" % (base_year_data_path), stdout=PIPE).communicate() #print stdout try: Popen( """ cd %s; pwd; ls -la; echo wget --timestamping %s --ftp-user=%s --ftp-password=%s > /dev/null 2>&1; rm -rf 2008; unzip -o %s """ % (base_year_data_path, ftp_url, ftp_user, ftp_password, file_name), shell = True ).communicate() except: print "Error when downloading and unzipping file from %s." % ftp_url raise services_db = ServicesDatabaseConfiguration( database_name = 'services', database_configuration = 'services_database_server' ) run_manager = RunManager(services_db) run_as_multiprocess = True xml_config = XMLConfiguration(os.path.join(self.opus_home, 'project_configs', 'washtenaw_parcel.xml')) for scenario_name in ['washtenaw_baseline_test']: config = xml_config.get_run_configuration(scenario_name) insert_auto_generated_cache_directory_if_needed(config) # base_year = config['base_year'] # config['years_to_run'] = (base_year+1, base_year+2) run_manager.setup_new_run(cache_directory = config['cache_directory'], configuration = config) run_manager.run_run(config, run_as_multiprocess = run_as_multiprocess)
def run(self, config, executable): #--config=opus_matsim/sustain_city/configs/seattle_parcel.xml --executable=Seattle_baseline config = XMLConfiguration(config).get_run_configuration(executable) insert_auto_generated_cache_directory_if_needed(config) run_manager = RunManager(ServicesDatabaseConfiguration()) run_manager.setup_new_run(cache_directory = config['cache_directory'],configuration = config) run_manager.run_run(config, run_as_multiprocess = True )
def __init__(self, xml_config, scenario, calib_datasets, target_expression, target_file, subset=None, subset_patterns=None, skip_cache_cleanup=False, log_directory=None): """ - xml_config: xml configuration file, for ex '/home/atschirhar/opus/project_configs/paris_zone.xml' - scenario: name of scenario to run for calibration, where models_to_run and simulation years are specified - calib_datasets: dictionary specifying dataset names and attributes to be calibrated, e.g. {'establishment_location_choice_model_coefficients': 'estimate'} - target_expression: opus expression computing values from prediction to be compared with targets - target_file: name of csv file providing targets - subset: dictionary specifying the dataset to be calibrated, {'etablishment_location_choice_model_coefficients': ['coefficient_name', ['paris_celcm, 'biotech_celcm']]} subset and subset_patterns can not be both specified for the same dataset - subset_patterns: dictionary specifying the dataset to be calibrated through a regular expression (re) pattern {'etablishment_location_choice_model_coefficients': ['coefficient_name', '*_celcm']} subset and subset_patterns can not be both specified for the same dataset """ self.target_expression = target_expression self.target = self.read_target(target_file) self.run_manager = None self.xml_config = xml_config self.scenario = scenario self.skip_cache_cleanup = skip_cache_cleanup self.run_id, self.cache_directory = self.init_run() self.run_ids = [ self.run_id ] #allow starting of multiple runs for parallel optimization self.log_directory = log_directory if self.log_directory is None: self.log_directory = self.cache_directory #legacy log_file = os.path.join(self.log_directory, "calibration.log") logger.enable_file_logging(log_file) dict_config = XMLConfiguration(self.xml_config).get_run_configuration( self.scenario) ## get parameters from config self.base_year = dict_config['base_year'] self.start_year, self.end_year = dict_config['years'] self.project_name = dict_config['project_name'] self.package_order = dict_config[ 'dataset_pool_configuration'].package_order
def setUp(self): """ set up opus data path, a base_year_data cache directory needs to exists or be created through downloading and unzipping etc """ self.opus_home = paths.OPUS_HOME self.data_path = paths.get_opus_data_path_path('sanfrancisco') self.xml_config = XMLConfiguration(os.path.join(self.opus_home, 'project_configs', 'sanfrancisco.xml')) base_year_data_path = os.path.join(self.data_path, 'base_year_data') if not os.path.exists(base_year_data_path): os.makedirs(base_year_data_path)
def setUp(self): """ set up opus data path, a base_year_data cache directory needs to exists or be created through downloading and unzipping etc """ self.opus_home = paths.OPUS_HOME self.data_path = paths.get_opus_data_path_path("psrc_parcel") self.xml_config = XMLConfiguration(os.path.join(self.opus_home, "project_configs", "psrc_parcel_test.xml")) base_year_data_path = os.path.join(self.data_path, "base_year_data") if not os.path.exists(base_year_data_path): os.makedirs(base_year_data_path)
class StartRunOptionGroup(object): """ Helper class to start model from an xml config file. """ logger.start_block("Starting UrbanSim") # get program arguments from the command line program_arguments = sys.argv[1:] # default parameters are: # --config=opus_matsim/sustain_city/configs/seattle_parcel_prescheduled_events.xml # --executable=Seattle_baseline parser = optparse.OptionParser() parser.add_option("-c", "--config", dest="config_file_name", action="store", type="string", help="Name of file containing urbansim config") parser.add_option("-e", "--executable", dest="scenario_executable", action="store", type="string", help="Model to execute") (options, args) = parser.parse_args() if options.config_file_name == None: logger.log_error("Missing path to the urbansim config file") if options.scenario_executable == None: logger.log_error("Missing name of executable scenario") config = XMLConfiguration(options.config_file_name).get_run_configuration( options.scenario_executable) insert_auto_generated_cache_directory_if_needed(config) run_manager = RunManager(ServicesDatabaseConfiguration()) run_manager.setup_new_run(cache_directory=config['cache_directory'], configuration=config) #try: #tnicolai # import pydevd # pydevd.settrace() #except: pass run_manager.run_run(config, run_as_multiprocess=True)
def __init__(self): print "Entering setup" logger.log_status('Running UrbanSim to test the impact of travel costs (provided from an dummy travel model)') # get sensitivity test path self.test_dir_path = test_dir.__path__[0] self.config_file = os.path.join( self.test_dir_path, "configs", "seattle_parcel_travel_cost_test.xml") print 'Loding UrbanSim config file: %s' % self.config_file # get seattle_parcel configuration self.config = XMLConfiguration( self.config_file ).get_run_configuration( "Seattle_baseline" ) insert_auto_generated_cache_directory_if_needed(self.config) print "Leaving setup"
def setUp(self): """ set up opus data path, a base_year_data cache directory needs to exists or be created through downloading and unzipping etc """ self.opus_home = os.environ["OPUS_HOME"] if os.environ.has_key('OPUS_DATA_PATH'): self.data_path = os.path.join(os.environ['OPUS_DATA_PATH'], 'san_antonio_zone') else: self.data_path = os.path.join(self.opus_home, 'data', 'san_antonio_zone') self.xml_config = XMLConfiguration(os.path.join(self.opus_home, 'project_configs', 'san_antonio_zone.xml')) base_year_data_path = os.path.join(self.data_path, 'base_year_data') if not os.path.exists(base_year_data_path): os.makedirs(base_year_data_path)
class StartRunOptionGroup(object): """ Helper class to start model from an xml config file. """ config = XMLConfiguration("opus_matsim/configs/seattle_parcel.xml" ).get_run_configuration("Seattle_baseline") # config = XMLConfiguration("opus_matsim/configs/psrc_parcel.xml").get_run_configuration("PSRC_baseline") insert_auto_generated_cache_directory_if_needed(config) run_manager = RunManager(ServicesDatabaseConfiguration()) run_manager.setup_new_run(cache_directory=config['cache_directory'], configuration=config) # run_manager.create_baseyear_cache(config) run_manager.run_run(config, run_as_multiprocess=True)
def test_estimation(self): # open the configuration for seattle_parcel.xml seattle_parcel_dir = __import__('seattle_parcel').__path__[0] xml_config = XMLConfiguration( os.path.join(seattle_parcel_dir, 'configs', 'seattle_parcel.xml')) for model_name in [ 'real_estate_price_model', 'household_location_choice_model' ]: er = EstimationRunner(model=model_name, xml_configuration=xml_config, configuration=None) er.estimate() # test run with group members er = EstimationRunner(model='employment_location_choice_model', xml_configuration=xml_config, model_group='home_based', configuration=None) er.estimate()
def __init__(self, output_dir=None, year=None): ''' Constructor ''' # get working path as an anchor e.g. to determine the config file location. self.working_path = test_path.__path__[0] print "Working path: %s" % self.working_path # get config file location self.config_file = os.path.join( self.working_path, 'configs', 'seattle_parcel_travel_cost_test.xml') # get seattle_parcel configuration config = XMLConfiguration( self.config_file ).get_run_configuration( "Seattle_baseline" ) self.input_storage = None # get first simulation year self.year = year if self.year == None: self.year = config['base_year'] base_year_data_path = os.path.join( os.environ['OPUS_DATA_PATH'], 'seattle_parcel', 'base_year_data') attribute_cache = AttributeCache(cache_directory=base_year_data_path) self.input_storage = attribute_cache.get_flt_storage_for_year(self.year) else: attribute_cache = AttributeCache().get_flt_storage_for_year(self.year) self.input_storage = attribute_cache # get output dir path output_directory = output_dir if output_directory == None: # set deafult output_directory = os.path.join( os.environ['OPUS_HOME'], 'opus_matsim', 'tmp') if not os.path.exists( output_directory ): try: os.mkdir( output_directory ) except: pass # init self.csv_data_path = output_directory # os.path.join(output_directory, 'travel_data_dir')
dest="stdout", default=False, action="store_true", help='print results into stdout') parser.add_option("-l", "--latex", dest="latex", default=None, help="latex output file") (options, args) = parser.parse_args() if options.xml_configuration == None: raise "Requires an xml configuration argument." chart = DependencyChart(XMLConfiguration(options.xml_configuration), model=options.model, model_group=options.model_group) #print chart.model_table(options.model) #temp = chart.query.vars_tree(chart.query.var_list) #print pretty_tree(chart.query.all_models_tree()) #auto = AutogenVariableFactory("(urbansim_parcel.parcel.building_sqft/(parcel.parcel_sqft).astype(float32)).astype(float32)") #auto._analyze_tree(auto._expr_parsetree) #auto._analyze_dataset_names() #print(auto._generate_compute_method()) if options.stdout: if options.model != None: chart.print_model_dependencies() else:
action="store", help="Name of the scenario database server configuration in database_server_configurations.xml.") (options, args) = parser.parse_args() if options.configuration_path is not None: opus_path = options.configuration_path try: config = get_config_from_opus_path(opus_path) except ImportError: import_stmt = 'from %s import run_configuration as config' % opus_path exec(import_stmt) elif options.xml_configuration is not None: if options.scenario_name is None: parser.print_help() sys.exit(1) config = XMLConfiguration(options.xml_configuration).get_run_configuration(options.scenario_name) else: parser.print_help() sys.exit(1) if options.cache_directory is not None: config['cache_directory'] = options.cache_directory if options.database_name is not None or options.database_configuration is not None: if not config.has_key('scenario_database_configuration'): config['scenario_database_configuration'] = ScenarioDatabaseConfiguration(database_name = options.database_name, database_configuration = options.database_configuration ) else: if options.database_configuration is not None: config['scenario_database_configuration'] = ScenarioDatabaseConfiguration(database_configuration = options.database_configuration)
parser.add_option("-x", "--xml-configuration", dest="xml_configuration", default = None, action="store", help="Full path to an XML configuration file (must also provide a scenario name using -s). Either -x or -r must be given.") parser.add_option("-s", "--scenario_name", dest="scenario_name", default=None, help="Name of the scenario. Must be given if option -x is used.") parser.add_option("-d", "--directory", dest="cache_directory", default = None, action="store", help="Cache directory with urbansim output.") (options, args) = parser.parse_args() if options.year is None: raise StandardError, "Year (argument -y) must be given." if (options.scenario_name is None) and (options.xml_configuration is not None): raise StandardError, "No scenario given (argument -s). Must be specified if option -x is used." r = None xconfig = None if options.resources_file_name is not None: r = get_resources_from_file(options.resources_file_name) resources = Resources(get_resources_from_file(options.resources_file_name)) elif options.xml_configuration is not None: xconfig = XMLConfiguration(options.xml_configuration) resources = xconfig.get_run_configuration(options.scenario_name) else: raise StandardError, "Either option -r or -x must be used." files = GetCacheDataIntoDaysim(resources).run(options.year, cache_directory=options.cache_directory) if options.output_directory is not None: for file in files: copy(file, options.output_directory) logger.log_status('Files copied into %s' % options.output_directory) # For a test run, use options # -x opus_daysim/configs/sample_daysim_configuration.xml -s daysim_scenario -d opus_core/data/test_cache -y 1980
elif options.configuration_path is not None: opus_path = options.configuration_path try: config = get_config_from_opus_path(opus_path) except ImportError: # TODO: Once all fully-specified configurations are stored as classes, # get rid of this use. import_stmt = 'from %s import run_configuration as config' % opus_path exec(import_stmt) insert_auto_generated_cache_directory_if_needed(config) elif options.xml_configuration is not None: if options.scenario_name is None: parser.print_help() sys.exit(1) config = XMLConfiguration( options.xml_configuration).get_run_configuration( options.scenario_name) insert_auto_generated_cache_directory_if_needed(config) else: parser.print_help() sys.exit(1) if options.existing_cache_to_copy is not None: config[ 'creating_baseyear_cache_configuration'].cache_from_database = False config[ 'creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy = options.existing_cache_to_copy if options.years_to_cache is not None: config[ 'creating_baseyear_cache_configuration'].baseyear_cache.years_to_cache = eval( options.years_to_cache)
def copy_to_parent(self, node): ''' Copies a local node to the parent configuration without deleting it. @node (Element) to copy to parent ''' # Helper routines: # Never copy description and parent nodes to parent def delete_immutable(): id_strings = self.IMMUTABLE_NODE_IDS for id_string in id_strings: for n in self.find_all_by_id_string(id_string, clone): n.getparent().remove(n) # Find deepest parent node of to-be-inserted node that is also in the parent config def get_insert_node(merge_node, nodes): nodes.append(merge_node) ins_node = self.find_by_id_string(node_identity_string(merge_node), parent_root) if ins_node is not None: return ins_node merge_node = merge_node.getparent() return get_insert_node(merge_node, nodes) # Remove all children for all nodes in the nodes list def strip_children(nodes): for n in nodes[:-1]: for subelement in n.getparent().getchildren(): if subelement is not n: n.getparent().remove(subelement) # Remove "inherited" attribute from all nodes below tree_node that were # introduced by the immediate parent. Remove all inherited nodes # that were introduced by a grandparent -- copying them to the parent # leads to incorrect results. def clear_inherited_attribute_or_delete_inherited_nodes(tree_node): nodes_to_delete = [] for node in tree_node.iterchildren(): if node.get('inherited') is not None: if node.get('inherited') == parent_name: del node.attrib['inherited'] clear_inherited_attribute_or_delete_inherited_nodes(node) else: nodes_to_delete.append(node) else: clear_inherited_attribute_or_delete_inherited_nodes(node) for node in nodes_to_delete: tree_node.remove(node) #work on clone_node id_string = node_identity_string(node) clone = copy.deepcopy(self.root_node()) node = self.find_by_id_string(id_string, clone) #get parent project parent_file = self.get_last_writable_parent_file() parent_project = OpusProject() parent_project.open(parent_file) parent_name = parent_project.xml_config.name parent_root = parent_project.xml_config.tree.getroot() delete_immutable() parents_to_insert = [] if node is not clone: insert_node = get_insert_node(node, parents_to_insert) node = parents_to_insert[-1] strip_children(parents_to_insert) clear_inherited_attribute_or_delete_inherited_nodes(node) XMLConfiguration._merge_nodes(insert_node, node) insert_parent = insert_node.getparent() insert_parent.replace(insert_node, node) # using parent_project.save() adds unnecessary attributes for some reason. parent_project.xml_config.save_as(parent_file)
elif options.configuration_path is not None: opus_path = options.configuration_path try: config = get_config_from_opus_path(opus_path) except ImportError: # TODO: Once all fully-specified configurations are stored as classes, # get rid of this use. import_stmt = 'from %s import run_configuration as config' % opus_path exec(import_stmt) insert_auto_generated_cache_directory_if_needed(config) elif options.xml_configuration is not None: if options.scenario_name is None: parser.print_help() sys.exit(1) config = XMLConfiguration(options.xml_configuration).get_run_configuration(options.scenario_name) insert_auto_generated_cache_directory_if_needed(config) else: parser.print_help() sys.exit(1) if options.existing_cache_to_copy is not None: config['creating_baseyear_cache_configuration'].cache_from_database = False config['creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy = options.existing_cache_to_copy if options.years_to_cache is not None: config['creating_baseyear_cache_configuration'].baseyear_cache.years_to_cache = eval(options.years_to_cache) number_of_runs = config.get("number_of_runs", 1) number_of_runs_in_parallel = min(config.get("parallel_runs", 1), number_of_runs) # generate seeds for multiple runs root_seed = config.get("seed", None)
config = {} if options.configuration_path is not None: opus_path = options.configuration_path try: config = get_config_from_opus_path(opus_path) except ImportError: # TODO: Once all fully-specified configurations are stored as classes, # get rid of this use. import_stmt = 'from %s import run_configuration as config' % opus_path exec(import_stmt) elif options.xml_configuration is not None: if options.scenario_name is None: parser.print_help() sys.exit(1) config = XMLConfiguration(options.xml_configuration).get_run_configuration(options.scenario_name) from_database_configuration = config.get('scenario_database_configuration', ScenarioDatabaseConfiguration(database_name = options.scenario_database_name, database_configuration=options.scenario_database_configuration, )) to_database_name = options.flattened_database_name or (from_database_configuration.database_name + '_flattened') to_database_configuration = ScenarioDatabaseConfiguration( protocol = from_database_configuration.protocol, host_name = from_database_configuration.host_name, user_name = from_database_configuration.user_name, password = from_database_configuration.password, database_name = to_database_name) if config.get('creating_baseyear_cache_configuration', None): tables_to_copy = config['creating_baseyear_cache_configuration'].tables_to_cache else:
class TestSimulation(opus_unittest.OpusIntegrationTestCase): """ this integration test runs san_antonio_baseline_test in project_configs/san_antonio_zone.xml """ def setUp(self): """ set up opus data path, a base_year_data cache directory needs to exists or be created through downloading and unzipping etc """ self.opus_home = os.environ["OPUS_HOME"] if os.environ.has_key('OPUS_DATA_PATH'): self.data_path = os.path.join(os.environ['OPUS_DATA_PATH'], 'san_antonio_zone') else: self.data_path = os.path.join(self.opus_home, 'data', 'san_antonio_zone') self.xml_config = XMLConfiguration(os.path.join(self.opus_home, 'project_configs', 'san_antonio_zone.xml')) base_year_data_path = os.path.join(self.data_path, 'base_year_data') if not os.path.exists(base_year_data_path): os.makedirs(base_year_data_path) # ftp_url = os.environ["FTP_URL"] # file_name = os.path.split(ftp_url)[1] # ftp_user = os.environ["FTP_USERNAME"] # ftp_password = os.environ["FTP_PASSWORD"] # # try: # Popen( """ # cd %s; # pwd; # ls -la; # echo wget --timestamping %s --ftp-user=%s --ftp-password=%s > /dev/null 2>&1; # rm -rf 2008; # unzip -o %s # """ % (base_year_data_path, ftp_url, ftp_user, ftp_password, file_name), # shell = True # ).communicate() # except: # print "Error when downloading and unzipping file from %s." % ftp_url # raise def tearDown(self): """ delete [project_name]/runs directory to free up disk space """ runs_path = os.path.join(self.data_path, 'runs') #if os.path.exists(runs_path): # Popen( "rm -rf %s" % runs_path, shell=True) def test_estimation(self): for model_name in ['real_estate_price_model', 'household_location_choice_model', ('employment_location_choice_model', 'home_based'), ('employment_location_choice_model', 'non_home_based'), 'residential_development_project_location_choice_model', 'non_residential_development_project_location_choice_model', ]: if type(model_name)==tuple: model_name, group_member = model_name else: group_member = None estimator = EstimationRunner(model=model_name, model_group=group_member, xml_configuration=self.xml_config, configuration = None ) estimator.estimate() def test_simulation(self): services_db = ServicesDatabaseConfiguration( database_name = 'services', database_configuration = 'services_database_server' ) run_manager = RunManager(services_db) run_as_multiprocess = True for scenario_name in ['san_antonio_baseline_test']: config = self.xml_config.get_run_configuration(scenario_name) insert_auto_generated_cache_directory_if_needed(config) run_manager.setup_new_run(cache_directory = config['cache_directory'], configuration = config) run_manager.run_run(config, run_as_multiprocess = run_as_multiprocess)
class TestSimulation(opus_unittest.OpusIntegrationTestCase): """ this integration test runs san_antonio_baseline_test in project_configs/psrc_parcel_test.xml """ def setUp(self): """ set up opus data path, a base_year_data cache directory needs to exists or be created through downloading and unzipping etc """ self.opus_home = paths.OPUS_HOME self.data_path = paths.get_opus_data_path_path("psrc_parcel") self.xml_config = XMLConfiguration(os.path.join(self.opus_home, "project_configs", "psrc_parcel_test.xml")) base_year_data_path = os.path.join(self.data_path, "base_year_data") if not os.path.exists(base_year_data_path): os.makedirs(base_year_data_path) # ftp_url = os.environ["FTP_URL"] # file_name = os.path.split(ftp_url)[1] # ftp_user = os.environ["FTP_USERNAME"] # ftp_password = os.environ["FTP_PASSWORD"] # # try: # Popen( """ # cd %s; # pwd; # ls -la; # echo wget --timestamping %s --ftp-user=%s --ftp-password=%s > /dev/null 2>&1; # rm -rf 2008; # unzip -o %s # """ % (base_year_data_path, ftp_url, ftp_user, ftp_password, file_name), # shell = True # ).communicate() # except: # print "Error when downloading and unzipping file from %s." % ftp_url # raise def tearDown(self): """ delete [project_name]/runs directory to free up disk space """ runs_path = os.path.join(self.data_path, "runs") # if os.path.exists(runs_path): # Popen( "rm -rf %s" % runs_path, shell=True) def test_estimation(self): for model_name in [ "residential_building_type_choice_model", "tenure_choice_model", "work_at_home_choice_model", "workplace_choice_model_for_resident", "auto_ownership_choice_model", "hbw_mode_choice_model", "real_estate_price_model", "household_location_choice_model", "shopping_destination_choice_model", ("employment_location_choice_model", "home_based"), ("employment_location_choice_model", "non_home_based"), ]: if type(model_name) == tuple: model_name, group_member = model_name else: group_member = None estimator = EstimationRunner( model=model_name, model_group=group_member, xml_configuration=self.xml_config, configuration=None ) estimator.estimate() def test_simulation(self): services_db = ServicesDatabaseConfiguration( database_name="services", database_configuration="services_database_server" ) run_manager = RunManager(services_db) run_as_multiprocess = True for scenario_name in ["psrc_baseline_test"]: config = self.xml_config.get_run_configuration(scenario_name) insert_auto_generated_cache_directory_if_needed(config) run_manager.setup_new_run(cache_directory=config["cache_directory"], configuration=config) run_manager.run_run(config, run_as_multiprocess=run_as_multiprocess)
(options, args) = parser.parse_args() if options.configuration_path is not None: opus_path = options.configuration_path try: config = get_config_from_opus_path(opus_path) except ImportError: import_stmt = 'from %s import run_configuration as config' % opus_path exec(import_stmt) elif options.xml_configuration is not None: if options.scenario_name is None: parser.print_help() sys.exit(1) config = XMLConfiguration( options.xml_configuration).get_run_configuration( options.scenario_name) else: parser.print_help() sys.exit(1) if options.cache_directory is not None: config['cache_directory'] = options.cache_directory if options.database_name is not None or options.database_configuration is not None: if not config.has_key('scenario_database_configuration'): config[ 'scenario_database_configuration'] = ScenarioDatabaseConfiguration( database_name=options.database_name, database_configuration=options.database_configuration) else:
class OpusProject(object): ''' Methods and attributes for handling an OPUS project file. Inheritance is done by letting a XmlConfiguration object handle loading and saving. Each project has two root nodes, one for the full tree and one for the tree of all inherited values. Nodes that exists in the full tree AND in the inherited tree, but with different values are called "shadowing nodes", since they shadow the inherited value with a local variation. ''' def __init__(self): self.name = '' # Project name self.filename = '' # Filename of loaded project self.xml_config = None # XMLController object self._root_node = None # Full project tree root self._inherited_root = None # Root node of the tree with inherited nodes (original values) self._shadowing_nodes = {} # Mapping of nodes in local tree to nodes in inherited tree self._dirty = False # Dirty flag def __is_dirty(self): return self._dirty def __set_dirty(self, dirty): self._dirty = dirty update_mainwindow_savestate() dirty = property(__is_dirty, __set_dirty) def _read_xml_from_config(self, xml_config): ''' Extract XML information from the XML Configuration object ''' self.xml_config = xml_config self._root_node = xml_config.full_tree.getroot() if xml_config.inherited_tree is None: self._inherited_root = XMLConfiguration().full_tree.getroot() else: self._inherited_root = xml_config.inherited_tree self._init_shadowing_nodes() self._set_project_name() def _init_shadowing_nodes(self): self._shadowing_nodes = {} self._add_shadowing_nodes(self._root_node, '') def _add_shadowing_nodes(self, root_node, root_node_id): # map id's to nodes for the inherited and the local nodes inherited_ids_to_nodes = dict((node_identity_string(n), n) for n in self._inherited_root.getiterator()) local_ids_to_nodes = dict((root_node_id + node_identity_string(n), n) for n in root_node.getiterator() if not n.get('inherited')) # join the local and inherited nodes on id-match for id_, node in local_ids_to_nodes.items(): if id_ in inherited_ids_to_nodes: shadowing_node = inherited_ids_to_nodes[id_] assert node.tag == shadowing_node.tag self._shadowing_nodes[node] = shadowing_node def _set_project_name(self): if self.find('./general/project_name') is not None: self.name = self.find('./general/project_name').text else: self.name = 'unnamed_project' os.environ['OPUSPROJECTNAME'] = self.name self.dirty = False def load_minimal_project(self): ''' Setup the project as if it was loaded with an absolute minimal project config file ''' minimal_config = XMLConfiguration() self._read_xml_from_config(minimal_config) def open(self, filename): ''' Load a project file from XML. @return: flag and message (tuple(boolean, String)) The flag is only True if the project loaded without problems. ''' # Always close the project before loading another one to avoid mixing # data if the load is only partly successful self.close() filename = str(filename) if not os.path.exists(filename): return (False, "Tried to load project from file '%s', but that file does not exist" %filename) default_path = os.path.dirname(filename) filename = os.path.basename(filename) try: xml_config = XMLConfiguration(filename, default_path) self._read_xml_from_config(xml_config) self.filename = os.path.normpath(os.path.join(default_path, filename)) return (True, 'Project %s loaded OK' % filename) # Catch only the errors that XMLConfiguration is known to throw except (IOError, SyntaxError, ValueError, SyntaxError, XMLVersionException), ex: self.close() return (False, str(ex))
def __init__(self, xml = '<opus_project />'): OpusProject.__init__(self) xml_config = XMLConfiguration() xml_config.update(xml) self._read_xml_from_config(xml_config)
help="model for which you want to chart dependencies") parser.add_option("--group", dest="model_group", default = None, action="store", help="name of the model group") parser.add_option("-o", "--output", dest="output", default=None, help="output image") parser.add_option("-p", dest="stdout",default=False, action="store_true", help='print results into stdout') parser.add_option("-l", "--latex", dest="latex", default=None, help="latex output file") (options, args) = parser.parse_args() if options.xml_configuration == None: raise "Requires an xml configuration argument." chart = DependencyChart(XMLConfiguration(options.xml_configuration), model=options.model, model_group=options.model_group) #print chart.model_table(options.model) #temp = chart.query.vars_tree(chart.query.var_list) #print pretty_tree(chart.query.all_models_tree()) #auto = AutogenVariableFactory("(urbansim_parcel.parcel.building_sqft/(parcel.parcel_sqft).astype(float32)).astype(float32)") #auto._analyze_tree(auto._expr_parsetree) #auto._analyze_dataset_names() #print(auto._generate_compute_method()) if options.stdout: if options.model != None: chart.print_model_dependencies() else: chart.print_dependencies(options.variable) elif options.latex != None:
config = {} if options.configuration_path is not None: opus_path = options.configuration_path try: config = get_config_from_opus_path(opus_path) except ImportError: # TODO: Once all fully-specified configurations are stored as classes, # get rid of this use. import_stmt = 'from %s import run_configuration as config' % opus_path exec(import_stmt) elif options.xml_configuration is not None: if options.scenario_name is None: parser.print_help() sys.exit(1) config = XMLConfiguration( options.xml_configuration).get_run_configuration( options.scenario_name) from_database_configuration = config.get( 'scenario_database_configuration', ScenarioDatabaseConfiguration( database_name=options.scenario_database_name, database_configuration=options.scenario_database_configuration, )) to_database_name = options.flattened_database_name or ( from_database_configuration.database_name + '_flattened') to_database_configuration = ScenarioDatabaseConfiguration( protocol=from_database_configuration.protocol, host_name=from_database_configuration.host_name, user_name=from_database_configuration.user_name, password=from_database_configuration.password,
option_group = ModelExplorerOptionGroup() parser = option_group.parser (options, args) = parser.parse_args() if options.year is None: raise StandardError, "Year (argument -y) must be given." if options.cache_directory is None: raise StandardError, "Cache directory (argument -d) must be given." if (options.configuration_path is None) and (options.xml_configuration is None): raise StandardError, "Configuration path (argument -c) or XML configuration (argument -x) must be given." if (options.scenario_name is None) and (options.xml_configuration is not None): raise StandardError, "No scenario given (argument -s). Must be specified if option -x is used." if options.xml_configuration is not None: xconfig = XMLConfiguration(options.xml_configuration) else: xconfig = None if options.configuration_path is None: config = None else: config = get_config_from_opus_path(options.configuration_path) if options.cache_directory == 'BASE': cache_directory = None else: cache_directory = options.cache_directory explorer = ModelExplorer(model=options.model_name, year=int(options.year), scenario_name=options.scenario_name, model_group=options.model_group,
seed = 100 nchunks = 1 ##TODO if __name__ == '__main__': try: import wingdbstub except: pass from psrc_parcel.household.aliases import aliases aliases += ["customized_est_filter= (household.household_id > 6000000) * " + \ "numpy.logical_or(household.tenure_id==1, household.tenure_id==3) * " + \ #numpy.setmember1d(household.tenure, (1,3)) * \ ## buggy "(( (psrc_parcel.household.building_type_id==4) + (psrc_parcel.household.building_type_id==11) + (psrc_parcel.household.building_type_id==12) + (psrc_parcel.household.building_type_id==19)) >= 1) * " + \ #numpy.setmember1d(psrc_parcel.household.building_type_id, (4, 11, 12, 19)) * \ ##ideally the above line can be replaced with this, but this is buggy "(household.aggregate((person.worker1==1)*(urbansim_parcel.person.is_non_home_based_worker_with_job)))", "customized_run_filter= household.aggregate(urbansim_parcel.person.is_non_home_based_worker)>=1" ] xmlconfig = XMLConfiguration(options.xml_configuration) ## training data (start_estimation) training_data = [] for h, hierarchy in enumerate(options.meta_models): model_data = [] for i, model_name in enumerate(hierarchy): logger.start_block('%s' % model_name) estimator = EstimationRunner(model=model_name, specification_module=None, xml_configuration=xmlconfig, model_group=None, configuration=None, save_estimation_results=True) #estimator = EstimationRunner(model=model_name, #xml_configuration=xmlconfig, #save_estimation_results=False)
class IOCacheMATSimTestRun(opus_unittest.OpusTestCase): ''' Tests the UrbanSim export and import functionallity for the travel model. ''' def setUp(self): print "entering setUp" logger.log_status('Testing UrbanSim export and import functionallity for MATSim...') # get root path to test cases self.path = test_path.__path__[0] logger.log_status('Set root path for MATSim config file to: %s' % self.path) if not os.path.exists(self.path): raise StandardError("Root path doesn't exist: %s" % self.path) # get path to MATSim source files and base_year_cache self.matsim_source, self.base_year_data_source = self.get_source_files() self.destination = tempfile.mkdtemp(prefix='opus_tmp') #self.destination = '/Users/thomas/Desktop/x' # for debugging #if not os.path.exists(self.destination): # for debugging # os.mkdir(self.destination) # for debugging # load UrbanSim config to run MATSim test urbansim_config_location = os.path.join( self.path, 'configs', 'urbansim_config') #'/Users/thomas/Development/workspace/urbansim_trunk/opus_matsim/tests/test_config.xml' logger.log_status('Loading UrbanSim config: %s' % urbansim_config_location) #self.run_config = XMLConfiguration( urbansim_config_location ).get_run_configuration("Test") urbansim_config_name = "urbansim_config_for_matsim_run_test.xml" self.run_config = XMLConfiguration( os.path.join(urbansim_config_location, urbansim_config_name)).get_run_configuration("Test") # set destination for MATSim config file self.matsim_config_full = os.path.join( self.destination, "test_matsim_config.xml" ) print "leaving setUp" def tearDown(self): print "entering tearDown" logger.log_status('Removing extracted MATSim files...') if os.path.exists(self.destination): rmtree(self.destination) logger.log_status('... cleaning up finished.') print "leaving tearDown" def testName(self): print "entering test_run" logger.log_status('Preparing MATsim test run ...') # unzip MATSim files matsim_zip = ExtractZipFile(self.matsim_source, self.destination) matsim_zip.extract() matsim_extracted_files = os.path.join(self.destination, 'MATSimTestClasses') # location of unziped MATSim files # unzip base_year_cache base_year_data_zip = ExtractZipFile(self.base_year_data_source, self.destination) base_year_data_zip.extract() base_year_data_extracted_files = os.path.join(self.destination, 'base_year_data') # location of unziped base_year_cache # updating location of base_year_data self.run_config['creating_baseyear_cache_configuration'].cache_directory_root = self.destination self.run_config['creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy = base_year_data_extracted_files self.run_config['cache_directory'] = base_year_data_extracted_files self.run_config.add('matsim_files', matsim_extracted_files) self.run_config.add('matsim_config', self.matsim_config_full) self.run_config.add('root', self.destination) insert_auto_generated_cache_directory_if_needed(self.run_config) run_manager = RunManager(ServicesDatabaseConfiguration()) run_manager.setup_new_run(cache_directory = self.run_config['cache_directory'], configuration = self.run_config) logger.log_status('Strating UrbanSim run ... ') run_manager.run_run(self.run_config, run_as_multiprocess = True ) # after the UrbanSim run the travel data sets schould be equal # self.assertTrue( self.compare_travel_data_sets() ) logger.log_status('... UrbanSim run finished.') print "leaving test_run" def get_source_files(self): ''' Returns the path to the MATSim and base_year_data source files ''' matsim_source_files = os.path.join( self.path, 'data', 'MATSimTestClasses.zip') if not os.path.exists(matsim_source_files): raise StandardError("MATSim source file not found: %s" % matsim_source_files) logger.log_status('Referering to MATSim source file: %s' % matsim_source_files) base_year_data_source_files = os.path.join( self.path, 'data', 'base_year_data.zip') if not os.path.exists(base_year_data_source_files): raise StandardError("Base year data zip file not found: %s" % base_year_data_source_files) logger.log_status('Referering to base year cache file: %s' % base_year_data_source_files) return matsim_source_files, base_year_data_source_files def compare_travel_data_sets(self): # get copied travel data csv copied_travel_data_location = os.path.join( self.destination, 'opus_matsim', 'tmp') if not os.path.exists(copied_travel_data_location): raise StandardError('Travel data not found: %s' % copied_travel_data_location) logger.log_status('Get copied travel data: %s' % copied_travel_data_location) # convert travel data csv into travel data set matrix in_storage = csv_storage(storage_location = copied_travel_data_location) table_name = "travel_data" travel_data_attribute = 'single_vehicle_to_work_travel_cost' travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name ) travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999) # get exsisting travel data set and convert it also into travel data set matrix year = self.run_config['base_year']+2 attribute_cache = AttributeCache(cache_directory=self.run_config['cache_directory']) cache_storage = attribute_cache.get_flt_storage_for_year(year) existing_travel_data_set = TravelDataDataset( in_storage=cache_storage, in_table_name=table_name ) existing_travel_data_attribute_mat = existing_travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999) from numpy import savetxt # for debugging savetxt( os.path.join(self.destination, 'origin_travel_data.txt'), travel_data_attribute_mat , fmt="%f") savetxt( os.path.join(self.destination, 'existing_travel_data') , existing_travel_data_attribute_mat, fmt="%f") # compare both data set matices compare = travel_data_attribute_mat == existing_travel_data_attribute_mat # return result return compare.all()
def load_minimal_project(self): ''' Setup the project as if it was loaded with an absolute minimal project config file ''' minimal_config = XMLConfiguration() self._read_xml_from_config(minimal_config)
class IOCacheMATSimTestRun(opus_unittest.OpusTestCase): """ Tests the UrbanSim export and import functionallity for the travel model. """ def setUp(self): print "entering setUp" logger.log_status("Testing UrbanSim export and import functionallity for MATSim...") # get root path to test cases self.path = test_path.__path__[0] logger.log_status("Set root path for MATSim config file to: %s" % self.path) if not os.path.exists(self.path): raise StandardError("Root path doesn't exist: %s" % self.path) # get path to MATSim source files and base_year_cache self.matsim_source, self.base_year_data_source = self.get_source_files() self.destination = tempfile.mkdtemp(prefix="opus_tmp") # self.destination = '/Users/thomas/Desktop/x' # for debugging # if not os.path.exists(self.destination): # for debugging # os.mkdir(self.destination) # for debugging # load UrbanSim config to run MATSim test urbansim_config_location = os.path.join( self.path, "configs", "urbansim_config" ) #'/Users/thomas/Development/workspace/urbansim_trunk/opus_matsim/tests/test_config.xml' logger.log_status("Loading UrbanSim config: %s" % urbansim_config_location) # self.run_config = XMLConfiguration( urbansim_config_location ).get_run_configuration("Test") urbansim_config_name = "urbansim_config_for_matsim_run_test.xml" self.run_config = XMLConfiguration( os.path.join(urbansim_config_location, urbansim_config_name) ).get_run_configuration("Test") # set destination for MATSim config file self.matsim_config_full = os.path.join(self.destination, "test_matsim_config.xml") print "leaving setUp" def tearDown(self): print "entering tearDown" logger.log_status("Removing extracted MATSim files...") if os.path.exists(self.destination): rmtree(self.destination) logger.log_status("... cleaning up finished.") print "leaving tearDown" def testName(self): print "entering test_run" logger.log_status("Preparing MATsim test run ...") # unzip MATSim files matsim_zip = ExtractZipFile(self.matsim_source, self.destination) matsim_zip.extract() matsim_extracted_files = os.path.join(self.destination, "MATSimTestClasses") # location of unziped MATSim files # unzip base_year_cache base_year_data_zip = ExtractZipFile(self.base_year_data_source, self.destination) base_year_data_zip.extract() base_year_data_extracted_files = os.path.join( self.destination, "base_year_data" ) # location of unziped base_year_cache # updating location of base_year_data self.run_config["creating_baseyear_cache_configuration"].cache_directory_root = self.destination self.run_config[ "creating_baseyear_cache_configuration" ].baseyear_cache.existing_cache_to_copy = base_year_data_extracted_files self.run_config["cache_directory"] = base_year_data_extracted_files self.run_config.add("matsim_files", matsim_extracted_files) self.run_config.add("matsim_config", self.matsim_config_full) self.run_config.add("root", self.destination) insert_auto_generated_cache_directory_if_needed(self.run_config) run_manager = RunManager(ServicesDatabaseConfiguration()) run_manager.setup_new_run(cache_directory=self.run_config["cache_directory"], configuration=self.run_config) logger.log_status("Strating UrbanSim run ... ") run_manager.run_run(self.run_config, run_as_multiprocess=True) # after the UrbanSim run the travel data sets schould be equal # self.assertTrue( self.compare_travel_data_sets() ) logger.log_status("... UrbanSim run finished.") print "leaving test_run" def get_source_files(self): """ Returns the path to the MATSim and base_year_data source files """ matsim_source_files = os.path.join(self.path, "data", "MATSimTestClasses.zip") if not os.path.exists(matsim_source_files): raise StandardError("MATSim source file not found: %s" % matsim_source_files) logger.log_status("Referering to MATSim source file: %s" % matsim_source_files) base_year_data_source_files = os.path.join(self.path, "data", "base_year_data.zip") if not os.path.exists(base_year_data_source_files): raise StandardError("Base year data zip file not found: %s" % base_year_data_source_files) logger.log_status("Referering to base year cache file: %s" % base_year_data_source_files) return matsim_source_files, base_year_data_source_files def compare_travel_data_sets(self): # get copied travel data csv copied_travel_data_location = os.path.join(self.destination, "opus_matsim", "tmp") if not os.path.exists(copied_travel_data_location): raise StandardError("Travel data not found: %s" % copied_travel_data_location) logger.log_status("Get copied travel data: %s" % copied_travel_data_location) # convert travel data csv into travel data set matrix in_storage = csv_storage(storage_location=copied_travel_data_location) table_name = "travel_data" travel_data_attribute = "single_vehicle_to_work_travel_cost" travel_data_set = TravelDataDataset(in_storage=in_storage, in_table_name=table_name) travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999) # get exsisting travel data set and convert it also into travel data set matrix year = self.run_config["base_year"] + 2 attribute_cache = AttributeCache(cache_directory=self.run_config["cache_directory"]) cache_storage = attribute_cache.get_flt_storage_for_year(year) existing_travel_data_set = TravelDataDataset(in_storage=cache_storage, in_table_name=table_name) existing_travel_data_attribute_mat = existing_travel_data_set.get_attribute_as_matrix( travel_data_attribute, fill=999 ) from numpy import savetxt # for debugging savetxt(os.path.join(self.destination, "origin_travel_data.txt"), travel_data_attribute_mat, fmt="%f") savetxt(os.path.join(self.destination, "existing_travel_data"), existing_travel_data_attribute_mat, fmt="%f") # compare both data set matices compare = travel_data_attribute_mat == existing_travel_data_attribute_mat # return result return compare.all()