def execute_cmd_and_get_stdout(self, cmd, raise_at_error=False): """TODO: rewrite for python2.6 using subprocess module """ cmdfilename = self._write_cmd_string_to_temp_file(cmd) stdin, stdout, stderr = os.popen3( "%s -ssh -l %s -pw %s %s -m %s " % (self.plink, self.username, self.password, self.hostname, cmdfilename)) stderr_msg = stderr.readlines() if len(stderr_msg) > 0: logger.log_error("Error encountered executing cmd through ssh:\n" + ''.join(stderr_msg)) if raise_at_error: raise RuntimeError, "Error encountered executing cmd through ssh:\n" + ''.join( stderr_msg) results = stdout.readlines() if len(results) == 1: results = results[0].strip() stdin.close() stdout.close() stderr.close() os.remove(cmdfilename) return results
def test_get_schema_from_table(self): import copy test_table_schema = { 'col1': "INTEGER", 'col2': "FLOAT", 'col3': "DOUBLE", 'col4': "VARCHAR", 'col5': "BOOLEAN", 'col6': "MEDIUMTEXT", 'col7': "SHORT" } expected_schema = copy.copy(test_table_schema) expected_schema.update({ 'col2': 'DOUBLE', 'col6': 'VARCHAR', 'col7': 'INTEGER' }) test_table = 'test_table' for db, server in self.dbs: try: self.assertFalse(db.table_exists(test_table)) db.create_table_from_schema(test_table, test_table_schema) new_schema = db.get_schema_from_table(test_table) self.assertEqual(new_schema, expected_schema) except: logger.log_error('Protocol %s' % server.config.protocol) raise
def _invoke_with_paramiko_wait_until_done_or_failed(self, ssh_client, run_id=None, raise_at_error=False, msg='\n'): stdin, stdout, stderr = ssh_client.ssh.exec_command(cmd) while True: #handle when command returns an error in stderr try: stdout_msg = stdout.readlines() except: stdout_msg = '' try: stderr_msg = stderr.readlines() except: stderr_msg = '' if len(stderr_msg) > 0: logger.log_error('[' + time.ctime + '] ' + "Error encountered executing cmd through ssh:\n" + ''.join(stderr_msg)) if raise_at_error: raise RuntimeError, "Error encountered executing cmd through ssh:\n" + ''.join(stderr_msg) if len(stdout_msg) > 0: logger.log_status('[' + time.ctime + '] ' + 'stdout:' + ''.join(stdout_msg)) if run_id: runs_by_status = self.get_run_manager().get_runs_by_status([run_id]) if run_id in runs_by_status.get('done', []): break if run_id in runs_by_status.get('failed', []): raise RuntimeError, "run failed: %s." % msg time.sleep(60)
def _put_one_matrix_into_travel_data_set(self, travel_data_set, max_zone_id, matrix_name, attribute_name, bank_path, matrices_created=False): """ Adds to the given travel_data_set the data for the given matrix that is in the emme/2 data bank. """ logger.start_block('Copying data for matrix %s into variable %s' % (matrix_name, attribute_name)) try: if not matrices_created: self._get_matrix_into_data_file(matrix_name, max_zone_id, bank_path) file_name = "_one_matrix.txt" else: file_name = "%s_one_matrix.txt" % matrix_name file_contents = self._get_emme2_data_from_file(join(bank_path, file_name)) travel_data_set.add_primary_attribute(data=zeros(travel_data_set.size(), dtype=float32), name=attribute_name) odv = array([line.split() for line in file_contents], dtype=float32) if odv.size == 0: logger.log_error("Skipped exporting travel_data attribute %s: No data is exported from EMME matrix." % attribute_name) else: travel_data_set.set_values_of_one_attribute_with_od_pairs(attribute=attribute_name, values=odv[:,2], O=odv[:,0].astype('int32'), D=odv[:,1].astype('int32') ) finally: logger.end_block()
def drop_database(self, server, database_name): database_path = self._get_database_path(database_name = database_name) try: os.remove(database_path) except: logger.log_error('Could not remove sqlite database file at %'%database_path) raise
def formatExceptionInfo(custom_message = 'Unexpected error', maxTBlevel=5, plainText=False): import traceback # cla, exc, trbk = sys.exc_info() # excTb = traceback.format_tb(trbk, maxTBlevel) fExc = traceback.format_exc(limit=maxTBlevel) # excName = cla.__name__ # try: # excArgs = exc.__dict__["args"] # except KeyError: # excArgs = "<no args>" # errorinfo = (excName, excArgs, excTb) logger.log_error(traceback.format_exc()) fExc_plain = fExc fExc = fExc.replace('\t',' ').replace('\n','<br>').replace(' ', ' ') errorinfo = ('''<qt>%s</qt> '''%(fExc)) if plainText: return fExc_plain else: return errorinfo
def check_reports_creation_and_sizes( self, travel_model_year_dir, expected_files_and_sizes_dir="expected_file_sizes_e05"): """If there are missing files or any files over 10% missized then an exception will be thrown. If there are files 5-10% missized then a warning will be printed.""" if self.find_errors( travel_model_year_dir, os.path.join( expected_files_and_sizes_dir, os.path.abspath(travel_model_year_dir).split(os.sep)[-1])): # range: (-5%, 50%) -- smaller than 5% or larger than 50% original size are not allowed if self.missing_files: logger.log_error( "%d missing report files: %s" % (len(self.missing_files), str(self.missing_files))) if self.missized_files: logger.log_error("The following files are out of range(-5%, 50%) : " + \ reduce(lambda prev, file: prev + file[0] + " off by %d percent, \n" \ % (file[1]['percent_difference']*100) , \ self.missized_files, "")) if self.missing_files: raise LookupError( "Error, %d missing report files: %s" % (len(self.missing_files), str(self.missing_files))) if self.missized_files: raise StandardError("Error, the following files are out of range(-5%, 50%) : " + \ reduce(lambda prev, file: prev + file[0] + " off by %d percent, \n" \ % (file[1]['percent_difference']*100) , \ self.missized_files, ""))
def get_resources_for_run_id_from_history(self, run_id, filter_by_status=False): """Returns the resources for this run_id, as stored in the run_activity table. """ run_activity = self.services_db.get_table('run_activity') if filter_by_status: whereclause = and_(run_activity.c.status == 'started', run_activity.c.run_id == int(run_id)) else: whereclause = run_activity.c.run_id == int(run_id) query = select(columns=[run_activity.c.resources], whereclause=whereclause) run_resources = self.services_db.execute(query).fetchone() if not run_resources: raise StandardError( "run_id %s doesn't exist on server %s" % (run_id, self.services_db.get_connection_string(scrub=True))) try: r = self._unpickle(run_resources[0]) config = Configuration(r) except: logger.log_error( 'Could not create the configuration file for run %i' % run_id) raise return config
def predict(self, predicted_choice_id_name, agents_index=None): """ Run prediction. Currently makes sense only for choice models.""" # Create temporary configuration where all words 'estimate' are replaced by 'run' tmp_config = Resources(self.config) if self.agents_index_for_prediction is None: self.agents_index_for_prediction = self.get_agent_set_index().copy() if agents_index is None: agents_index = self.agents_index_for_prediction tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['coefficients'] = "coeff_est" tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['agents_index'] = "agents_index" tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['chunk_specification'] = "{'nchunks':1}" ### save specification and coefficients to cache (no matter the save_estimation_results flag) ### so that the prepare_for_run method could load specification and coefficients from there #output_configuration = self.config['output_configuration'] #del self.config['output_configuration'] #self.save_results() #self.config['output_configuration'] = output_configuration #self.model_system.run_year_namespace["coefficients"] = self.coefficients #del tmp_config['models_configuration'][self.model_name]['controller']['prepare_for_run'] try: run_year_namespace = copy.copy(self.model_system.run_year_namespace) except: logger.log_error("The estimate() method must be run first") return False try: agents = self.get_agent_set() choice_id_name = self.get_choice_set().get_id_name()[0] # save current locations of agents current_choices = agents.get_attribute(choice_id_name).copy() dummy_data = zeros(current_choices.size, dtype=current_choices.dtype)-1 #agents.modify_attribute(name=choice_id_name, data=dummy_data) #reset choices for all agents agents.modify_attribute(name=choice_id_name, data=dummy_data, index=agents_index) #reset choices for agents in agents_index run_year_namespace["process"] = "run" run_year_namespace["coeff_est"] = self.coefficients run_year_namespace["agents_index"] = agents_index run_year_namespace["processmodel_config"] = tmp_config['models_configuration'][self.model_name]['controller']['run'] new_choices = self.model_system.do_process(run_year_namespace) #self.model_system.run(tmp_config, write_datasets_to_cache_at_end_of_year=False) #new_choices = agents.get_attribute(choice_id_name).copy() agents.modify_attribute(name=choice_id_name, data=current_choices) dummy_data[agents_index] = new_choices if predicted_choice_id_name not in agents.get_known_attribute_names(): agents.add_primary_attribute(name=predicted_choice_id_name, data=dummy_data) else: agents.modify_attribute(name=predicted_choice_id_name, data=dummy_data) logger.log_status("Predictions saved into attribute " + predicted_choice_id_name) return True except Exception, e: logger.log_error("Error encountered in prediction: %s" % e) logger.log_stack_trace()
def formatExceptionInfo(custom_message='Unexpected error', maxTBlevel=5, plainText=False): import traceback # cla, exc, trbk = sys.exc_info() # excTb = traceback.format_tb(trbk, maxTBlevel) fExc = traceback.format_exc(limit=maxTBlevel) # excName = cla.__name__ # try: # excArgs = exc.__dict__["args"] # except KeyError: # excArgs = "<no args>" # errorinfo = (excName, excArgs, excTb) logger.log_error(traceback.format_exc()) fExc_plain = fExc fExc = fExc.replace('\t', ' ').replace('\n', '<br>').replace(' ', ' ') errorinfo = ('''<qt>%s</qt> ''' % (fExc)) if plainText: return fExc_plain else: return errorinfo
def visualize(self, indicators_to_visualize, computed_indicators, visualization_type, *args, **kwargs): class_names = { "mapnik_map": "MapnikMap", "mapnik_animated_map": "MapnikAnimation", "matplotlib_chart": "MatplotlibChart", "table": "Table", } module = "opus_gui.results_manager.run.indicator_framework.visualizer.visualizers" module_composed_name = module + "." + visualization_type example_indicator = computed_indicators[indicators_to_visualize[0]] indicator_directory = example_indicator.source_data.get_indicator_directory() additional_args = {"indicator_directory": indicator_directory} kwargs.update(additional_args) visualization = ClassFactory().get_class( module_composed_name=module_composed_name, class_name=class_names[visualization_type], arguments=kwargs ) try: visualization = visualization.visualize( indicators_to_visualize=indicators_to_visualize, computed_indicators=computed_indicators, *args ) except Exception, e: msg = "Could not create the %s visualization for %s" % (visualization_type, module_composed_name) logger.log_error(msg) raise
def _clone(self, agents_pool, amount, agent_dataset, location_dataset, this_refinement, dataset_pool ): """ clone certain amount of agent satisfying condition specified by agent_expression and location_expression and add them to agents_pool. Useful to add certain agents to location where there is no such agent exist previously. """ fit_index = self.get_fit_agents_index(agent_dataset, this_refinement.agent_expression, this_refinement.location_expression, dataset_pool) if fit_index.size == 0: logger.log_error("Refinement requests to clone %i agents, but there are no agents satisfying %s." \ % (amount, ' and '.join( [this_refinement.agent_expression, this_refinement.location_expression] ).strip(' and '), )) return clone_index = sample_replace( fit_index, amount ) agents_pool += clone_index.tolist() agent_dataset.modify_attribute(location_dataset.get_id_name()[0], -1 * ones( clone_index.size, dtype='int32' ), index = clone_index ) self._add_refinement_info_to_dataset(agent_dataset, self.id_names, this_refinement, index=clone_index)
def __create_symbolic_link(self, source_file, link_name): ''' this creates a symbolic link / shortcut to a given source_file unfortunately java can't handle shortcuts this is why this method is not used anymore files are renamed instead ''' symbolic_link = os.path.join( self.target_path, link_name ) if self.is_windows_os: # Windows symbolic_link = symbolic_link + '.lnk' try: # removing symbolic link / short cut os.unlink( symbolic_link ) except: pass if self.is_windows_os: # Windows try: logger.log_status('Creating shortcut %s to (%s) ...' %(symbolic_link, source_file) ) from win32com.client import Dispatch shell = Dispatch('WScript.Shell') shortcut = shell.CreateShortCut( symbolic_link ) shortcut.Targetpath = source_file shortcut.save() except: logger.log_error('Error while creating a shortcut to %s!' % source_file) logger.log_error('Installation not successful. Try manual installation as described in opus_matsim_user_guide.pdf in opus_matsim/docs') exit() else: # Mac, Linux logger.log_status('Creating symbolic link %s to (%s) ...' %(symbolic_link, source_file) ) os.symlink( source_file , symbolic_link )
def formatExceptionInfo(custom_message='Unexpected error', maxTBlevel=5, plainText=False, log=True): # cla, exc, trbk = sys.exc_info() # excTb = traceback.format_tb(trbk, maxTBlevel) # excName = cla.__name__ # try: # excArgs = exc.__dict__["args"] # except KeyError: # excArgs = "<no args>" # errorinfo = (excName, excArgs, excTb) format_message_and_error = lambda m, e: ('%s\n%s' % (m, e)) if log: logger.log_error( format_message_and_error(custom_message, traceback.format_exc())) fExc = format_message_and_error(custom_message, traceback.format_exc(limit=maxTBlevel)) if plainText: return fExc else: fExc = fExc.replace('\t', ' ').replace('\n', '<br>').replace(' ', ' ') errorinfo = ('''<qt>%s</qt> ''' % (fExc)) return errorinfo
def _check_syntax_for_dir(self, root_dir, file_names_that_do_not_need_gpl=[]): py_file_names = reduce(self._py_file_names, os.walk(root_dir), []) files_with_no_license = [] files_with_tab = [] for py_file_name in py_file_names: lines_with_tabs = self._has_tabs(py_file_name) if self._file_needs_GPL(os.path.basename(py_file_name), file_names_that_do_not_need_gpl ) and not self._has_GPL(py_file_name): logger.log_error("missing GPL in file %s" % py_file_name) files_with_no_license.append(py_file_name) if lines_with_tabs: logger.log_error( "tab(s) in file %s, line(s) %s" % (py_file_name, list2string(lines_with_tabs, ", "))) files_with_tab.append(py_file_name) if files_with_no_license or files_with_tab: files_with_problems = files_with_no_license + files_with_tab raise SyntaxError( "Please fix reported syntax problems with python files: %s." % (','.join(files_with_problems)))
def invoke_run_travel_model(config, year): """ """ tm_config = config['travel_model_configuration'] scenario = tm_config['travel_model_scenario'] try: travel_model_year = travel_model_year_mapping[year] except KeyError: logger.log_warning("no travel model year mapping for %d." % year) travel_model_year = year # put the travel model input data in place data_exchange_dir = mtc_common.tm_get_data_exchange_dir(config, year) cache_directory = config['cache_directory'] mtc_dir = os.path.join(cache_directory, "mtc_data") for f in glob.glob(os.path.join(mtc_dir, '*' + str(year) + '*')): logger.log_status("Copying over travel model input " + f + " to " + data_exchange_dir) shutil.copy(f, data_exchange_dir) my_location = os.path.split(__file__)[0] script_filepath = os.path.join(my_location, "run_travel_model.py") cmd = "%s %s -s %s -y %s -n" % (sys.executable, script_filepath, scenario, travel_model_year) # form the desired output dir for the travel model data. Make it look # like the urbansim run cache for easy association. Note that we # explicitly use the forward slash instead of os.sep and friends # because the travel model is managed via ssh on a cygwin machine, not # run on the local machine. outdir = "runs/" + config['cache_directory'].split(os.sep)[-1] outdir = outdir + "/%d_%s" % (year, scenario) cmd = cmd + " -o " + outdir logger.log_status("Launching %s" % cmd) if os.system(cmd) != 0: raise TravelModelError # Run the emfac report # TODO: the travel model server should come from the configuration. But # for this we must migrate the configuration from mtc_config.py to the # top-level xml. So for now we just hard-code it :-/ Same for # travel_model_home. tm_server = "*****@*****.**" travel_model_home = "/cygdrive/e/mtc_travel_model/" server_model = winssh.winssh(tm_server, "OPUS_MTC_SERVER_PASSWD") (rc, emfac_windir) = server_model.cmd("cygpath -w " + outdir) if rc != 0: logger.log_error("Failed to find windows path for emfac dir " + outdir) sys.exit(1) emfac_windir = emfac_windir.replace('\r', '').replace('\n','') logger.log_status("Attempting to generate EMFAC report...") cmd = 'cd ' + travel_model_home + 'model_support_files/EMFAC_Files' logger.log_status(cmd) server_model.cmd_or_fail(cmd) cmd = "cmd /c 'RunEmfac.bat " + emfac_windir + " " + str(year) + "' | tee emfac.log" logger.log_status(cmd) (rc, out) = server_model.cmd(cmd, supress_output=False, pipe_position=0) if rc != 0: logger.log_warning("WARNING: Failed to prepare emfac report")
def create_storage(self): try: server = DatabaseServer(self.server_config) except: logger.log_error( 'Cannot connect to the database server that the services database is hosted on %s.' % self.server_config.database_name) raise if not server.has_database(self.server_config.database_name): server.create_database(self.server_config.database_name) try: services_db = server.get_database(self.server_config.database_name) except: logger.log_error('Cannot connect to a services database on %s.' % server.get_connection_string(scrub=True)) raise metadata.bind = services_db.engine setup_all() create_all() return services_db
def run(self, year, output_file=None): """Runs the emme2 executables, using appropriate info from config. Assumes the emme2 input files are present. Raise an exception if the emme2 run fails. """ emme2_batch_file_path = self.get_emme2_batch_file_path(year) emme2_dir, emme2_batch_file_name = os.path.split(emme2_batch_file_path) logger.log_status('Using emme2 dir %s for year %d' % (emme2_dir, year)) os.chdir(emme2_dir) if output_file is None: log_file_path = os.path.join(self.config['cache_directory'], 'emme2_%d_log.txt' % year) else: log_file_path = output_file # if log_file_path is a remote sftp URL, redirect the log file to tempdir log_file_path = redirect_sftp_url_to_local_tempdir(log_file_path) cmd = """%(system_cmd)s"%(emme2_batch_file_name)s" > %(log_file_path)s""" % { 'system_cmd': self.config['travel_model_configuration'].get('system_command', 'cmd /c '), 'emme2_batch_file_name':emme2_batch_file_path, 'log_file_path':log_file_path, } logger.log_status('Running command %s' % cmd) cmd_result = os.system(cmd) if cmd_result != 0: error_msg = "Emme2 Run failed. Code returned by cmd was %d" % (cmd_result) logger.log_error(error_msg) raise StandardError(error_msg)
def get_resources_for_run_id_from_history(self, run_id, filter_by_status = False): """Returns the resources for this run_id, as stored in the run_activity table. """ run_activity = self.services_db.get_table('run_activity') if filter_by_status: whereclause = and_( run_activity.c.status=='started', run_activity.c.run_id==int(run_id)) else: whereclause = run_activity.c.run_id==int(run_id) query = select( columns = [run_activity.c.resources], whereclause = whereclause) run_resources = self.services_db.execute(query).fetchone() if not run_resources: raise StandardError("run_id %s doesn't exist on server %s" % (run_id, self.services_db.get_connection_string(scrub = True))) try: r = self._unpickle(run_resources[0]) config = Configuration(r) except: logger.log_error('Could not create the configuration file for run %i'%run_id) raise return config
def test_get_schema_from_table(self): import copy test_table_schema = { 'col1':"INTEGER", 'col2':"FLOAT", 'col3':"DOUBLE", 'col4':"VARCHAR", 'col5':"BOOLEAN", 'col6':"MEDIUMTEXT", 'col7':"SHORT" } expected_schema = copy.copy(test_table_schema) expected_schema.update({'col2':'DOUBLE', 'col6':'VARCHAR', 'col7':'INTEGER'}) test_table = 'test_table' for db,server in self.dbs: try: self.assertFalse(db.table_exists(test_table)) db.create_table_from_schema(test_table, test_table_schema) new_schema = db.get_schema_from_table(test_table) self.assertEqual(new_schema, expected_schema) except: logger.log_error('Protocol %s'%server.config.protocol) raise
def test_get_schema_from_table(self): import copy test_table_schema = { "col1": "INTEGER", "col2": "FLOAT", "col3": "DOUBLE", "col4": "VARCHAR", "col5": "BOOLEAN", "col6": "MEDIUMTEXT", "col7": "SHORT", } expected_schema = copy.copy(test_table_schema) expected_schema.update({"col2": "DOUBLE", "col6": "VARCHAR", "col7": "INTEGER"}) test_table = "test_table" for db, server in self.dbs: try: self.assertFalse(db.table_exists(test_table)) db.create_table_from_schema(test_table, test_table_schema) new_schema = db.get_schema_from_table(test_table) self.assertEqual(new_schema, expected_schema) except: logger.log_error("Protocol %s" % server.config.protocol) raise
def _find_opus_test_cases_for_package(self, package, test_case_class): root = OpusPackage().get_path_for_package(package) modules_with_test_cases = [] for path, dirs, files in os.walk(root, topdown=True): for file in files: if not file.endswith('.py'): continue f = open(os.path.join(path, file), 'r') import_pattern = re.compile('^\s*(import|from).*unittest') skip_pattern = re.compile('^.*#.*IGNORE_THIS_FILE') found_import = False for line in f: if skip_pattern.match(line): break if import_pattern.match(line): found_import = True break if not found_import: # No unittest import found in file. continue module_name = self._get_module_name(package, root, path, file) try: exec('import %s' % module_name) except Exception, val: logger.log_error("Could not import %s!" % module_name) traceback.print_exc() continue module = eval(module_name) if inspect.ismodule(module): members = inspect.getmembers(module) member_dict = {} for key, value in members: member_dict[key] = value for key in member_dict.keys(): try: is_subclass = issubclass(member_dict[key], test_case_class) except: pass else: if is_subclass: class_name = member_dict[key].__name__ modules_with_test_cases.append( (module_name, class_name)) else: logger.log_warning('WARNING: %s is not a module!' % module)
def run(self, year, output_file=None): """Runs the daysim executables, using appropriate info from config. Assumes the daysim input files are present. Raise an exception if the daysim run fails. """ daysim_config_file = self.get_daysim_config_file(year) daysim_dir = self.get_daysim_dir(year) logger.log_status('Using DaySim directory %s for year %d' % (daysim_dir, year)) os.chdir(daysim_dir) if output_file is None: log_file_path = os.path.join(self.config['cache_directory'], 'daysim_%d_log.txt' % year) else: log_file_path = output_file # if log_file_path is a remote sftp URL, redirect the log file to tempdir log_file_path = redirect_sftp_url_to_local_tempdir(log_file_path) cmd = """%(system_cmd)s"%(config_file)s" > %(log_file_path)s""" % { 'system_cmd': self.config['travel_model_configuration'].get('daysim_system_command', 'cmd /c '), 'config_file':daysim_config_file, 'log_file_path':log_file_path, } logger.log_status('Running command %s' % cmd) cmd_result = os.system(cmd) if cmd_result != 0: error_msg = "DaySim Run failed. Code returned by cmd was %d" % (cmd_result) logger.log_error(error_msg) raise StandardError(error_msg)
def check_reports_creation_and_sizes(self, travel_model_year_dir, expected_files_and_sizes_dir="expected_file_sizes_e05"): """If there are missing files or any files over 10% missized then an exception will be thrown. If there are files 5-10% missized then a warning will be printed.""" if self.find_errors(travel_model_year_dir, os.path.join(expected_files_and_sizes_dir, os.path.abspath(travel_model_year_dir).split(os.sep)[-1])): # range: (-5%, 50%) -- smaller than 5% or larger than 50% original size are not allowed if self.missing_files: logger.log_error("%d missing report files: %s" % (len(self.missing_files), str(self.missing_files))) if self.missized_files: logger.log_error("The following files are out of range(-5%, 50%) : " + \ reduce(lambda prev, file: prev + file[0] + " off by %d percent, \n" \ % (file[1]['percent_difference']*100) , \ self.missized_files, "")) if self.missing_files: raise LookupError("Error, %d missing report files: %s" % (len(self.missing_files), str(self.missing_files))) if self.missized_files: raise StandardError("Error, the following files are out of range(-5%, 50%) : " + \ reduce(lambda prev, file: prev + file[0] + " off by %d percent, \n" \ % (file[1]['percent_difference']*100) , \ self.missized_files, ""))
def visualize(self, indicators_to_visualize, computed_indicators, visualization_type, *args, **kwargs): class_names = { 'mapnik_map': 'MapnikMap', 'mapnik_animated_map': 'MapnikAnimation', 'matplotlib_chart': 'MatplotlibChart', 'table': 'Table' } module = 'opus_gui.results_manager.run.indicator_framework.visualizer.visualizers' module_composed_name = module + '.' + visualization_type example_indicator = computed_indicators[indicators_to_visualize[0]] indicator_directory = example_indicator.source_data.get_indicator_directory( ) additional_args = {'indicator_directory': indicator_directory} kwargs.update(additional_args) visualization = ClassFactory().get_class( module_composed_name=module_composed_name, class_name=class_names[visualization_type], arguments=kwargs) try: visualization = visualization.visualize( indicators_to_visualize=indicators_to_visualize, computed_indicators=computed_indicators, *args) except Exception, e: msg = 'Could not create the %s visualization for %s' % ( visualization_type, module_composed_name) logger.log_error(msg) raise
def convert_table(self, db, table_name, patterns, backup=True, backup_postfix='_old'): try: db.DoQuery('select * from %s' % table_name) except: return if backup: backup_table_name = '%s%s' % (table_name, backup_postfix) i = 0 while self._table_exists(db, backup_table_name): i += 1 backup_table_name = '%s%s%d' % (table_name, backup_postfix, i) db.DoQuery( 'create table %(backup_table)s select * from %(table)s;' % { 'backup_table': backup_table_name, 'table': table_name }) try: db.DoQuery('select * from %s' % backup_table_name) except: logger.log_error("Back up of table '%s' to '%s' failed. " "Skipping conversion." % (table_name, backup_table_name)) return results = db.GetResultsFromQuery('select variable_name from %s' % table_name)[1:] results = [i[0] for i in results] for i in range(len(results)): try: new_row = results[i] for pattern, replacement in patterns: new_row = re.sub(pattern, replacement, new_row) except TypeError: continue # Not dealing with a string here. if new_row == results[i]: continue # Nothing changed. Don't bother with the update query. new_row = '"%s"' % new_row query = ('update %(table)s set variable_name=%(new_row)s where ' 'variable_name="%(old_row)s";' % { 'table': table_name, 'new_row': new_row, 'old_row': results[i] }) db.DoQuery(query)
def _get_viz_spec(self): viz_type = self.cboVizType.currentText() translation = self._get_output_types(viz_type) dataset_name = self.dataset_name output_type = QString(translation[str(self.cboOutputType.currentText())]) indicators = QString(str(self._get_column_values(column = 0))) vals = { 'indicators': indicators, 'output_type': output_type, 'dataset_name': dataset_name, 'visualization_type': QString(self._get_type_mapper()[str(viz_type)]) } if output_type == 'mapnik_map' or output_type == 'mapnik_animated_map': vals['mapnik_bucket_labels'] = self.mapnik_options['mapnik_bucket_labels'] vals['mapnik_bucket_colors'] = self.mapnik_options['mapnik_bucket_colors'] vals['mapnik_bucket_ranges'] = self.mapnik_options['mapnik_bucket_ranges'] vals['mapnik_resolution'] = self.mapnik_options['mapnik_resolution'] vals['mapnik_page_dims'] = self.mapnik_options['mapnik_page_dims'] vals['mapnik_map_lower_left'] = self.mapnik_options['mapnik_map_lower_left'] vals['mapnik_map_upper_right'] = self.mapnik_options['mapnik_map_upper_right'] vals['mapnik_legend_lower_left'] = self.mapnik_options['mapnik_legend_lower_left'] vals['mapnik_legend_upper_right'] = self.mapnik_options['mapnik_legend_upper_right'] elif output_type == 'fixed_field': try: fixed_field_params = QString(str(self._get_column_values(column = 1))) except: errorInfo = formatExceptionInfo() logger.log_error(errorInfo) MessageBox.error(mainwindow = self, text = 'Could not get fixed field parameters for all columns', detailed_text = '') return None vals['fixed_field_specification'] = fixed_field_params vals['id_format'] = self.leOption1.text() elif output_type == 'sql': vals['database_name'] = self.leOption1.text() elif output_type == 'esri': vals['storage_location'] = self.leOption1.text() elif output_type in ('tab', 'xls'): if self.rbSingleTable.isChecked(): output_style = Table.ALL elif self.rbTablePerIndicator.isChecked(): output_style = Table.PER_ATTRIBUTE elif self.rbTablePerYear.isChecked(): output_style = Table.PER_YEAR vals['output_style'] = QString(str(output_style)) if self.appendTypeCheckBox.isChecked(): vals['append_col_type'] = True else: vals['append_col_type'] = False if output_type == 'xls': vals['storage_location'] = self.leOption1.text() return vals
def _get_prediction_year(self, current_year, years): current_year_index = -1 for i in range(len(years)): if current_year == years[i]: prediction_year_index = i+1 if i <= 0 or i >= len(years): logger.log_error("invalid year " + str(current_year)) return years[prediction_year_index]
def _get_previous_year(self, current_year, years): current_year_index = -1 for i in range(len(years)): if current_year == years[i]: current_year_index = i if i <= 0 or i >= len(years): logger.log_error("invalid year " + str(current_year)) return years[current_year_index-1]
def fork_new_process( self, module_name, resources, delete_temp_dir=True, optional_args=[], stdin=None, stdout=None, stderr=None, run_in_background=False, ): """Invoke the module whose fully-qualified opus name is module_name and pass it the pickled resources. Stores resources in pickle_file_path. If quiet=True, the console output for the command will not appear. """ self.module_name = module_name self._pickle_dir = mkdtemp() try: if resources is None: pickle_file_path = None else: pickle_file_path = os.path.join(self._pickle_dir, "resources.pickle") write_resources_to_file(pickle_file_path, resources) self.python_cmd = self._assemble_command_line_call(module_name, resources, pickle_file_path, optional_args) if stdin == PIPE: stdin = subprocess.PIPE if stdout == PIPE: stdout = subprocess.PIPE elif stdout == LOG: log_file_path = os.path.join(self._pickle_dir, "_log_.log") stdout = open(log_file_path, "w") if stderr == PIPE: stderr = subprocess.PIPE elif stderr == STDOUT: stderr = subprocess.STDOUT elif stderr == LOG: log_file_path = os.path.join(self._pickle_dir, "_errlog_.log") stderr = open(log_file_path, "w") logger.log_status("Invoking: %s" % " ".join(self.python_cmd)) self.popen = subprocess.Popen(self.python_cmd, stdin=stdin, stdout=stdout, stderr=stderr) if not run_in_background: self.wait() finally: if not run_in_background and delete_temp_dir: self.cleanup() returncode = self.popen.poll() if returncode != 0: if returncode != 10: logger.log_error("Error encountered in forked process with traceback:") return False else: return True
def _find_opus_test_cases_for_package(self, package, test_case_class): root = OpusPackage().get_path_for_package(package) modules_with_test_cases = [] for path, dirs, files in os.walk(root, topdown=True): for file in files: if not file.endswith(".py"): continue f = open(os.path.join(path, file), "r") import_pattern = re.compile("^\s*(import|from).*unittest") skip_pattern = re.compile("^.*#.*IGNORE_THIS_FILE") found_import = False for line in f: if skip_pattern.match(line): break if import_pattern.match(line): found_import = True break if not found_import: # No unittest import found in file. continue module_name = self._get_module_name(package, root, path, file) try: exec("import %s" % module_name) except Exception, val: logger.log_error("Could not import %s!" % module_name) traceback.print_exc() continue module = eval(module_name) if inspect.ismodule(module): members = inspect.getmembers(module) member_dict = {} for key, value in members: member_dict[key] = value for key in member_dict.keys(): try: is_subclass = issubclass(member_dict[key], test_case_class) except: pass else: if is_subclass: class_name = member_dict[key].__name__ modules_with_test_cases.append((module_name, class_name)) else: logger.log_warning("WARNING: %s is not a module!" % module)
def checkAndCreateFolder(self, path): if not os.path.exists(path): msg = "Folder %s dosn't exist and is created ..." % (path) try: logger.log_status(msg) os.makedirs(path) logger.log_status("done!") except: logger.log_error("Folder could not be created!")
def fork_new_process(self, module_name, resources, delete_temp_dir=True, optional_args=[], stdin=None, stdout=None, stderr=None, run_in_background=False): """Invoke the module whose fully-qualified opus name is module_name and pass it the pickled resources. Stores resources in pickle_file_path. If quiet=True, the console output for the command will not appear. """ self.module_name = module_name self._pickle_dir = mkdtemp() try: if resources is None: pickle_file_path = None else: pickle_file_path = os.path.join(self._pickle_dir, 'resources.pickle') write_resources_to_file(pickle_file_path, resources) self.python_cmd = \ self._assemble_command_line_call(module_name, resources, pickle_file_path, optional_args) if stdin == PIPE: stdin = subprocess.PIPE if stdout == PIPE: stdout = subprocess.PIPE elif stdout == LOG: log_file_path = os.path.join(self._pickle_dir, '_log_.log') stdout = open(log_file_path, "w") if stderr == PIPE: stderr = subprocess.PIPE elif stderr == STDOUT: stderr = subprocess.STDOUT elif stderr == LOG: log_file_path = os.path.join(self._pickle_dir, '_errlog_.log') stderr = open(log_file_path, "w") logger.log_status("Invoking: %s" % " ".join(self.python_cmd)) self.popen = subprocess.Popen(self.python_cmd, stdin=stdin, stdout=stdout, stderr=stderr) if not run_in_background: self.wait() finally: if not run_in_background and delete_temp_dir: self.cleanup() returncode = self.popen.poll() if returncode != 0: if returncode != 10: logger.log_error("Error encountered in forked process with traceback:" ) return False else: return True
def load_table(self, table_name, column_names=Storage.ALL_COLUMNS, lowercase=True): file_path = self._get_file_path_for_table(table_name) if not os.path.exists(file_path): raise NameError("Table '%s' could not be found in %s." % (table_name, self._output_directory)) # load header available_column_names, available_column_types = self._get_header_information(table_name) if lowercase: available_column_names = self._lower_case(available_column_names) input = open(file_path, 'rb') try: reader = csv.reader(input, self._dialect_name) reader.next() # skip header because it was already read above index_map = self._get_index_map(column_names, available_column_names) # load data result = [] for dummy_column_name in available_column_types: result.append([]) for row in reader: for i in range(len(row)): if i in index_map: if available_column_types[i] == 'b1': result[i].append(row[i] == 'True') elif available_column_types[i] == 'f4': result[i].append(float(row[i])) else: result[i].append(row[i]) table = {} for index in index_map: column_dtype = dtype(available_column_types[index]) # Cast each value to its appropriate python type. for j in range(len(result[index])): try: result[index][j] = column_dtype.type(result[index][j]) except: logger.log_error("Error encountered when processing row %s for column %s of type %s: %s." % \ (j+1, available_column_names[index], column_dtype, result[index][j]) ) raise table[available_column_names[index]] = array(result[index], dtype=column_dtype) finally: input.close() return table
def run_transcad_macro(macroname, dbname, args=None): tc = w32c.Dispatch("TransCAD.AutomationServer") try: return_val = tc.Macro(macroname, dbname, args) if return_val is not None: try:logger.log_error( return_val[-1][-1] ) #if the return is an error msg except: return return_val #else pass the return_val to the caller finally: del tc
def post_check(self, values, dataset_pool): """Check for where there is population on fully-water zones (e.g. on no land). """ from numpy import isinf, isnan if isinf(values).sum(): logger.log_error("Variable %s has %d Infinity values" % (self.__class__.__module__, isinf(values).sum())) if isnan(values).sum(): logger.log_error("Variable %s has %d NaN values" % (self.__class__.__module__, isnan(values).sum()))
def load_table(self, table_name, column_names=Storage.ALL_COLUMNS, lowercase=True): db = self._get_db() table = db.get_table( table_name) #Table(table_name, db.metadata, autoload=True) available_column_names = self.get_column_names(table_name, lowercase) final_cols = self._select_columns(column_names, available_column_names) if final_cols == []: return {} col_data = {} selectable_columns = [] table_data = {} for column in table.columns: if lowercase: col_name = column.name.lower() else: col_name = column.name if col_name in final_cols: col_type = self._get_numpy_dtype_from_sql_alchemy_type( column.type) col_data[col_name] = (column, col_type) table_data[col_name] = [] selectable_columns.append(column) query = select(columns=selectable_columns) query_results = db.execute(query) while True: row = query_results.fetchone() if row is None: break for col_name, (column, col_type) in col_data.items(): table_data[col_name].append(row[column]) for col_name, (column, col_type) in col_data.items(): try: table_data[col_name] = array(table_data[col_name], dtype=col_type) except: logger.log_error( "Error occurred when exporting column %s; it may be caused by NULL values." % col_name) raise self._dispose_db(db) return table_data
def on_buttonBox_accepted(self): path = str(self.lePath.text()) if not os.path.exists(path): msg = 'Cannot import, %s does not exist' % path logger.log_warning(msg) MessageBox.warning(mainwindow=self, text=msg, detailed_text='') else: cache_directory = path years = [] for dir in os.listdir(cache_directory): if len(dir) == 4 and dir.isdigit(): years.append(int(dir)) if years == []: msg = 'Cannot import, %s has no run data' % path logger.log_warning(msg) MessageBox.warning(mainwindow=self, text=msg, detailed_text='') else: start_year = min(years) end_year = max(years) project_name = os.environ['OPUSPROJECTNAME'] run_name = os.path.basename(path) server_config = ServicesDatabaseConfiguration() run_manager = RunManager(server_config) run_id = run_manager._get_new_run_id() resources = { 'cache_directory': cache_directory, 'description': '', 'years': (start_year, end_year), 'project_name': project_name } try: run_manager.add_row_to_history(run_id=run_id, resources=resources, status='done', run_name=run_name) update_available_runs(self.project) logger.log_status( 'Added run %s of project %s to run_activity table' % (run_name, project_name)) except: errorInfo = formatExceptionInfo() logger.log_error(errorInfo) MessageBox.error( mainwindow=self, text= 'Could not add run %s of project %s to run_activity table' % (run_name, project_name), detailed_text=errorInfo) self.close()
def on_buttonBox_accepted(self): path = str(self.lePath.text()) if not os.path.exists(path): msg = 'Cannot import, %s does not exist' % path logger.log_warning(msg) MessageBox.warning(mainwindow = self, text = msg, detailed_text = '') else: cache_directory = path years = [] for dir in os.listdir(cache_directory): if len(dir) == 4 and dir.isdigit(): years.append(int(dir)) if years == []: msg = 'Cannot import, %s has no run data'%path logger.log_warning(msg) MessageBox.warning(mainwindow = self, text = msg, detailed_text = '') else: start_year = min(years) end_year = max(years) project_name = os.environ['OPUSPROJECTNAME'] run_name = os.path.basename(path) server_config = ServicesDatabaseConfiguration() run_manager = RunManager(server_config) run_id = run_manager._get_new_run_id() resources = { 'cache_directory': cache_directory, 'description': '', 'years': (start_year, end_year), 'project_name': project_name } try: run_manager.add_row_to_history(run_id = run_id, resources = resources, status = 'done', run_name = run_name) update_available_runs(self.project) logger.log_status('Added run %s of project %s to run_activity table'%(run_name, project_name)) except: errorInfo = formatExceptionInfo() logger.log_error(errorInfo) MessageBox.error(mainwindow = self, text = 'Could not add run %s of project %s to run_activity table'%(run_name, project_name), detailed_text = errorInfo) self.close()
def run_transcad_macro(macroname, dbname, args=None): tc = w32c.Dispatch("TransCAD.AutomationServer") try: return_val = tc.Macro(macroname, dbname, args) if return_val is not None: try: logger.log_error( return_val[-1][-1]) #if the return is an error msg except: return return_val #else pass the return_val to the caller finally: del tc
def get_travel_data_from_travel_model(self, config, year, zone_set ): """ Returns a new travel data set populated by a travel model """ logger.log_status("Running GetTravelModelDataIntoCache with year %d" % (year)) logger.log_status(zone_set) tm_config = config['travel_model_configuration'] base_dir = tm_config['travel_model_base_directory'] run_dir = os.path.join(base_dir, tm_config[year]['year_dir']) in_storage = StorageFactory().get_storage('dict_storage') max_zone_id = zone_set.get_id_attribute().max() in_storage.write_table( table_name=self.TABLE_NAME, table_data=self._read_skims(run_dir, max_zone_id) ) travel_data_set = TravelDataDataset(in_storage=in_storage, in_table_name=self.TABLE_NAME) # Disabling for now. This is problematic because the dispatcher doesn't give # up a handle and things fail. if False: # zip up model_dir cmd = r'"C:\Program Files\7-Zip\7z.exe"' cmd = cmd + ' a %s.7z %s' % (tm_config[year]['year_dir'], tm_config[year]['year_dir']) logger.start_block("Running [%s]" % (cmd)) zipproc = subprocess.Popen( cmd, cwd = base_dir, stdout=subprocess.PIPE ) for line in zipproc.stdout: logger.log_status(line.strip('\r\n')) zipret = zipproc.wait() logger.log_status("Returned %d" % (zipret)) if zipret != 0: print "Zip (%s) exited with bad return code" % (cmd) logger.end_block() # delete everything except for the triptables subdir try: delete_list = os.listdir(run_dir) for del_file in delete_list: if del_file == "triptables": continue del_file_path = os.path.join(run_dir, del_file) if os.path.isfile(del_file_path): os.remove(del_file_path) elif os.path.isdir(del_file_path): shutil.rmtree(del_file_path) # recycle bin remove (drive, tail) = os.path.splitdrive(run_dir) recycle_dir = os.path.join(drive + r"\\", RECYCLE_BIN, tail.lstrip(r"\\")) shutil.rmtree(recycle_dir) except Exception, err: logger.log_error("Error: %s" % str(err))
def prepare_for_run(self, config, year, check_tcw_process=False): """before calling travel model macro, check if transcad GUI is running, if not, try to start transcad binary process""" ## TODO: TransCAD COM server is very picky about tcw.exe process in memory ## as of April 2007, a tcw process started by python won't work ## so manually start TransCAD program is needed before running this script set_project_ini_file(config, year) if not check_tcw_process: return cmdline = config['transcad_binary'] head, tail = os.path.split(cmdline) procname, ext = os.path.splitext(tail) #tcw kill_process = False start_program = False tc_program_classname = "tcFrame" #ClassName for TransCAD program try: hwnd = win32gui.FindWindow(tc_program_classname, None) except: start_program = True # No Transcand Window found, we'll need to start TransCAD program else: try: #first check if tcw process is in memory win32pdhutil.GetPerformanceAttributes('Process', 'ID Process', procname) pids = win32pdhutil.FindPerformanceAttributesByName(procname) for pid in pids: win32process.TerminateProcess(pid) start_program = True except: raise RuntimeError, "Unable to kill TransCAD process in memory" ##transcad not started, try to start it if start_program: try: pass cmdline = win32api.GetShortPathName(cmdline) cmdline = cmdline + " -q" os.system('start /B "start TransCAD" ' + cmdline) #start TransCAD in background time.sleep(9) #procHandles = win32process.CreateProcess(None, cmdline, None, None, 0, 0, None, None, #win32process.STARTUPINFO()) #self.hProcess, hThread, PId, TId = procHandles except: logger.log_error( "Unable to start TransCAD in %s; it must be running to invoke travel model macro." % cmdline) sys.exit(1)
def test_get_table(self): test_table_schema = {"col1": "INTEGER", "col2": "FLOAT"} test_table = "test_table" for db, server in self.dbs: try: self.assertFalse(db.table_exists(test_table)) db.create_table_from_schema(test_table, test_table_schema) t = db.get_table(test_table) self.assertTrue(isinstance(t, Table)) self.assertTrue(t.name == test_table) except: logger.log_error("Protocol %s" % server.config.protocol) raise
def test_get_table(self): test_table_schema = {'col1': "INTEGER", 'col2': "FLOAT"} test_table = 'test_table' for db, server in self.dbs: try: self.assertFalse(db.table_exists(test_table)) db.create_table_from_schema(test_table, test_table_schema) t = db.get_table(test_table) self.assertTrue(isinstance(t, Table)) self.assertTrue(t.name == test_table) except: logger.log_error('Protocol %s' % server.config.protocol) raise
def get_variables_coefficients_from_list(list_spec, definition={}): variables = [] coefficients = [] fixed_values = [] error = False for var_coef in list_spec: if isinstance(var_coef, str): #var_coef is just variables long names or alias var_name, var_index = get_full_variable_specification_for_var_coef( var_coef, definition) variables.append(var_name) if var_index is not None: coefficients.append(definition["coefficients"][var_index]) fixed_values.append(definition["fixed_values"][var_index]) else: coefficients.append(VariableName(var_coef).get_alias()) fixed_values.append(0) elif isinstance(var_coef, tuple) or isinstance(var_coef, list): var_name, var_index = get_full_variable_specification_for_var_coef( var_coef[0], definition) variables.append(var_name) if len(var_coef ) == 1: # coefficient name is created from variable alias coefficients.append(VariableName(var_coef[0]).get_alias()) fixed_values.append(0) elif len(var_coef) > 1: # coefficient names explicitly given coefficients.append(var_coef[1]) if len(var_coef ) > 2: # third item is the coefficient fixed value fixed_values.append(var_coef[2]) else: fixed_values.append(0) else: logger.log_error("Wrong specification format for variable %s" % var_coef) error = True elif isinstance(var_coef, dict): var_name, var_index = get_full_variable_specification_for_var_coef( var_coef.keys()[0], definition) variables.append(var_name) coefficients.append(var_coef.values()[0]) fixed_values.append(0) else: logger.log_error("Wrong specification format for variable %s" % var_coef) error = True return (variables, coefficients, fixed_values, error)
def run(self, config, year, *args, **kwargs): """Runs the travel model, using appropriate info from config. """ tm_config = config["travel_model_configuration"] tm_data_dir = tm_config["directory"] self.prepare_for_run(config, year) logger.log_status('Start travel model from directory %s for year %d' % (tm_data_dir, year)) cmd_batch = self.create_travel_model_command_batch(tm_config, year, *args, **kwargs) logger.log_status('Running command %s' % cmd_batch) cmd_result = os.system(cmd_batch) if cmd_result != 0: error_msg = "Run travel model failed. Code returned by cmd was %d" % (cmd_result) logger.log_error(error_msg) raise StandardError(error_msg)
def __check_and_get_matsim_url(self): logger.log_status('Checking availability of necessary MATSim files (nightly builds): %s' %self.source_url) # parse matsim.org nightly builds webside self.html_finder.feed( urllib2.urlopen( self.source_url ).read() ) if not self.html_finder.allLinksFound(): logger.log_error('Aborting MATSim4UrbanSim installation!') exit() # building usrls for download preparation self.matsim_jar_url = self.source_url + self.html_finder.getMATSimJar() # matsim url self.matsim_lib_url = self.source_url + self.html_finder.getMATSimLib() # lib url self.matsim_contrib_url = self.source_url + self.html_finder.getMATSimContrib() # contrib url logger.log_status('Availability check done!')
def _cast_values(self, values, arguments): """Change the return values to be of type self._return_type. If "should_check" is defined, first check for values that are too large for the destination type or integer wrap-around.""" type = values.dtype.str if self._return_type == type: return values if self.should_check(arguments): max_value = ma.maximum(values) if max_value > self._max_storable_value[self._return_type]: max_value_str = str(max_value) logger.log_error("Variable '%s' is being cast to type '%s', but contains a value (%s) too large to fit into that type." % (self.name(), self._return_type, max_value_str)) return values.astype(self._return_type)
def get_variables_coefficients_equations_for_submodel(submodel_spec, sub_model, definition={}): variables = [] coefficients = [] fixed_values = [] equations = [] submodels = [] other_fields = {} error = False if isinstance(submodel_spec, tuple) or isinstance( submodel_spec, list): # no equations or other fields given variables, coefficients, fixed_values, error = get_variables_coefficients_from_list( submodel_spec, definition) elif isinstance(submodel_spec, dict): name = submodel_spec.get('name', 'equation') if name.startswith( 'equation' ): # by default the dictionary is on an equation level variables, coefficients, fixed_values, equations, error = get_variables_coefficients_equations_from_dict( submodel_spec, definition) else: del submodel_spec['name'] other_fields['dim_%s' % name] = [] for other_field_value, spec in submodel_spec.iteritems(): variable, coefficient, equation, submodel, fixed_value, other_field = \ get_variables_coefficients_equations_for_submodel(spec, sub_model, definition) variables += variable coefficients += coefficient equations += equation submodels += submodel fixed_values += fixed_value other_fields['dim_%s' % name] += len(variable) * [other_field_value] for key, value in other_field.iteritems(): if key in other_fields: other_fields[key] = concatenate( (other_fields[key], value)) else: other_fields[key] = array(value) else: logger.log_error("Error in specification of submodel %s." % sub_model) return ([], [], [], [], [], {}) if error: logger.log_error("Error in specification of submodel %s" % sub_model) submodels = len(variables) * [sub_model] return (variables, coefficients, equations, submodels, fixed_values, other_fields)
class StartRunOptionGroup(object): """ Helper class to start model from an xml config file. """ logger.start_block("Starting UrbanSim") # get program arguments from the command line program_arguments = sys.argv[1:] # default parameters are: # --config=opus_matsim/sustain_city/configs/seattle_parcel_prescheduled_events.xml # --executable=Seattle_baseline parser = optparse.OptionParser() parser.add_option("-c", "--config", dest="config_file_name", action="store", type="string", help="Name of file containing urbansim config") parser.add_option("-e", "--executable", dest="scenario_executable", action="store", type="string", help="Model to execute") (options, args) = parser.parse_args() if options.config_file_name == None: logger.log_error("Missing path to the urbansim config file") if options.scenario_executable == None: logger.log_error("Missing name of executable scenario") config = XMLConfiguration(options.config_file_name).get_run_configuration( options.scenario_executable) insert_auto_generated_cache_directory_if_needed(config) run_manager = RunManager(ServicesDatabaseConfiguration()) run_manager.setup_new_run(cache_directory=config['cache_directory'], configuration=config) #try: #tnicolai # import pydevd # pydevd.settrace() #except: pass run_manager.run_run(config, run_as_multiprocess=True)
def _cast_values(self, values, arguments): """Change the return values to be of type self._return_type. If "should_check" is defined, first check for values that are too large for the destination type or integer wrap-around.""" type = values.dtype.str if self._return_type == type: return values if self.should_check(arguments): max_value = ma.maximum(values) if max_value > self._max_storable_value[self._return_type]: max_value_str = str(max_value) logger.log_error( "Variable '%s' is being cast to type '%s', but contains a value (%s) too large to fit into that type." % (self.name(), self._return_type, max_value_str)) return values.astype(self._return_type)
def execute_cmd_and_get_stdout(self, cmd, raise_at_error=False): """ """ stdin, stdout, stderr = self.ssh.exec_command(cmd) stderr_msg = stderr.readlines() if len(stderr_msg) > 0: logger.log_error("Error encountered executing cmd through ssh:\n" + ''.join(stderr_msg)) if raise_at_error: raise RuntimeError, "Error encountered executing cmd through ssh:\n" + ''.join(stderr_msg) results = stdout.readlines() if len(results)==1: results=results[0].strip() stdin.close(); stdout.close(); stderr.close() return results
def test_create_drop_and_has_table(self): test_table_schema = {'col1': "INTEGER", 'col2': "FLOAT"} test_table = 'test_table' for db, server in self.dbs: try: self.assertFalse(db.table_exists(test_table)) db.create_table_from_schema(test_table, test_table_schema) self.assertTrue(db.table_exists(test_table)) db.drop_table(test_table) self.assertFalse(db.table_exists(test_table)) except: logger.log_error('Protocol %s' % server.config.protocol) raise
def cache_database_tables(self, config, database_name, tables): """Loads cache with all tables from this database. """ database = self.database_server.get_database(database_name) in_storage = StorageFactory().get_storage(type='sql_storage', storage_location=database) for table_name in tables: if table_name in self.tables_to_cache: try: self.cache_database_table(table_name, config['base_year'], database, in_storage, config) except: logger.log_error( "Problem caching table %s from database %s" % (table_name, database_name)) raise self.tables_cached.add(table_name)