def test_delete_some_years(self): cache_dir = self.resources['cache_directory'] self.assert_(os.path.exists(cache_dir)) self.assert_(os.path.exists(os.path.join(cache_dir, '1981'))) path = module_path_from_opus_path('opus_core.tools.delete_run') cmd_template = sys.executable + ' %(path)s --run-id=%(run_id)d --years-to-delete=%(years_to_delete)s --services_database=services_test' # First just delete 2 years of data. python_cmd = cmd_template % { 'path':path, 'run_id':self.resources['run_id'], 'years_to_delete':'[1982,1983]'} # Close all log files so we can delete the cache. logger.disable_all_file_logging() self.do_cmd(python_cmd) self.assert_(os.path.exists(cache_dir)) self.assert_(os.path.exists(os.path.join(cache_dir, '1981'))) self.assert_(not os.path.exists(os.path.join(cache_dir, '1982'))) self.assert_(not os.path.exists(os.path.join(cache_dir, '1983'))) self.assert_(os.path.exists(os.path.join(cache_dir, '1984'))) # Now see if we can delete another year of data. python_cmd = cmd_template % { 'path':path, 'run_id':self.resources['run_id'], 'years_to_delete':'1984'} # Close all log files so we can delete the cache. logger.disable_all_file_logging() self.do_cmd(python_cmd) self.assert_(os.path.exists(cache_dir)) self.assert_(os.path.exists(os.path.join(cache_dir, '1981'))) self.assert_(not os.path.exists(os.path.join(cache_dir, '1982'))) self.assert_(not os.path.exists(os.path.join(cache_dir, '1983'))) self.assert_(not os.path.exists(os.path.join(cache_dir, '1984'))) # Trying to delete the same year again should be a no-op. python_cmd = cmd_template % { 'path':path, 'run_id':self.resources['run_id'], 'years_to_delete':'1982'} # Close all log files so we can delete the cache. logger.disable_all_file_logging() self.do_cmd(python_cmd) self.assert_(os.path.exists(cache_dir)) self.assert_(os.path.exists(os.path.join(cache_dir, '1981'))) self.assert_(not os.path.exists(os.path.join(cache_dir, '1982'))) self.assert_(not os.path.exists(os.path.join(cache_dir, '1983'))) self.assert_(not os.path.exists(os.path.join(cache_dir, '1984'))) # Now try to delete the rest of the years of data python_cmd = '%(executable)s %(path)s --run-id=%(run_id)d --services_database=services_test' % { 'executable':sys.executable, 'path':path, 'run_id':self.resources['run_id']} # Close all log files so we can delete the cache. logger.disable_all_file_logging() self.do_cmd(python_cmd) self.assert_(not os.path.exists(cache_dir))
def _assemble_command_line_call(self, module_name, resources, pickle_file_path, optional_args=[]): module_path = module_path_from_opus_path(module_name) python_cmd = [sys.executable, module_path] if pickle_file_path: python_cmd += ["-r", pickle_file_path] for optional_arg in optional_args: python_cmd += [str(optional_arg)] return python_cmd
def _assemble_command_line_call(self, module_name, resources, pickle_file_path, optional_args=[]): module_path = module_path_from_opus_path(module_name) python_cmd = [sys.executable, module_path] if pickle_file_path: python_cmd += ["-r", pickle_file_path] for optional_arg in optional_args: python_cmd += [str(optional_arg)] return python_cmd
def _do_run(self, run_id, config, start_year=None, end_year=None): """ """ cache_directory = config['cache_directory'] if start_year is None: start_year = config['years'][0] if end_year is None: end_year = config['years'][1] travel_model_resources = None travel_model_years = [] if config.has_key('travel_model_configuration'): travel_model_resources = copy.deepcopy(config) if not self.is_localhost(self.urbansim_server_config['hostname']): travel_model_resources['cache_directory'] = "sftp://%s@%s%s" % (self.urbansim_server_config['username'], self.urbansim_server_config['hostname'], cache_directory) elif not self.is_localhost(self.travelmodel_server_config['hostname']): ## urbansim runs on localhost, and travel model runs on travelmodel_server ## set sftp_flt_storage to the hostname of localhost try: s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect(('www.google.com', 80)) urbansim_server = s.getsockname()[0] s.close() except: ## this won't work when the hostname cannot be converted to ip address urbansim_server=socket.gethostbyname(socket.gethostname()) urbansim_user = self.urbansim_server_config.get('username') if urbansim_user is None or len(urbansim_user)==0: urbansim_user = getuser() travel_model_resources['cache_directory'] = "sftp://%s@%s%s" % (urbansim_user, urbansim_server, cache_directory) #only keep sorted travel model years falls into years range for key in travel_model_resources['travel_model_configuration'].keys(): if type(key) == int: if key >= start_year and key <= end_year: travel_model_years.append(key) if end_year not in travel_model_years: travel_model_years.append(end_year) ## in the case end_year is not a travel_model year, appending it ## so we have 1 more iteration after the last travel_model_year travel_model_years.sort() this_start_year = start_year for travel_model_year in travel_model_years: if this_start_year > end_year: return #run finished, should not be needed this_end_year = travel_model_year config['years'] = (this_start_year, this_end_year) ## since there is no --skip-travel-model switch for restart_run yet ## delete travel_model_configuration, so travel model won't run on urbansim_server if config.has_key('travel_model_configuration'): del config['travel_model_configuration'] self.update_services_database(self.get_run_manager(), run_id, config) if not self.is_localhost(self.urbansim_server_config['hostname']): logger.start_block("Start UrbanSim Simulation on %s from %s to %s" % (self.urbansim_server_config['hostname'], this_start_year, this_end_year) ) cmd = 'python %(module)s %(run_id)s %(start_year)s ' % \ {'module':self.remote_module_path_from_opus_path(self.get_ssh_client(self.ssh['urbansim_server'], self.urbansim_server_config), 'opus_core.tools.restart_run'), 'run_id':run_id, 'start_year':this_start_year, 'services_hostname': self.services_db_config.host_name} cmd += ' --skip-cache-cleanup --create-baseyear-cache-if-not-exists >> ' + 'urbansim_run_%s.log' % run_id ## to avoid stdout overfilling sshclient buffer, redirect stdout to a log file ## TODO: better handle the location of the urbansim_remote_run.log logger.log_status("Call " + cmd) ssh_client = self.get_ssh_client(self.ssh['urbansim_server'], self.urbansim_server_config) self.invoke_remote_run(ssh_client, cmd, run_id=run_id) logger.end_block() ##TODO: open_sftp may need to be closed if not self.get_ssh_client(self.ssh['urbansim_server'], self.urbansim_server_config).exists_remotely( convertntslash(os.path.join(cache_directory, str(this_end_year))) ): raise StandardError, "cache for year %s doesn't exist in directory %s; there may be problem with urbansim run" % \ (this_end_year, cache_directory) else: cmd = 'python %(module)s %(run_id)s %(start_year)s ' % \ {'module':module_path_from_opus_path('opus_core.tools.restart_run'), 'run_id':run_id, 'start_year':this_start_year, 'services_hostname': self.services_db_config.host_name} cmd += ' --skip-cache-cleanup --create-baseyear-cache-if-not-exists' logger.log_status("Call " + cmd) os.system(cmd) if not os.path.exists(os.path.join(cache_directory, str(this_end_year))): raise StandardError, "cache for year %s doesn't exist in directory %s; there may be problem with urbansim run" % \ (this_end_year, cache_directory) if travel_model_resources is not None: if travel_model_resources['travel_model_configuration'].has_key(this_end_year): travel_model_resources['years'] = (this_end_year, this_end_year) self.update_services_database(self.get_run_manager(), run_id, travel_model_resources) if not self.is_localhost(self.travelmodel_server_config['hostname']): logger.start_block("Start Travel Model on %s from %s to %s" % (self.travelmodel_server_config['hostname'], this_start_year, this_end_year) ) cmd = 'python %(module)s %(run_id)s %(start_year)s ' % \ {'module':self.remote_module_path_from_opus_path(self.get_ssh_client(self.ssh['travelmodel_server'], self.travelmodel_server_config), 'opus_core.tools.restart_run'), 'run_id':run_id, 'start_year':this_end_year, 'services_hostname': self.services_db_config.host_name} cmd += ' --skip-cache-cleanup --skip-urbansim >> ' + 'travelmodel_run_%s.log' % run_id ## to avoid stdout overfilling sshclient buffer, redirect stdout to a log file ## TODO: better handle the location of the travelmodel_remote_run.log logger.log_status("Call " + cmd) ssh_client = self.get_ssh_client(self.ssh['urbansim_server'], self.urbansim_server_config) self.invoke_remote_run(ssh_client, cmd, run_id=run_id) logger.end_block() else: cmd = 'python %(module)s %(run_id)s %(start_year)s ' % \ {'module':module_path_from_opus_path('opus_core.tools.restart_run'), 'run_id':run_id, 'start_year':this_end_year, 'services_hostname': self.services_db_config.host_name} cmd += ' --skip-cache-cleanup --skip-urbansim' logger.log_status("Call " + cmd) os.system(cmd) flt_directory_for_next_year = os.path.join(cache_directory, str(this_end_year+1)) if not self.is_localhost(self.urbansim_server_config['hostname']): if not self.get_ssh_client(self.ssh['urbansim_server'], self.urbansim_server_config).exists_remotely( convertntslash(flt_directory_for_next_year) ): raise StandardError, "travel model didn't create any output for year %s in directory %s on %s; there may be problem with travel model run" % \ (this_end_year+1, cache_directory, self.urbansim_server_config['hostname']) elif not os.path.exists(flt_directory_for_next_year): raise StandardError, "travel model didn't create any output for year %s in directory %s; there may be problem with travel model run" % \ (this_end_year+1, cache_directory) this_start_year = travel_model_year + 1 #next run starting from the next year of a travel model year return
def _call_script(self, opus_path, args): Popen( " %s %s %s" % (sys.executable, module_path_from_opus_path(opus_path), ' '.join(args)), shell = True ).communicate()
def test_delete_some_years(self): cache_dir = self.resources['cache_directory'] self.assert_(os.path.exists(cache_dir)) self.assert_(os.path.exists(os.path.join(cache_dir, '1981'))) path = module_path_from_opus_path('opus_core.tools.delete_run') cmd_template = sys.executable + ' %(path)s --run-id=%(run_id)d --years-to-delete=%(years_to_delete)s --services_database=services_test' # First just delete 2 years of data. python_cmd = cmd_template % { 'path': path, 'run_id': self.resources['run_id'], 'years_to_delete': '[1982,1983]' } # Close all log files so we can delete the cache. logger.disable_all_file_logging() self.do_cmd(python_cmd) self.assert_(os.path.exists(cache_dir)) self.assert_(os.path.exists(os.path.join(cache_dir, '1981'))) self.assert_(not os.path.exists(os.path.join(cache_dir, '1982'))) self.assert_(not os.path.exists(os.path.join(cache_dir, '1983'))) self.assert_(os.path.exists(os.path.join(cache_dir, '1984'))) # Now see if we can delete another year of data. python_cmd = cmd_template % { 'path': path, 'run_id': self.resources['run_id'], 'years_to_delete': '1984' } # Close all log files so we can delete the cache. logger.disable_all_file_logging() self.do_cmd(python_cmd) self.assert_(os.path.exists(cache_dir)) self.assert_(os.path.exists(os.path.join(cache_dir, '1981'))) self.assert_(not os.path.exists(os.path.join(cache_dir, '1982'))) self.assert_(not os.path.exists(os.path.join(cache_dir, '1983'))) self.assert_(not os.path.exists(os.path.join(cache_dir, '1984'))) # Trying to delete the same year again should be a no-op. python_cmd = cmd_template % { 'path': path, 'run_id': self.resources['run_id'], 'years_to_delete': '1982' } # Close all log files so we can delete the cache. logger.disable_all_file_logging() self.do_cmd(python_cmd) self.assert_(os.path.exists(cache_dir)) self.assert_(os.path.exists(os.path.join(cache_dir, '1981'))) self.assert_(not os.path.exists(os.path.join(cache_dir, '1982'))) self.assert_(not os.path.exists(os.path.join(cache_dir, '1983'))) self.assert_(not os.path.exists(os.path.join(cache_dir, '1984'))) # Now try to delete the rest of the years of data python_cmd = '%(executable)s %(path)s --run-id=%(run_id)d --services_database=services_test' % { 'executable': sys.executable, 'path': path, 'run_id': self.resources['run_id'] } # Close all log files so we can delete the cache. logger.disable_all_file_logging() self.do_cmd(python_cmd) self.assert_(not os.path.exists(cache_dir))
def _call_script(self, opus_path, args): Popen(" %s %s %s" % (sys.executable, module_path_from_opus_path(opus_path), ' '.join(args)), shell=True).communicate()
def _do_run(self, run_id, config, start_year=None, end_year=None): """ """ cache_directory = config['cache_directory'] if start_year is None: start_year = config['years'][0] if end_year is None: end_year = config['years'][1] travel_model_resources = None travel_model_years = [] if config.has_key('travel_model_configuration'): travel_model_resources = copy.deepcopy(config) if not self.is_localhost(self.urbansim_server_config['hostname']): travel_model_resources['cache_directory'] = "sftp://%s@%s%s" % ( self.urbansim_server_config['username'], self.urbansim_server_config['hostname'], cache_directory) elif not self.is_localhost( self.travelmodel_server_config['hostname']): ## urbansim runs on localhost, and travel model runs on travelmodel_server ## set sftp_flt_storage to the hostname of localhost try: s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect(('www.google.com', 80)) urbansim_server = s.getsockname()[0] s.close() except: ## this won't work when the hostname cannot be converted to ip address urbansim_server = socket.gethostbyname( socket.gethostname()) urbansim_user = self.urbansim_server_config.get('username') if urbansim_user is None or len(urbansim_user) == 0: urbansim_user = getuser() travel_model_resources[ 'cache_directory'] = "sftp://%s@%s%s" % ( urbansim_user, urbansim_server, cache_directory) #only keep sorted travel model years falls into years range for key in travel_model_resources[ 'travel_model_configuration'].keys(): if type(key) == int: if key >= start_year and key <= end_year: travel_model_years.append(key) if end_year not in travel_model_years: travel_model_years.append(end_year) ## in the case end_year is not a travel_model year, appending it ## so we have 1 more iteration after the last travel_model_year travel_model_years.sort() this_start_year = start_year for travel_model_year in travel_model_years: if this_start_year > end_year: return #run finished, should not be needed this_end_year = travel_model_year config['years'] = (this_start_year, this_end_year) ## since there is no --skip-travel-model switch for restart_run yet ## delete travel_model_configuration, so travel model won't run on urbansim_server if config.has_key('travel_model_configuration'): del config['travel_model_configuration'] self.update_services_database(self.get_run_manager(), run_id, config) if not self.is_localhost(self.urbansim_server_config['hostname']): logger.start_block( "Start UrbanSim Simulation on %s from %s to %s" % (self.urbansim_server_config['hostname'], this_start_year, this_end_year)) cmd = 'python %(module)s %(run_id)s %(start_year)s ' % \ {'module':self.remote_module_path_from_opus_path(self.get_ssh_client(self.ssh['urbansim_server'], self.urbansim_server_config), 'opus_core.tools.restart_run'), 'run_id':run_id, 'start_year':this_start_year, 'services_hostname': self.services_db_config.host_name} cmd += ' --skip-cache-cleanup --create-baseyear-cache-if-not-exists >> ' + 'urbansim_run_%s.log' % run_id ## to avoid stdout overfilling sshclient buffer, redirect stdout to a log file ## TODO: better handle the location of the urbansim_remote_run.log logger.log_status("Call " + cmd) ssh_client = self.get_ssh_client(self.ssh['urbansim_server'], self.urbansim_server_config) self.invoke_remote_run(ssh_client, cmd, run_id=run_id) logger.end_block() ##TODO: open_sftp may need to be closed if not self.get_ssh_client( self.ssh['urbansim_server'], self.urbansim_server_config).exists_remotely( convertntslash( os.path.join(cache_directory, str(this_end_year)))): raise StandardError, "cache for year %s doesn't exist in directory %s; there may be problem with urbansim run" % \ (this_end_year, cache_directory) else: cmd = 'python %(module)s %(run_id)s %(start_year)s ' % \ {'module':module_path_from_opus_path('opus_core.tools.restart_run'), 'run_id':run_id, 'start_year':this_start_year, 'services_hostname': self.services_db_config.host_name} cmd += ' --skip-cache-cleanup --create-baseyear-cache-if-not-exists' logger.log_status("Call " + cmd) os.system(cmd) if not os.path.exists( os.path.join(cache_directory, str(this_end_year))): raise StandardError, "cache for year %s doesn't exist in directory %s; there may be problem with urbansim run" % \ (this_end_year, cache_directory) if travel_model_resources is not None: if travel_model_resources[ 'travel_model_configuration'].has_key(this_end_year): travel_model_resources['years'] = (this_end_year, this_end_year) self.update_services_database(self.get_run_manager(), run_id, travel_model_resources) if not self.is_localhost( self.travelmodel_server_config['hostname']): logger.start_block( "Start Travel Model on %s from %s to %s" % (self.travelmodel_server_config['hostname'], this_start_year, this_end_year)) cmd = 'python %(module)s %(run_id)s %(start_year)s ' % \ {'module':self.remote_module_path_from_opus_path(self.get_ssh_client(self.ssh['travelmodel_server'], self.travelmodel_server_config), 'opus_core.tools.restart_run'), 'run_id':run_id, 'start_year':this_end_year, 'services_hostname': self.services_db_config.host_name} cmd += ' --skip-cache-cleanup --skip-urbansim >> ' + 'travelmodel_run_%s.log' % run_id ## to avoid stdout overfilling sshclient buffer, redirect stdout to a log file ## TODO: better handle the location of the travelmodel_remote_run.log logger.log_status("Call " + cmd) ssh_client = self.get_ssh_client( self.ssh['urbansim_server'], self.urbansim_server_config) self.invoke_remote_run(ssh_client, cmd, run_id=run_id) logger.end_block() else: cmd = 'python %(module)s %(run_id)s %(start_year)s ' % \ {'module':module_path_from_opus_path('opus_core.tools.restart_run'), 'run_id':run_id, 'start_year':this_end_year, 'services_hostname': self.services_db_config.host_name} cmd += ' --skip-cache-cleanup --skip-urbansim' logger.log_status("Call " + cmd) os.system(cmd) flt_directory_for_next_year = os.path.join( cache_directory, str(this_end_year + 1)) if not self.is_localhost( self.urbansim_server_config['hostname']): if not self.get_ssh_client( self.ssh['urbansim_server'], self.urbansim_server_config).exists_remotely( convertntslash( flt_directory_for_next_year)): raise StandardError, "travel model didn't create any output for year %s in directory %s on %s; there may be problem with travel model run" % \ (this_end_year+1, cache_directory, self.urbansim_server_config['hostname']) elif not os.path.exists(flt_directory_for_next_year): raise StandardError, "travel model didn't create any output for year %s in directory %s; there may be problem with travel model run" % \ (this_end_year+1, cache_directory) this_start_year = travel_model_year + 1 #next run starting from the next year of a travel model year return