def test_delete_some_years(self):
cache_dir = self.resources['cache_directory']
self.assert_(os.path.exists(cache_dir))
self.assert_(os.path.exists(os.path.join(cache_dir, '1981')))
path = module_path_from_opus_path('opus_core.tools.delete_run')
cmd_template = sys.executable + ' %(path)s --run-id=%(run_id)d --years-to-delete=%(years_to_delete)s --services_database=services_test'
# First just delete 2 years of data.
python_cmd = cmd_template % {
'path':path,
'run_id':self.resources['run_id'],
'years_to_delete':'[1982,1983]'}
# Close all log files so we can delete the cache.
logger.disable_all_file_logging()
self.do_cmd(python_cmd)
self.assert_(os.path.exists(cache_dir))
self.assert_(os.path.exists(os.path.join(cache_dir, '1981')))
self.assert_(not os.path.exists(os.path.join(cache_dir, '1982')))
self.assert_(not os.path.exists(os.path.join(cache_dir, '1983')))
self.assert_(os.path.exists(os.path.join(cache_dir, '1984')))
# Now see if we can delete another year of data.
python_cmd = cmd_template % {
'path':path,
'run_id':self.resources['run_id'],
'years_to_delete':'1984'}
# Close all log files so we can delete the cache.
logger.disable_all_file_logging()
self.do_cmd(python_cmd)
self.assert_(os.path.exists(cache_dir))
self.assert_(os.path.exists(os.path.join(cache_dir, '1981')))
self.assert_(not os.path.exists(os.path.join(cache_dir, '1982')))
self.assert_(not os.path.exists(os.path.join(cache_dir, '1983')))
self.assert_(not os.path.exists(os.path.join(cache_dir, '1984')))
# Trying to delete the same year again should be a no-op.
python_cmd = cmd_template % {
'path':path,
'run_id':self.resources['run_id'],
'years_to_delete':'1982'}
# Close all log files so we can delete the cache.
logger.disable_all_file_logging()
self.do_cmd(python_cmd)
self.assert_(os.path.exists(cache_dir))
self.assert_(os.path.exists(os.path.join(cache_dir, '1981')))
self.assert_(not os.path.exists(os.path.join(cache_dir, '1982')))
self.assert_(not os.path.exists(os.path.join(cache_dir, '1983')))
self.assert_(not os.path.exists(os.path.join(cache_dir, '1984')))
# Now try to delete the rest of the years of data
python_cmd = '%(executable)s %(path)s --run-id=%(run_id)d --services_database=services_test' % {
'executable':sys.executable,
'path':path,
'run_id':self.resources['run_id']}
# Close all log files so we can delete the cache.
logger.disable_all_file_logging()
self.do_cmd(python_cmd)
self.assert_(not os.path.exists(cache_dir))
def _assemble_command_line_call(self, module_name, resources, pickle_file_path, optional_args=[]):
module_path = module_path_from_opus_path(module_name)
python_cmd = [sys.executable, module_path]
if pickle_file_path:
python_cmd += ["-r", pickle_file_path]
for optional_arg in optional_args:
python_cmd += [str(optional_arg)]
return python_cmd
def _assemble_command_line_call(self, module_name, resources,
pickle_file_path, optional_args=[]):
module_path = module_path_from_opus_path(module_name)
python_cmd = [sys.executable, module_path]
if pickle_file_path:
python_cmd += ["-r", pickle_file_path]
for optional_arg in optional_args:
python_cmd += [str(optional_arg)]
return python_cmd
def _do_run(self, run_id, config, start_year=None, end_year=None):
"""
"""
cache_directory = config['cache_directory']
if start_year is None:
start_year = config['years'][0]
if end_year is None:
end_year = config['years'][1]
travel_model_resources = None
travel_model_years = []
if config.has_key('travel_model_configuration'):
travel_model_resources = copy.deepcopy(config)
if not self.is_localhost(self.urbansim_server_config['hostname']):
travel_model_resources['cache_directory'] = "sftp://%s@%s%s" % (self.urbansim_server_config['username'],
self.urbansim_server_config['hostname'],
cache_directory)
elif not self.is_localhost(self.travelmodel_server_config['hostname']):
## urbansim runs on localhost, and travel model runs on travelmodel_server
## set sftp_flt_storage to the hostname of localhost
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('www.google.com', 80))
urbansim_server = s.getsockname()[0]
s.close()
except:
## this won't work when the hostname cannot be converted to ip address
urbansim_server=socket.gethostbyname(socket.gethostname())
urbansim_user = self.urbansim_server_config.get('username')
if urbansim_user is None or len(urbansim_user)==0:
urbansim_user = getuser()
travel_model_resources['cache_directory'] = "sftp://%s@%s%s" % (urbansim_user,
urbansim_server,
cache_directory)
#only keep sorted travel model years falls into years range
for key in travel_model_resources['travel_model_configuration'].keys():
if type(key) == int:
if key >= start_year and key <= end_year:
travel_model_years.append(key)
if end_year not in travel_model_years:
travel_model_years.append(end_year)
## in the case end_year is not a travel_model year, appending it
## so we have 1 more iteration after the last travel_model_year
travel_model_years.sort()
this_start_year = start_year
for travel_model_year in travel_model_years:
if this_start_year > end_year:
return #run finished, should not be needed
this_end_year = travel_model_year
config['years'] = (this_start_year, this_end_year)
## since there is no --skip-travel-model switch for restart_run yet
## delete travel_model_configuration, so travel model won't run on urbansim_server
if config.has_key('travel_model_configuration'):
del config['travel_model_configuration']
self.update_services_database(self.get_run_manager(), run_id, config)
if not self.is_localhost(self.urbansim_server_config['hostname']):
logger.start_block("Start UrbanSim Simulation on %s from %s to %s" % (self.urbansim_server_config['hostname'],
this_start_year, this_end_year) )
cmd = 'python %(module)s %(run_id)s %(start_year)s ' % \
{'module':self.remote_module_path_from_opus_path(self.get_ssh_client(self.ssh['urbansim_server'], self.urbansim_server_config),
'opus_core.tools.restart_run'),
'run_id':run_id, 'start_year':this_start_year,
'services_hostname': self.services_db_config.host_name}
cmd += ' --skip-cache-cleanup --create-baseyear-cache-if-not-exists >> ' + 'urbansim_run_%s.log' % run_id
## to avoid stdout overfilling sshclient buffer, redirect stdout to a log file
## TODO: better handle the location of the urbansim_remote_run.log
logger.log_status("Call " + cmd)
ssh_client = self.get_ssh_client(self.ssh['urbansim_server'], self.urbansim_server_config)
self.invoke_remote_run(ssh_client, cmd, run_id=run_id)
logger.end_block()
##TODO: open_sftp may need to be closed
if not self.get_ssh_client(self.ssh['urbansim_server'], self.urbansim_server_config).exists_remotely(
convertntslash(os.path.join(cache_directory, str(this_end_year))) ):
raise StandardError, "cache for year %s doesn't exist in directory %s; there may be problem with urbansim run" % \
(this_end_year, cache_directory)
else:
cmd = 'python %(module)s %(run_id)s %(start_year)s ' % \
{'module':module_path_from_opus_path('opus_core.tools.restart_run'),
'run_id':run_id, 'start_year':this_start_year,
'services_hostname': self.services_db_config.host_name}
cmd += ' --skip-cache-cleanup --create-baseyear-cache-if-not-exists'
logger.log_status("Call " + cmd)
os.system(cmd)
if not os.path.exists(os.path.join(cache_directory, str(this_end_year))):
raise StandardError, "cache for year %s doesn't exist in directory %s; there may be problem with urbansim run" % \
(this_end_year, cache_directory)
if travel_model_resources is not None:
if travel_model_resources['travel_model_configuration'].has_key(this_end_year):
travel_model_resources['years'] = (this_end_year, this_end_year)
self.update_services_database(self.get_run_manager(), run_id, travel_model_resources)
if not self.is_localhost(self.travelmodel_server_config['hostname']):
logger.start_block("Start Travel Model on %s from %s to %s" % (self.travelmodel_server_config['hostname'],
this_start_year, this_end_year) )
cmd = 'python %(module)s %(run_id)s %(start_year)s ' % \
{'module':self.remote_module_path_from_opus_path(self.get_ssh_client(self.ssh['travelmodel_server'], self.travelmodel_server_config),
'opus_core.tools.restart_run'),
'run_id':run_id, 'start_year':this_end_year,
'services_hostname': self.services_db_config.host_name}
cmd += ' --skip-cache-cleanup --skip-urbansim >> ' + 'travelmodel_run_%s.log' % run_id
## to avoid stdout overfilling sshclient buffer, redirect stdout to a log file
## TODO: better handle the location of the travelmodel_remote_run.log
logger.log_status("Call " + cmd)
ssh_client = self.get_ssh_client(self.ssh['urbansim_server'], self.urbansim_server_config)
self.invoke_remote_run(ssh_client, cmd, run_id=run_id)
logger.end_block()
else:
cmd = 'python %(module)s %(run_id)s %(start_year)s ' % \
{'module':module_path_from_opus_path('opus_core.tools.restart_run'),
'run_id':run_id, 'start_year':this_end_year,
'services_hostname': self.services_db_config.host_name}
cmd += ' --skip-cache-cleanup --skip-urbansim'
logger.log_status("Call " + cmd)
os.system(cmd)
flt_directory_for_next_year = os.path.join(cache_directory, str(this_end_year+1))
if not self.is_localhost(self.urbansim_server_config['hostname']):
if not self.get_ssh_client(self.ssh['urbansim_server'], self.urbansim_server_config).exists_remotely(
convertntslash(flt_directory_for_next_year) ):
raise StandardError, "travel model didn't create any output for year %s in directory %s on %s; there may be problem with travel model run" % \
(this_end_year+1, cache_directory, self.urbansim_server_config['hostname'])
elif not os.path.exists(flt_directory_for_next_year):
raise StandardError, "travel model didn't create any output for year %s in directory %s; there may be problem with travel model run" % \
(this_end_year+1, cache_directory)
this_start_year = travel_model_year + 1 #next run starting from the next year of a travel model year
return
def _call_script(self, opus_path, args):
Popen( " %s %s %s" % (sys.executable, module_path_from_opus_path(opus_path), ' '.join(args)),
shell = True
).communicate()
def test_delete_some_years(self):
cache_dir = self.resources['cache_directory']
self.assert_(os.path.exists(cache_dir))
self.assert_(os.path.exists(os.path.join(cache_dir, '1981')))
path = module_path_from_opus_path('opus_core.tools.delete_run')
cmd_template = sys.executable + ' %(path)s --run-id=%(run_id)d --years-to-delete=%(years_to_delete)s --services_database=services_test'
# First just delete 2 years of data.
python_cmd = cmd_template % {
'path': path,
'run_id': self.resources['run_id'],
'years_to_delete': '[1982,1983]'
}
# Close all log files so we can delete the cache.
logger.disable_all_file_logging()
self.do_cmd(python_cmd)
self.assert_(os.path.exists(cache_dir))
self.assert_(os.path.exists(os.path.join(cache_dir, '1981')))
self.assert_(not os.path.exists(os.path.join(cache_dir, '1982')))
self.assert_(not os.path.exists(os.path.join(cache_dir, '1983')))
self.assert_(os.path.exists(os.path.join(cache_dir, '1984')))
# Now see if we can delete another year of data.
python_cmd = cmd_template % {
'path': path,
'run_id': self.resources['run_id'],
'years_to_delete': '1984'
}
# Close all log files so we can delete the cache.
logger.disable_all_file_logging()
self.do_cmd(python_cmd)
self.assert_(os.path.exists(cache_dir))
self.assert_(os.path.exists(os.path.join(cache_dir, '1981')))
self.assert_(not os.path.exists(os.path.join(cache_dir, '1982')))
self.assert_(not os.path.exists(os.path.join(cache_dir, '1983')))
self.assert_(not os.path.exists(os.path.join(cache_dir, '1984')))
# Trying to delete the same year again should be a no-op.
python_cmd = cmd_template % {
'path': path,
'run_id': self.resources['run_id'],
'years_to_delete': '1982'
}
# Close all log files so we can delete the cache.
logger.disable_all_file_logging()
self.do_cmd(python_cmd)
self.assert_(os.path.exists(cache_dir))
self.assert_(os.path.exists(os.path.join(cache_dir, '1981')))
self.assert_(not os.path.exists(os.path.join(cache_dir, '1982')))
self.assert_(not os.path.exists(os.path.join(cache_dir, '1983')))
self.assert_(not os.path.exists(os.path.join(cache_dir, '1984')))
# Now try to delete the rest of the years of data
python_cmd = '%(executable)s %(path)s --run-id=%(run_id)d --services_database=services_test' % {
'executable': sys.executable,
'path': path,
'run_id': self.resources['run_id']
}
# Close all log files so we can delete the cache.
logger.disable_all_file_logging()
self.do_cmd(python_cmd)
self.assert_(not os.path.exists(cache_dir))
def _call_script(self, opus_path, args):
Popen(" %s %s %s" %
(sys.executable, module_path_from_opus_path(opus_path),
' '.join(args)),
shell=True).communicate()
def _do_run(self, run_id, config, start_year=None, end_year=None):
"""
"""
cache_directory = config['cache_directory']
if start_year is None:
start_year = config['years'][0]
if end_year is None:
end_year = config['years'][1]
travel_model_resources = None
travel_model_years = []
if config.has_key('travel_model_configuration'):
travel_model_resources = copy.deepcopy(config)
if not self.is_localhost(self.urbansim_server_config['hostname']):
travel_model_resources['cache_directory'] = "sftp://%s@%s%s" % (
self.urbansim_server_config['username'],
self.urbansim_server_config['hostname'], cache_directory)
elif not self.is_localhost(
self.travelmodel_server_config['hostname']):
## urbansim runs on localhost, and travel model runs on travelmodel_server
## set sftp_flt_storage to the hostname of localhost
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('www.google.com', 80))
urbansim_server = s.getsockname()[0]
s.close()
except:
## this won't work when the hostname cannot be converted to ip address
urbansim_server = socket.gethostbyname(
socket.gethostname())
urbansim_user = self.urbansim_server_config.get('username')
if urbansim_user is None or len(urbansim_user) == 0:
urbansim_user = getuser()
travel_model_resources[
'cache_directory'] = "sftp://%s@%s%s" % (
urbansim_user, urbansim_server, cache_directory)
#only keep sorted travel model years falls into years range
for key in travel_model_resources[
'travel_model_configuration'].keys():
if type(key) == int:
if key >= start_year and key <= end_year:
travel_model_years.append(key)
if end_year not in travel_model_years:
travel_model_years.append(end_year)
## in the case end_year is not a travel_model year, appending it
## so we have 1 more iteration after the last travel_model_year
travel_model_years.sort()
this_start_year = start_year
for travel_model_year in travel_model_years:
if this_start_year > end_year:
return #run finished, should not be needed
this_end_year = travel_model_year
config['years'] = (this_start_year, this_end_year)
## since there is no --skip-travel-model switch for restart_run yet
## delete travel_model_configuration, so travel model won't run on urbansim_server
if config.has_key('travel_model_configuration'):
del config['travel_model_configuration']
self.update_services_database(self.get_run_manager(), run_id,
config)
if not self.is_localhost(self.urbansim_server_config['hostname']):
logger.start_block(
"Start UrbanSim Simulation on %s from %s to %s" %
(self.urbansim_server_config['hostname'], this_start_year,
this_end_year))
cmd = 'python %(module)s %(run_id)s %(start_year)s ' % \
{'module':self.remote_module_path_from_opus_path(self.get_ssh_client(self.ssh['urbansim_server'], self.urbansim_server_config),
'opus_core.tools.restart_run'),
'run_id':run_id, 'start_year':this_start_year,
'services_hostname': self.services_db_config.host_name}
cmd += ' --skip-cache-cleanup --create-baseyear-cache-if-not-exists >> ' + 'urbansim_run_%s.log' % run_id
## to avoid stdout overfilling sshclient buffer, redirect stdout to a log file
## TODO: better handle the location of the urbansim_remote_run.log
logger.log_status("Call " + cmd)
ssh_client = self.get_ssh_client(self.ssh['urbansim_server'],
self.urbansim_server_config)
self.invoke_remote_run(ssh_client, cmd, run_id=run_id)
logger.end_block()
##TODO: open_sftp may need to be closed
if not self.get_ssh_client(
self.ssh['urbansim_server'],
self.urbansim_server_config).exists_remotely(
convertntslash(
os.path.join(cache_directory,
str(this_end_year)))):
raise StandardError, "cache for year %s doesn't exist in directory %s; there may be problem with urbansim run" % \
(this_end_year, cache_directory)
else:
cmd = 'python %(module)s %(run_id)s %(start_year)s ' % \
{'module':module_path_from_opus_path('opus_core.tools.restart_run'),
'run_id':run_id, 'start_year':this_start_year,
'services_hostname': self.services_db_config.host_name}
cmd += ' --skip-cache-cleanup --create-baseyear-cache-if-not-exists'
logger.log_status("Call " + cmd)
os.system(cmd)
if not os.path.exists(
os.path.join(cache_directory, str(this_end_year))):
raise StandardError, "cache for year %s doesn't exist in directory %s; there may be problem with urbansim run" % \
(this_end_year, cache_directory)
if travel_model_resources is not None:
if travel_model_resources[
'travel_model_configuration'].has_key(this_end_year):
travel_model_resources['years'] = (this_end_year,
this_end_year)
self.update_services_database(self.get_run_manager(),
run_id,
travel_model_resources)
if not self.is_localhost(
self.travelmodel_server_config['hostname']):
logger.start_block(
"Start Travel Model on %s from %s to %s" %
(self.travelmodel_server_config['hostname'],
this_start_year, this_end_year))
cmd = 'python %(module)s %(run_id)s %(start_year)s ' % \
{'module':self.remote_module_path_from_opus_path(self.get_ssh_client(self.ssh['travelmodel_server'], self.travelmodel_server_config),
'opus_core.tools.restart_run'),
'run_id':run_id, 'start_year':this_end_year,
'services_hostname': self.services_db_config.host_name}
cmd += ' --skip-cache-cleanup --skip-urbansim >> ' + 'travelmodel_run_%s.log' % run_id
## to avoid stdout overfilling sshclient buffer, redirect stdout to a log file
## TODO: better handle the location of the travelmodel_remote_run.log
logger.log_status("Call " + cmd)
ssh_client = self.get_ssh_client(
self.ssh['urbansim_server'],
self.urbansim_server_config)
self.invoke_remote_run(ssh_client, cmd, run_id=run_id)
logger.end_block()
else:
cmd = 'python %(module)s %(run_id)s %(start_year)s ' % \
{'module':module_path_from_opus_path('opus_core.tools.restart_run'),
'run_id':run_id, 'start_year':this_end_year,
'services_hostname': self.services_db_config.host_name}
cmd += ' --skip-cache-cleanup --skip-urbansim'
logger.log_status("Call " + cmd)
os.system(cmd)
flt_directory_for_next_year = os.path.join(
cache_directory, str(this_end_year + 1))
if not self.is_localhost(
self.urbansim_server_config['hostname']):
if not self.get_ssh_client(
self.ssh['urbansim_server'],
self.urbansim_server_config).exists_remotely(
convertntslash(
flt_directory_for_next_year)):
raise StandardError, "travel model didn't create any output for year %s in directory %s on %s; there may be problem with travel model run" % \
(this_end_year+1, cache_directory, self.urbansim_server_config['hostname'])
elif not os.path.exists(flt_directory_for_next_year):
raise StandardError, "travel model didn't create any output for year %s in directory %s; there may be problem with travel model run" % \
(this_end_year+1, cache_directory)
this_start_year = travel_model_year + 1 #next run starting from the next year of a travel model year
return
