def deploy_constellation(constellation_name, cloud_provider, machines,
openvpn_fname):
constellation = ConstellationState(constellation_name)
constellation_directory = \
constellation.get_value('constellation_directory')
deploy_fname = _create_deploy_zip_files(constellation_name,
constellation_directory, machines,
[openvpn_fname])
constellation.set_value('sim_launch_msg',
"waiting for machine to be online")
__wait_for_find_file(constellation_name, constellation_directory, ["sim"],
"launch_stdout_stderr.log", "packages_setup")
__wait_for_find_file(constellation_name, constellation_directory, ["sim"],
"cloudsim/setup/deploy_ready", "packages_setup")
constellation.set_value('sim_launch_msg', "deploying keys")
ssh_client = _get_ssh_client(constellation_name)
ssh_client.upload_file(deploy_fname, "cloudsim/deploy.zip")
ssh_client.cmd('cd cloudsim; unzip -o deploy.zip')
ssh_client.cmd('bash cloudsim/deploy/deploy.bash')
__wait_for_find_file(constellation_name, constellation_directory, ["sim"],
"cloudsim/setup/done", "running")
def tearDown(self):
print("teardown")
terminate(self.constellation_name)
constellation = ConstellationState(self.constellation_name)
constellation.set_value('constellation_state', 'terminated')
log("Deleting %s from the database" % self.constellation_name)
constellation.expire(1)
def _create_zip_files(constellation_name, constellation_directory, machines):
"""
Creates zip files for each machines. Different files are generated for
user roles (ex: user_router.zip has no router ssh key)
"""
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= launch_sequence.index('zip'):
return
for machine_name, _ in machines.iteritems():
machine_key_prefix = 'key-%s' % (machine_name)
msg_key = '%s_launch_msg' % machine_name
zip_ready_key = "%s_zip_file" % machine_name
zip_fname = os.path.join(constellation_directory,
"%s.zip" % machine_name)
zip_user_fname = os.path.join(constellation_directory,
"user_%s.zip" % machine_name)
constellation.set_value(msg_key, 'creating zip file')
if machine_name == "router":
'''router_ip = constellation.get_value("router_public_ip")
router_zip_fname, router_zip_user_fname = create_router_zip(
router_ip,
constellation_name,
machine_key_prefix,
constellation_directory)
shutil.copy(router_zip_fname, zip_fname)
shutil.copy(router_zip_user_fname, zip_user_fname)'''
pass
else:
constellation.set_value(msg_key, 'creating zip files')
ip = constellation.get_value('sim_public_ip')
machine_zip_fname = create_private_machine_zip(
machine_name, ip, constellation_name, constellation_directory,
machine_key_prefix)
shutil.copy(machine_zip_fname, zip_fname)
if machine_name != "sim":
shutil.copy(machine_zip_fname, zip_user_fname)
else:
zip_fname = os.path.join(constellation_directory,
"simulator.zip")
shutil.copy(machine_zip_fname, zip_fname)
constellation.set_value(zip_ready_key, 'ready')
constellation.set_value("launch_stage", "zip")
def update(constellation_name):
"""
Update the constellation software on the servers.
This function is a plugin function that should be implemented by
each constellation type
"""
constellation = ConstellationState(constellation_name)
ssh_router = _get_ssh_router(constellation_name)
machines = constellation.get_value('machines')
for machine in machines:
constellation.set_value("%s_state" % machine, "packages_setup")
constellation.set_value("%s_launch_msg" % machine, "updating software")
try:
o = ssh_router.cmd("cloudsim/update_constellation.bash")
log("UPDATE: %s" % o, "toto")
finally:
for machine in machines:
constellation.set_value("%s_state" % machine, "running")
constellation.set_value("%s_launch_msg" % machine, "complete")
log("UPDATE DONE", "toto")
def _reboot_machines(constellation_name, ssh_router, machine_names,
constellation_directory):
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= launch_sequence.index('reboot'):
return
#constellation.set_value('router_aws_state', m)
__wait_for_find_file(constellation_name, constellation_directory,
machine_names, "cloudsim/setup/done", "running")
constellation.set_value('router_launch_msg',
"Waiting for constellation reboot")
for machine_name in machine_names:
constellation.set_value('%s_launch_msg' % machine_name,
"Rebooting after software installation")
constellation.set_value('%s_aws_state' % machine_name, "rebooting")
for machine_name in machine_names:
ssh_router.cmd("cloudsim/reboot_%s.bash" % machine_name)
log("waiting before connecting after reboot")
time.sleep(20)
constellation.set_value("launch_stage", "reboot")
def deploy_constellation(constellation_name, cloud_provider, machines,
openvpn_fname):
constellation = ConstellationState(constellation_name)
constellation_directory = \
constellation.get_value('constellation_directory')
deploy_fname = _create_deploy_zip_files(constellation_name,
constellation_directory, machines,
[openvpn_fname])
constellation.set_value('router_launch_msg',
"waiting for machine to be online")
constellation.set_value('sim_launch_msg', "waiting for router access")
__wait_for_find_file(constellation_name, constellation_directory,
["router"], "launch_stdout_stderr.log",
"packages_setup")
constellation.set_value('router_launch_msg',
"waiting for network connectivity")
__wait_for_find_file(constellation_name, constellation_directory,
["router"], "cloudsim/setup/deploy_ready",
"packages_setup")
constellation.set_value('router_launch_msg', "deploying keys")
ssh_router = _get_ssh_router(constellation_name)
ssh_router.upload_file(deploy_fname, "cloudsim/deploy.zip")
ssh_router.cmd('cd cloudsim; unzip -o deploy.zip')
ssh_router.cmd('bash cloudsim/deploy/deploy.bash')
# cmd = ("nohup sudo bash cloudsim/deploy/deploy.bash "
# "> ssh_deploy.out 2> ssh_deploy.err < /dev/null &")
# ssh_router.cmd(cmd)
__wait_for_find_file(constellation_name, constellation_directory,
["router"], "cloudsim/setup/done", "running")
def terminate(constellation_name):
constellation = ConstellationState(constellation_name)
machines = constellation.get_value('machines')
constellation_directory = \
constellation.get_value("constellation_directory")
credentials_fname = os.path.join(constellation_directory,
'credentials.txt')
constellation.set_value('constellation_state', 'terminating')
constellation.set_value('sim_glx_state', "not running")
constellation.set_value('gazebo', "not running")
constellation.set_value('gz_web', "")
for machine_name in machines:
constellation.set_value("%s_state" % machine_name, 'terminating')
constellation.set_value("%s_launch_msg" % machine_name, 'terminating')
constellation.set_value("%s_aws_state" % machine_name, 'terminating')
constellation.set_value("launch_stage", "nothing")
cloud_provider = constellation.get_value('cloud_provider')
if cloud_provider == "aws":
terminate_aws_constellation(constellation_name, credentials_fname)
if cloud_provider == 'softlayer':
partial = False
constellation_prefix = constellation_name.split('_')[-1]
terminate_softlayer_constellation(constellation_name,
constellation_prefix, partial,
credentials_fname)
for machine_name in machines:
constellation.set_value("%s_state" % machine_name, 'terminated')
constellation.set_value("%s_launch_msg" % machine_name, 'terminated')
constellation.set_value("%s_aws_state" % machine_name, 'terminated')
constellation.set_value('constellation_state', 'terminated')
def launch(configuration, constellation_name, tags):
"""
Called by cloudsimd when it receives a launch message
"""
constellation = ConstellationState(constellation_name)
stable = constellation.get_value('configuration').find('stable') >= 0
use_latest_version = stable is False
has_fc1 = False
has_fc2 = False
scripts = {}
scripts['router'] = ''
scripts['sim'] = ''
scripts['fc1'] = ''
scripts['fc2'] = ''
cloud_provider = tags['cloud_provider']
#username = tags['username']
config = tags['configuration']
constellation_directory = tags['constellation_directory']
credentials_fname = os.path.join(constellation_directory,
'credentials.txt')
router_public_network_itf = "eth0"
router_private_network_itf = None
if cloud_provider == "softlayer":
router_public_network_itf = "bond1"
router_private_network_itf = "bond0"
log("launch constellation name: %s" % constellation_name)
constellation.set_value("launch_stage", "launch")
machines = configuration['machines']
_init_computer_data(constellation_name, machines)
ros_master_ip = SIM_IP
# Not required with any custom AMI
if use_latest_version:
drcsim_package_name = "drcsim"
ppa_list = [] # ['ubuntu-x-swat/x-updates']
gpu_driver_list = [
'nvidia-current', 'nvidia-settings', 'nvidia-current-dev',
'nvidia-cg-toolkit'
]
log("DRC package %s" % drcsim_package_name)
log("ppas: %s" % ppa_list)
log("gpu packages %s" % gpu_driver_list)
ubuntu_sources_repo = "http://us.archive.ubuntu.com/ubuntu/"
if cloud_provider == "aws":
ubuntu_sources_repo = \
get_aws_ubuntu_sources_repo(credentials_fname)
scripts['router'] = get_router_script(
ubuntu_sources_repo, router_public_network_itf,
router_private_network_itf, ROUTER_IP, SIM_IP, drcsim_package_name,
OPENVPN_SERVER_IP, OPENVPN_CLIENT_IP)
scripts['sim'] = get_drc_script(ubuntu_sources_repo,
drcsim_package_name, SIM_IP,
ros_master_ip, gpu_driver_list,
ppa_list, OPENVPN_CLIENT_IP, ROUTER_IP)
fc1_script = get_drc_script(ubuntu_sources_repo, drcsim_package_name,
FC1_IP, ros_master_ip, gpu_driver_list,
ppa_list, OPENVPN_CLIENT_IP, ROUTER_IP)
fc2_script = get_drc_script(ubuntu_sources_repo, drcsim_package_name,
FC2_IP, ros_master_ip, gpu_driver_list,
ppa_list, OPENVPN_CLIENT_IP, ROUTER_IP)
if has_fc1:
scripts['fc1'] = fc1_script
if has_fc2:
scripts['fc2'] = fc2_script
if cloud_provider == "softlayer":
log("softlayer %s" % credentials_fname)
constellation_prefix = config.split()[-1]
log("constellation_prefix %s" % constellation_prefix)
partial_deploy = False
if config.find("partial") > 0:
partial_deploy = True
log("partial deploy: %s (only sim and router)" % partial_deploy)
acquire_softlayer_constellation(constellation_name,
constellation_directory,
partial_deploy, constellation_prefix,
credentials_fname, tags,
scripts['router'], scripts['sim'],
fc1_script, fc2_script)
if cloud_provider == "aws":
log("credentials_ec2 %s" % credentials_fname)
acquire_aws_constellation(constellation_name, credentials_fname,
machines, scripts, tags)
# Setup the VPN
ssh_router = _get_ssh_router(constellation_name)
openvpn_fname = os.path.join(constellation_directory, 'openvpn.key')
create_openvpn_key(openvpn_fname)
_create_zip_files(constellation_name, constellation_directory, machines)
deploy_constellation(constellation_name, cloud_provider, machines,
openvpn_fname)
# Not required with any custom AMI
if use_latest_version:
# reboot all machines but not router
machines_to_reboot = machines.keys()
machines_to_reboot.remove('router')
_reboot_machines(constellation_name, ssh_router, machines_to_reboot,
constellation_directory)
# Waiting for machines to be ready
machine_names = machines.keys()
log("_run_machines wait for machines %s : %s" %
(constellation_name, machine_names))
__wait_for_find_file(constellation_name, constellation_directory,
machine_names, "cloudsim/setup/done", "running")
log("_run_machines machines %s : %s rebooted!" %
(constellation_name, machine_names))
# make sure the monitoring of package setup is complete
time.sleep(10)
constellation = ConstellationState(constellation_name)
for machine_name in machine_names:
constellation.set_value('%s_aws_state' % machine_name, "running")
constellation.set_value('%s_launch_state' % machine_name, "running")
ssh_router = _get_ssh_router(constellation_name)
log("_run_machines %s: simulator check" % (constellation_name))
if "sim" in machine_names:
_check_opengl_and_x(constellation, ssh_router)
# Install gazebo models locally
# using a utility script from cloudsim-client-tools
# careful, we are running as root here?
# Not required with any custom AMI
if constellation.get_value('configuration') == 'DRC' or \
constellation.get_value('configuration') == 'DRC with FC':
constellation.set_value('router_launch_msg', 'Final setup')
constellation.set_value('sim_launch_msg', 'Loading Gazebo models')
ssh_router.cmd("cloudsim/ssh-sim.bash "
"cloudsim/load_gazebo_models.bash")
constellation.set_value('sim_launch_msg', "complete")
log("_run_machines %s: wrap up" % (constellation_name))
constellation.set_value('router_launch_msg', "complete")
for machine_name in machine_names:
constellation.set_value('%s_launch_msg' % machine_name, "complete")
constellation.set_value('%s_aws_state' % machine_name, "running")
constellation.set_value('%s_launch_state' % machine_name, "running")
constellation.set_value("launch_stage", "running")
def _init_computer_data(constellation_name, machines):
constellation = ConstellationState(constellation_name)
# init the redis db info
constellation.set_value("gazebo", "not running")
constellation.set_value("sim_glx_state", "not running")
constellation.set_value("constellation_state", "launching")
constellation.set_value("error", "")
constellation.set_value("gazebo", "not running")
constellation.set_value("gzweb", "")
for prefix in machines:
machine = machines[prefix]
ip = machine['ip']
constellation.set_value("%s_ip" % prefix, ip)
constellation.set_value('%s_ip_address' % prefix, "nothing")
constellation.set_value('%s_state' % prefix, "nothing")
constellation.set_value('%s_aws_state' % prefix, 'pending')
constellation.set_value('%s_launch_msg' % prefix, 'starting')
constellation.set_value('%s_zip_file' % prefix, 'not ready')
constellation.set_value('%s_latency' % prefix, '[]')
constellation.set_value('%s_machine_name' % prefix,
'%s_%s' % (prefix, constellation_name))
constellation.set_value('%s_key_name' % prefix, None)
def setUp(self):
print("setup")
self.constellation_name = "test_xxx"
print(self.constellation_name)
self.config = 'AWS DRC with FC' # "AWS DRC"
self.username = "******"
print("%s %s" % (self.config, self.constellation_name))
self.constellation_directory = os.path.join(get_test_dir(),
self.constellation_name)
print(self.constellation_directory)
if os.path.exists(self.constellation_directory):
bk_name = self.constellation_directory + get_unique_short_name('_')
print("backup: %s" % bk_name)
os.rename(self.constellation_directory, bk_name)
os.makedirs(self.constellation_directory)
constellation = ConstellationState(self.constellation_name)
constellation.set_value("constellation_directory",
self.constellation_directory)
constellation.set_value('username', self.username)
constellation.set_value('constellation_name', self.constellation_name)
constellation.set_value('gmt', 'gmt')
constellation.set_value('configuration', self.config)
constellation.set_value('constellation_directory',
self.constellation_directory)
constellation.set_value('error', '')
constellation.set_value('current_task', "")
constellation.set_value('tasks', [])
def atest_launch(self):
constellation_prefix = "02"
#launch_stage = None # use the current stage
#launch_stage = "nothing"
#launch_stage = "os_reload"
# "nothing", "os_reload", "init_router", "init_privates", "zip",
# "change_ip", "startup", "reboot", "running"
launch_stage = "running"
self.constellation_name = 'test_vrc_contest_%s' % constellation_prefix
self.username = "******"
self.credentials_softlayer = get_softlayer_path()
CONFIGURATION = 'vrc_contest'
test_name = "test_" + CONFIGURATION
if not self.constellation_name:
self.constellation_name = get_unique_short_name(test_name + "_")
P = os.path.join(get_test_path(test_name), self.constellation_name)
self.constellation_directory = os.path.abspath(P)
# print("creating: %s" % self.constellation_directory )
os.makedirs(self.constellation_directory)
else:
p = os.path.join(get_test_path(test_name), self.constellation_name)
self.constellation_directory = os.path.abspath(p)
constellation = ConstellationState(self.constellation_name)
constellation.set_value("constellation_name", self.constellation_name)
constellation.set_value("constellation_directory",
self.constellation_directory)
constellation.set_value("configuration", 'vrc_contest')
constellation.set_value('current_task', "")
constellation.set_value('tasks', [])
log(self.constellation_directory)
self.tags = {
'TestCase': CONFIGURATION,
'configuration': CONFIGURATION,
'constellation': self.constellation_name,
'user': self.username,
'GMT': "now"
}
if launch_stage:
constellation.set_value("launch_stage", launch_stage)
config = "OSRF VRC Constellation %s" % constellation_prefix
launch(self.username, config, self.constellation_name, self.tags,
self.constellation_directory)
sweep_count = 2
for i in range(sweep_count):
print("monitoring %s/%s" % (i, sweep_count))
monitor(self.username, self.constellation_name, i)
time.sleep(1)
terminate(self.constellation_name, self.credentials_softlayer)
def terminate(constellation_name):
constellation = ConstellationState(constellation_name)
constellation.set_value('constellation_state', 'terminating')
machine_name = constellation.get_value('machine_name')
constellation_directory = \
constellation.get_value("constellation_directory")
credentials_fname = os.path.join(constellation_directory,
'credentials.txt')
constellation.set_value('sim_glx_state', "not running")
constellation.set_value('gazebo', "not running")
constellation.set_value('gz_web', "")
constellation.set_value("%s_state" % machine_name, 'terminating')
constellation.set_value("%s_launch_msg" % machine_name, 'terminating')
constellation.set_value("%s_aws_state" % machine_name, 'terminating')
constellation.set_value("launch_stage", "nothing")
cloud_provider = constellation.get_value('cloud_provider')
if cloud_provider == "aws":
terminate_aws_server(constellation_name, credentials_fname)
constellation.set_value("%s_state" % machine_name, 'terminated')
constellation.set_value("%s_launch_msg" % machine_name, 'terminated')
constellation.set_value("%s_aws_state" % machine_name, 'terminated')
constellation.set_value('constellation_state', 'terminated')
def launch(configuration, constellation_name, tags):
"""
Called by cloudsimd when it receives a launch message
"""
constellation = ConstellationState(constellation_name)
use_latest_version = configuration['name'].find('stable') == -1
scripts = {}
scripts['sim'] = ''
cloud_provider = tags['cloud_provider']
constellation_directory = tags['constellation_directory']
credentials_fname = os.path.join(constellation_directory,
'credentials.txt')
log("launch constellation name: %s" % constellation_name)
constellation.set_value("launch_stage", "launch")
ip = "127.0.0.1"
machines = configuration['machines']
constellation.set_value('machines', machines)
_init_computer_data(constellation_name, machines)
# Required only if we are not using a prepolated AMI
if use_latest_version:
drcsim_package_name = "drcsim"
ppa_list = [] # ['ubuntu-x-swat/x-updates']
gpu_driver_list = [
'nvidia-current', 'nvidia-settings', 'nvidia-current-dev',
'nvidia-cg-toolkit'
]
log("DRC package %s" % drcsim_package_name)
log("ppas: %s" % ppa_list)
log("gpu packages %s" % gpu_driver_list)
ubuntu_sources_repo = get_aws_ubuntu_sources_repo(credentials_fname)
script = get_simulator_script(ubuntu_sources_repo,
drcsim_package_name,
ip,
ros_master_ip=ip,
gpu_driver_list=gpu_driver_list,
ppa_list=ppa_list,
OPENVPN_CLIENT_IP=OPENVPN_CLIENT_IP,
OPENVPN_SERVER_IP=OPENVPN_SERVER_IP)
else:
script = ""
if cloud_provider == "aws":
acquire_aws_single_server(
constellation_name,
credentials_ec2=credentials_fname,
constellation_directory=constellation_directory,
machine_prefix='sim',
machine_data=machines['sim'],
startup_script=script,
tags=tags)
else:
raise LaunchException('Unsupported cloud '
'provider "%s"' % cloud_provider)
# Setup the VPN
openvpn_fname = os.path.join(constellation_directory, 'openvpn.key')
create_openvpn_key(openvpn_fname)
_create_zip_files(constellation_name, constellation_directory, machines)
deploy_constellation(constellation_name, cloud_provider, machines,
openvpn_fname)
ssh_client = _get_ssh_client(constellation_name)
# Required only if we are not using a prepolated AMI
if use_latest_version:
# reboot all machines but not router
machines_to_reboot = machines.keys()
_reboot_machines(constellation_name, ssh_client, machines_to_reboot,
constellation_directory)
# Waiting for machines to be ready
machine_names = machines.keys()
log("_run_machines wait for machines %s : %s" %
(constellation_name, machine_names))
__wait_for_find_file(constellation_name, constellation_directory,
machine_names, "cloudsim/setup/done", "running")
log("_run_machines machines %s : %s rebooted!" %
(constellation_name, machine_names))
# make sure the monitoring of package setup is complete
time.sleep(10)
constellation = ConstellationState(constellation_name)
for machine_name in machine_names:
constellation.set_value('%s_aws_state' % machine_name, "running")
constellation.set_value('%s_launch_state' % machine_name, "running")
ssh_client = _get_ssh_client(constellation_name)
log("_run_machines %s: simulator check" % (constellation_name))
if "sim" in machine_names:
_check_opengl_and_x(constellation, ssh_client)
# Install gazebo models locally
# using a utility script from cloudsim-client-tools
# careful, we are running as root here?
# Required only if we are not using a prepolated AMI
if use_latest_version:
constellation.set_value('sim_launch_msg', 'Loading Gazebo models')
ssh_client.cmd("cloudsim/load_gazebo_models.bash")
constellation.set_value('sim_launch_msg', "complete")
log("_run_machines %s: wrap up" % (constellation_name))
constellation.set_value('sim_launch_msg', "complete")
for machine_name in machine_names:
constellation.set_value('%s_launch_msg' % machine_name, "complete")
constellation.set_value('%s_aws_state' % machine_name, "running")
constellation.set_value('%s_launch_state' % machine_name, "running")
constellation.set_value("launch_stage", "running")
return ssh_client.ip