def __wait_for_find_file(constellation_name,
constellation_directory,
machine_names,
ls_cmd,
end_state,
set_cloud_state=False):
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= 'running':
return
ssh_router = _get_ssh_router(constellation_name)
q = []
for machine_name in machine_names:
key_name = "%s_state" % machine_name
if set_cloud_state:
key_name = "%s_aws_state" % machine_name
q.append(
get_ssh_cmd_generator(ssh_router,
"timeout -k 1 10 "
"cloudsim/find_file_%s.bash %s" %
(machine_name, ls_cmd),
ls_cmd,
constellation,
key_name,
end_state,
max_retries=500))
empty_ssh_queue(q, sleep=2)
def monitor_launch(constellation_name, machine_name, counter):
ssh_router = _get_ssh_router(constellation_name)
constellation = ConstellationState(constellation_name)
machine_state = constellation.get_value('%s_state' % machine_name)
monitor_launch_state(constellation_name, ssh_router, machine_state,
"cloudsim/dpkg_log_%s.bash" % machine_name,
'%s_launch_msg' % machine_name)
log("monitor_launch() ENDS %s %s" % (machine_name, counter))
def _get_ssh_router(constellation_name):
constellation = ConstellationState(constellation_name)
constellation_directory = \
constellation.get_value('constellation_directory')
router_ip = constellation.get_value("router_public_ip")
ssh_router = SshClient(constellation_directory, "key-router", 'ubuntu',
router_ip)
return ssh_router
def _get_ssh_client(constellation_name):
constellation = ConstellationState(constellation_name)
constellation_directory = \
constellation.get_value('constellation_directory')
sim_ip = constellation.get_value("sim_public_ip")
ssh_client = SshClient(constellation_directory, "key-sim", 'ubuntu',
sim_ip)
return ssh_client
def monitor(constellation_name, counter):
time.sleep(1)
if constellation_is_terminated(constellation_name):
log("monitor done for %s" % (constellation_name))
return True # stop the monitoring loop
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) < launch_sequence.index('launch'):
return False # do it again later
machines = constellation.get_value('machines')
for machine_name in machines:
monitor_launch(constellation_name, machine_name, counter)
procs = []
p = multiprocessing.Process(target=monitor_simulator_proc,
args=(constellation_name, counter))
procs.append(p)
p = multiprocessing.Process(target=monitor_task_proc,
args=(constellation_name, counter))
procs.append(p)
p = multiprocessing.Process(target=monitor_gzweb_proc,
args=(constellation_name, counter))
procs.append(p)
p = multiprocessing.Process(target=monitor_notebook_proc,
args=(constellation_name, counter))
procs.append(p)
for machine_name, data in machines.iteritems():
ip = data['ip']
if machine_name == "router":
ip = OPENVPN_CLIENT_IP
p = multiprocessing.Process(target=ssh_ping_proc,
args=(constellation_name, ip,
'%s_latency' % machine_name,
counter))
procs.append(p)
for p in procs:
p.start()
for p in procs:
p.join()
return False
def deploy_constellation(constellation_name, cloud_provider, machines,
openvpn_fname):
constellation = ConstellationState(constellation_name)
constellation_directory = \
constellation.get_value('constellation_directory')
deploy_fname = _create_deploy_zip_files(constellation_name,
constellation_directory, machines,
[openvpn_fname])
constellation.set_value('sim_launch_msg',
"waiting for machine to be online")
__wait_for_find_file(constellation_name, constellation_directory, ["sim"],
"launch_stdout_stderr.log", "packages_setup")
__wait_for_find_file(constellation_name, constellation_directory, ["sim"],
"cloudsim/setup/deploy_ready", "packages_setup")
constellation.set_value('sim_launch_msg', "deploying keys")
ssh_client = _get_ssh_client(constellation_name)
ssh_client.upload_file(deploy_fname, "cloudsim/deploy.zip")
ssh_client.cmd('cd cloudsim; unzip -o deploy.zip')
ssh_client.cmd('bash cloudsim/deploy/deploy.bash')
__wait_for_find_file(constellation_name, constellation_directory, ["sim"],
"cloudsim/setup/done", "running")
def atest_launch(self):
print("test_launch")
tags = {}
p = get_boto_path()
launch(self.username,
self.config,
self.constellation_name,
tags,
self.constellation_directory,
credentials_override=p)
print("launched")
for i in range(50):
print('monitor %s' % i)
monitor(self.constellation_name, i)
time.sleep(5)
constellation = ConstellationState(self.constellation_name)
self.assert_(constellation.get_value('gazebo') == "not running", "!")
task = {}
task['latency'] = 0
task['uplink_data_cap'] = 0
task['downlink_data_cap'] = 0
task['ros_package'] = "drcsim_gazebo"
task['ros_launch'] = "vrc_task_1.launch"
task['ros_args'] = ""
task['timeout'] = 60
start_task(self.constellation_name, task)
for i in range(3):
print('monitor %s' % i)
monitor(self.constellation_name, i)
time.sleep(5)
self.assert_(constellation.get_value('gazebo') == "running", "!")
for i in range(20):
print('monitor %s' % i)
monitor(self.constellation_name, i)
time.sleep(5)
def tearDown(self):
print("teardown")
terminate(self.constellation_name)
constellation = ConstellationState(self.constellation_name)
constellation.set_value('constellation_state', 'terminated')
log("Deleting %s from the database" % self.constellation_name)
constellation.expire(1)
def launch(configuration, constellation_name, tags):
"""
Called by cloudsimd when it receives a launch message
"""
constellation = ConstellationState(constellation_name)
use_latest_version = configuration['name'].find('stable') == -1
scripts = {}
scripts['sim'] = ''
cloud_provider = tags['cloud_provider']
constellation_directory = tags['constellation_directory']
credentials_fname = os.path.join(constellation_directory,
'credentials.txt')
log("launch constellation name: %s" % constellation_name)
constellation.set_value("launch_stage", "launch")
ip = "127.0.0.1"
machines = configuration['machines']
constellation.set_value('machines', machines)
_init_computer_data(constellation_name, machines)
# Required only if we are not using a prepolated AMI
if use_latest_version:
drcsim_package_name = "drcsim"
ppa_list = [] # ['ubuntu-x-swat/x-updates']
gpu_driver_list = [
'nvidia-current', 'nvidia-settings', 'nvidia-current-dev',
'nvidia-cg-toolkit'
]
log("DRC package %s" % drcsim_package_name)
log("ppas: %s" % ppa_list)
log("gpu packages %s" % gpu_driver_list)
ubuntu_sources_repo = get_aws_ubuntu_sources_repo(credentials_fname)
script = get_simulator_script(ubuntu_sources_repo,
drcsim_package_name,
ip,
ros_master_ip=ip,
gpu_driver_list=gpu_driver_list,
ppa_list=ppa_list,
OPENVPN_CLIENT_IP=OPENVPN_CLIENT_IP,
OPENVPN_SERVER_IP=OPENVPN_SERVER_IP)
else:
script = ""
if cloud_provider == "aws":
acquire_aws_single_server(
constellation_name,
credentials_ec2=credentials_fname,
constellation_directory=constellation_directory,
machine_prefix='sim',
machine_data=machines['sim'],
startup_script=script,
tags=tags)
else:
raise LaunchException('Unsupported cloud '
'provider "%s"' % cloud_provider)
# Setup the VPN
openvpn_fname = os.path.join(constellation_directory, 'openvpn.key')
create_openvpn_key(openvpn_fname)
_create_zip_files(constellation_name, constellation_directory, machines)
deploy_constellation(constellation_name, cloud_provider, machines,
openvpn_fname)
ssh_client = _get_ssh_client(constellation_name)
# Required only if we are not using a prepolated AMI
if use_latest_version:
# reboot all machines but not router
machines_to_reboot = machines.keys()
_reboot_machines(constellation_name, ssh_client, machines_to_reboot,
constellation_directory)
# Waiting for machines to be ready
machine_names = machines.keys()
log("_run_machines wait for machines %s : %s" %
(constellation_name, machine_names))
__wait_for_find_file(constellation_name, constellation_directory,
machine_names, "cloudsim/setup/done", "running")
log("_run_machines machines %s : %s rebooted!" %
(constellation_name, machine_names))
# make sure the monitoring of package setup is complete
time.sleep(10)
constellation = ConstellationState(constellation_name)
for machine_name in machine_names:
constellation.set_value('%s_aws_state' % machine_name, "running")
constellation.set_value('%s_launch_state' % machine_name, "running")
ssh_client = _get_ssh_client(constellation_name)
log("_run_machines %s: simulator check" % (constellation_name))
if "sim" in machine_names:
_check_opengl_and_x(constellation, ssh_client)
# Install gazebo models locally
# using a utility script from cloudsim-client-tools
# careful, we are running as root here?
# Required only if we are not using a prepolated AMI
if use_latest_version:
constellation.set_value('sim_launch_msg', 'Loading Gazebo models')
ssh_client.cmd("cloudsim/load_gazebo_models.bash")
constellation.set_value('sim_launch_msg', "complete")
log("_run_machines %s: wrap up" % (constellation_name))
constellation.set_value('sim_launch_msg', "complete")
for machine_name in machine_names:
constellation.set_value('%s_launch_msg' % machine_name, "complete")
constellation.set_value('%s_aws_state' % machine_name, "running")
constellation.set_value('%s_launch_state' % machine_name, "running")
constellation.set_value("launch_stage", "running")
return ssh_client.ip
def terminate(constellation_name):
constellation = ConstellationState(constellation_name)
machines = constellation.get_value('machines')
constellation_directory = \
constellation.get_value("constellation_directory")
credentials_fname = os.path.join(constellation_directory,
'credentials.txt')
constellation.set_value('constellation_state', 'terminating')
constellation.set_value('sim_glx_state', "not running")
constellation.set_value('gazebo', "not running")
constellation.set_value('gz_web', "")
for machine_name in machines:
constellation.set_value("%s_state" % machine_name, 'terminating')
constellation.set_value("%s_launch_msg" % machine_name, 'terminating')
constellation.set_value("%s_aws_state" % machine_name, 'terminating')
constellation.set_value("launch_stage", "nothing")
cloud_provider = constellation.get_value('cloud_provider')
if cloud_provider == "aws":
terminate_aws_constellation(constellation_name, credentials_fname)
if cloud_provider == 'softlayer':
partial = False
constellation_prefix = constellation_name.split('_')[-1]
terminate_softlayer_constellation(constellation_name,
constellation_prefix, partial,
credentials_fname)
for machine_name in machines:
constellation.set_value("%s_state" % machine_name, 'terminated')
constellation.set_value("%s_launch_msg" % machine_name, 'terminated')
constellation.set_value("%s_aws_state" % machine_name, 'terminated')
constellation.set_value('constellation_state', 'terminated')
def update(constellation_name):
"""
Update the constellation software on the servers.
This function is a plugin function that should be implemented by
each constellation type
"""
constellation = ConstellationState(constellation_name)
ssh_router = _get_ssh_router(constellation_name)
machines = constellation.get_value('machines')
for machine in machines:
constellation.set_value("%s_state" % machine, "packages_setup")
constellation.set_value("%s_launch_msg" % machine, "updating software")
try:
o = ssh_router.cmd("cloudsim/update_constellation.bash")
log("UPDATE: %s" % o, "toto")
finally:
for machine in machines:
constellation.set_value("%s_state" % machine, "running")
constellation.set_value("%s_launch_msg" % machine, "complete")
log("UPDATE DONE", "toto")
def launch(configuration, constellation_name, tags):
"""
Called by cloudsimd when it receives a launch message
"""
constellation = ConstellationState(constellation_name)
stable = constellation.get_value('configuration').find('stable') >= 0
use_latest_version = stable is False
has_fc1 = False
has_fc2 = False
scripts = {}
scripts['router'] = ''
scripts['sim'] = ''
scripts['fc1'] = ''
scripts['fc2'] = ''
cloud_provider = tags['cloud_provider']
#username = tags['username']
config = tags['configuration']
constellation_directory = tags['constellation_directory']
credentials_fname = os.path.join(constellation_directory,
'credentials.txt')
router_public_network_itf = "eth0"
router_private_network_itf = None
if cloud_provider == "softlayer":
router_public_network_itf = "bond1"
router_private_network_itf = "bond0"
log("launch constellation name: %s" % constellation_name)
constellation.set_value("launch_stage", "launch")
machines = configuration['machines']
_init_computer_data(constellation_name, machines)
ros_master_ip = SIM_IP
# Not required with any custom AMI
if use_latest_version:
drcsim_package_name = "drcsim"
ppa_list = [] # ['ubuntu-x-swat/x-updates']
gpu_driver_list = [
'nvidia-current', 'nvidia-settings', 'nvidia-current-dev',
'nvidia-cg-toolkit'
]
log("DRC package %s" % drcsim_package_name)
log("ppas: %s" % ppa_list)
log("gpu packages %s" % gpu_driver_list)
ubuntu_sources_repo = "http://us.archive.ubuntu.com/ubuntu/"
if cloud_provider == "aws":
ubuntu_sources_repo = \
get_aws_ubuntu_sources_repo(credentials_fname)
scripts['router'] = get_router_script(
ubuntu_sources_repo, router_public_network_itf,
router_private_network_itf, ROUTER_IP, SIM_IP, drcsim_package_name,
OPENVPN_SERVER_IP, OPENVPN_CLIENT_IP)
scripts['sim'] = get_drc_script(ubuntu_sources_repo,
drcsim_package_name, SIM_IP,
ros_master_ip, gpu_driver_list,
ppa_list, OPENVPN_CLIENT_IP, ROUTER_IP)
fc1_script = get_drc_script(ubuntu_sources_repo, drcsim_package_name,
FC1_IP, ros_master_ip, gpu_driver_list,
ppa_list, OPENVPN_CLIENT_IP, ROUTER_IP)
fc2_script = get_drc_script(ubuntu_sources_repo, drcsim_package_name,
FC2_IP, ros_master_ip, gpu_driver_list,
ppa_list, OPENVPN_CLIENT_IP, ROUTER_IP)
if has_fc1:
scripts['fc1'] = fc1_script
if has_fc2:
scripts['fc2'] = fc2_script
if cloud_provider == "softlayer":
log("softlayer %s" % credentials_fname)
constellation_prefix = config.split()[-1]
log("constellation_prefix %s" % constellation_prefix)
partial_deploy = False
if config.find("partial") > 0:
partial_deploy = True
log("partial deploy: %s (only sim and router)" % partial_deploy)
acquire_softlayer_constellation(constellation_name,
constellation_directory,
partial_deploy, constellation_prefix,
credentials_fname, tags,
scripts['router'], scripts['sim'],
fc1_script, fc2_script)
if cloud_provider == "aws":
log("credentials_ec2 %s" % credentials_fname)
acquire_aws_constellation(constellation_name, credentials_fname,
machines, scripts, tags)
# Setup the VPN
ssh_router = _get_ssh_router(constellation_name)
openvpn_fname = os.path.join(constellation_directory, 'openvpn.key')
create_openvpn_key(openvpn_fname)
_create_zip_files(constellation_name, constellation_directory, machines)
deploy_constellation(constellation_name, cloud_provider, machines,
openvpn_fname)
# Not required with any custom AMI
if use_latest_version:
# reboot all machines but not router
machines_to_reboot = machines.keys()
machines_to_reboot.remove('router')
_reboot_machines(constellation_name, ssh_router, machines_to_reboot,
constellation_directory)
# Waiting for machines to be ready
machine_names = machines.keys()
log("_run_machines wait for machines %s : %s" %
(constellation_name, machine_names))
__wait_for_find_file(constellation_name, constellation_directory,
machine_names, "cloudsim/setup/done", "running")
log("_run_machines machines %s : %s rebooted!" %
(constellation_name, machine_names))
# make sure the monitoring of package setup is complete
time.sleep(10)
constellation = ConstellationState(constellation_name)
for machine_name in machine_names:
constellation.set_value('%s_aws_state' % machine_name, "running")
constellation.set_value('%s_launch_state' % machine_name, "running")
ssh_router = _get_ssh_router(constellation_name)
log("_run_machines %s: simulator check" % (constellation_name))
if "sim" in machine_names:
_check_opengl_and_x(constellation, ssh_router)
# Install gazebo models locally
# using a utility script from cloudsim-client-tools
# careful, we are running as root here?
# Not required with any custom AMI
if constellation.get_value('configuration') == 'DRC' or \
constellation.get_value('configuration') == 'DRC with FC':
constellation.set_value('router_launch_msg', 'Final setup')
constellation.set_value('sim_launch_msg', 'Loading Gazebo models')
ssh_router.cmd("cloudsim/ssh-sim.bash "
"cloudsim/load_gazebo_models.bash")
constellation.set_value('sim_launch_msg', "complete")
log("_run_machines %s: wrap up" % (constellation_name))
constellation.set_value('router_launch_msg', "complete")
for machine_name in machine_names:
constellation.set_value('%s_launch_msg' % machine_name, "complete")
constellation.set_value('%s_aws_state' % machine_name, "running")
constellation.set_value('%s_launch_state' % machine_name, "running")
constellation.set_value("launch_stage", "running")
def deploy_constellation(constellation_name, cloud_provider, machines,
openvpn_fname):
constellation = ConstellationState(constellation_name)
constellation_directory = \
constellation.get_value('constellation_directory')
deploy_fname = _create_deploy_zip_files(constellation_name,
constellation_directory, machines,
[openvpn_fname])
constellation.set_value('router_launch_msg',
"waiting for machine to be online")
constellation.set_value('sim_launch_msg', "waiting for router access")
__wait_for_find_file(constellation_name, constellation_directory,
["router"], "launch_stdout_stderr.log",
"packages_setup")
constellation.set_value('router_launch_msg',
"waiting for network connectivity")
__wait_for_find_file(constellation_name, constellation_directory,
["router"], "cloudsim/setup/deploy_ready",
"packages_setup")
constellation.set_value('router_launch_msg', "deploying keys")
ssh_router = _get_ssh_router(constellation_name)
ssh_router.upload_file(deploy_fname, "cloudsim/deploy.zip")
ssh_router.cmd('cd cloudsim; unzip -o deploy.zip')
ssh_router.cmd('bash cloudsim/deploy/deploy.bash')
# cmd = ("nohup sudo bash cloudsim/deploy/deploy.bash "
# "> ssh_deploy.out 2> ssh_deploy.err < /dev/null &")
# ssh_router.cmd(cmd)
__wait_for_find_file(constellation_name, constellation_directory,
["router"], "cloudsim/setup/done", "running")
def _reboot_machines(constellation_name, ssh_router, machine_names,
constellation_directory):
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= launch_sequence.index('reboot'):
return
#constellation.set_value('router_aws_state', m)
__wait_for_find_file(constellation_name, constellation_directory,
machine_names, "cloudsim/setup/done", "running")
constellation.set_value('router_launch_msg',
"Waiting for constellation reboot")
for machine_name in machine_names:
constellation.set_value('%s_launch_msg' % machine_name,
"Rebooting after software installation")
constellation.set_value('%s_aws_state' % machine_name, "rebooting")
for machine_name in machine_names:
ssh_router.cmd("cloudsim/reboot_%s.bash" % machine_name)
log("waiting before connecting after reboot")
time.sleep(20)
constellation.set_value("launch_stage", "reboot")
def _create_zip_files(constellation_name, constellation_directory, machines):
"""
Creates zip files for each machines. Different files are generated for
user roles (ex: user_router.zip has no router ssh key)
"""
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= launch_sequence.index('zip'):
return
for machine_name, machine_data in machines.iteritems():
machine_key_prefix = 'key-%s' % (machine_name)
msg_key = '%s_launch_msg' % machine_name
zip_ready_key = "%s_zip_file" % machine_name
zip_fname = os.path.join(constellation_directory,
"%s.zip" % machine_name)
zip_user_fname = os.path.join(constellation_directory,
"user_%s.zip" % machine_name)
constellation.set_value(msg_key, 'creating zip file')
if machine_name == "router":
router_ip = constellation.get_value("router_public_ip")
router_zip_fname, router_zip_user_fname = create_router_zip(
router_ip, constellation_name, machine_key_prefix,
constellation_directory)
shutil.copy(router_zip_fname, zip_fname)
shutil.copy(router_zip_user_fname, zip_user_fname)
else:
constellation.set_value(msg_key, 'creating zip files')
ip = machine_data['ip']
machine_zip_fname = create_private_machine_zip(
machine_name, ip, constellation_name, constellation_directory,
machine_key_prefix)
shutil.copy(machine_zip_fname, zip_fname)
if machine_name != "sim":
shutil.copy(machine_zip_fname, zip_user_fname)
constellation.set_value(zip_ready_key, 'ready')
constellation.set_value("launch_stage", "zip")
def _init_computer_data(constellation_name, machines):
constellation = ConstellationState(constellation_name)
# init the redis db info
constellation.set_value("gazebo", "not running")
constellation.set_value("sim_glx_state", "not running")
constellation.set_value("constellation_state", "launching")
constellation.set_value("error", "")
constellation.set_value("gazebo", "not running")
constellation.set_value("gzweb", "")
for prefix in machines:
machine = machines[prefix]
ip = machine['ip']
constellation.set_value("%s_ip" % prefix, ip)
constellation.set_value('%s_ip_address' % prefix, "nothing")
constellation.set_value('%s_state' % prefix, "nothing")
constellation.set_value('%s_aws_state' % prefix, 'pending')
constellation.set_value('%s_launch_msg' % prefix, 'starting')
constellation.set_value('%s_zip_file' % prefix, 'not ready')
constellation.set_value('%s_latency' % prefix, '[]')
constellation.set_value('%s_machine_name' % prefix,
'%s_%s' % (prefix, constellation_name))
constellation.set_value('%s_key_name' % prefix, None)
def terminate(constellation_name):
constellation = ConstellationState(constellation_name)
constellation.set_value('constellation_state', 'terminating')
machine_name = constellation.get_value('machine_name')
constellation_directory = \
constellation.get_value("constellation_directory")
credentials_fname = os.path.join(constellation_directory,
'credentials.txt')
constellation.set_value('sim_glx_state', "not running")
constellation.set_value('gazebo', "not running")
constellation.set_value('gz_web', "")
constellation.set_value("%s_state" % machine_name, 'terminating')
constellation.set_value("%s_launch_msg" % machine_name, 'terminating')
constellation.set_value("%s_aws_state" % machine_name, 'terminating')
constellation.set_value("launch_stage", "nothing")
cloud_provider = constellation.get_value('cloud_provider')
if cloud_provider == "aws":
terminate_aws_server(constellation_name, credentials_fname)
constellation.set_value("%s_state" % machine_name, 'terminated')
constellation.set_value("%s_launch_msg" % machine_name, 'terminated')
constellation.set_value("%s_aws_state" % machine_name, 'terminated')
constellation.set_value('constellation_state', 'terminated')
def notify_portal(constellation, task):
try:
root_log_dir = '/tmp/cloudsim_logs'
# Get metadata (team, competition, ...)
config = get_cloudsim_config()
portal_info_fname = config['cloudsim_portal_json_path']
log("** Portal JSON path: %s ***" % portal_info_fname)
portal_info = None
with open(portal_info_fname, 'r') as f:
portal_info = json.loads(f.read())
log("** Portal JSON file opened ***")
team = portal_info['team']
comp = portal_info['event']
task_num = task['vrc_num']
log("** Team: %s, Event: %s ***" % (team, comp))
if task_num < '1' or task_num > '3':
task_num = '1'
task_run = task['vrc_id']
if task_run < '1' or task_run > '5':
task_run = '1'
start_time = task['start_time']
start_task = dateutil.parser.parse(start_time)
start_task = start_task.strftime("%d/%m/%y %H:%M:%S")
const = ConstellationState(constellation)
constellation_dict = get_constellation_data(constellation)
constellation_directory = constellation_dict['constellation_directory']
router_ip = const.get_value("router_public_ip")
task_id = task['ros_launch']
task_dirname = task_id.split('.')[0]
# Store in this cloudsim the network and sim logs
router_key = os.path.join(constellation_directory, 'key-router.pem')
new_msg = task['task_message'] + '<B> Getting logs</B>'
const.update_task_value(task['task_id'], 'task_message', new_msg)
cmd = ('bash /var/www/bin/get_logs.bash %s %s %s' %
(task_dirname, router_ip, router_key))
subprocess.check_call(cmd.split())
log("** Log directory created***")
# Get the score and falls
score = '0'
#falls = 'N/A'
runtime = 'N/A'
try:
p = os.path.join(root_log_dir, task_dirname, 'score.log')
with open(p) as f:
log("** score.log found **")
data = f.read()
log("** Reading score.log file **")
lines = data.split('\n')
last_line = lines[-2]
log("** Last line: %s **" % last_line)
score = last_line.split(',')[4]
#falls = last_line.split(',')[5]
# Time when the task stopped
runtime = last_line.split(',')[1]
log("** All sim score fields parsed **")
except Exception:
None
# Create JSON file with the task metadata
data = json.dumps(
{
'team': team,
'event': comp,
'task': task_num,
'start_time': start_task,
'result': 'Terminated',
'runtime': runtime,
'score': score
},
sort_keys=True,
indent=4,
separators=(',', ': '))
log("** JSON data created **")
with open(os.path.join(root_log_dir, task_dirname, 'end_task.json'),
'w') as f:
f.write(str(data))
log("** JSON file created ***")
new_msg = new_msg.replace('Getting logs', 'Creating tar file')
const.update_task_value(task['task_id'], 'task_message', new_msg)
# Tar all the log content
tar_name = team + '_' + comp + '_' + str(task_num)
tar_name += '_' + str(task_run) + '.tar'
p = os.path.join(root_log_dir, task_dirname)
cmd = 'tar cf /tmp/' + tar_name + ' -C ' + p + ' .'
subprocess.check_call(cmd.split())
log("** Log directory stored in a tar file ***")
new_msg = new_msg.replace('Creating tar file',
'Uploading logs to the portal')
const.update_task_value(task['task_id'], 'task_message', new_msg)
# Send the log to the portal
config = get_cloudsim_config()
portal_info_fname = config['cloudsim_portal_json_path']
portal_info = None
with open(portal_info_fname, 'r') as f:
portal_info = json.loads(f.read())
ssh_portal = SshClient('xxx', 'xxx', portal_info['user'],
portal_info['hostname'])
# this is a hack
ssh_portal.key_fname = config['cloudsim_portal_key_path']
# Upload the file to the Portal temp dir
dest = os.path.join('/tmp', tar_name)
cmd = ('scp -o UserKnownHostsFile=/dev/null'
'-o StrictHostKeyChecking=no'
' -i ' + ssh_portal.key_fname + ' ' + dest + ' ubuntu@' +
portal_info['hostname'] + ':/tmp')
log('cmd: %s' % cmd)
subprocess.check_call(cmd.split())
# Move the file to the final destination into the Portal
final_dest = os.path.join(portal_info['final_destination_dir'],
tar_name)
cmd = 'sudo mv %s %s' % (dest, final_dest)
ssh_portal.cmd(cmd)
new_msg = new_msg.replace('Uploading logs to the portal',
'Logs uploaded to the portal')
const.update_task_value(task['task_id'], 'task_message', new_msg)
except Exception, excep:
log('notify_portal() Exception: %s' % (repr(excep)))
raise
def setUp(self):
print("setup")
self.constellation_name = "test_xxx"
print(self.constellation_name)
self.config = 'AWS DRC with FC' # "AWS DRC"
self.username = "******"
print("%s %s" % (self.config, self.constellation_name))
self.constellation_directory = os.path.join(get_test_dir(),
self.constellation_name)
print(self.constellation_directory)
if os.path.exists(self.constellation_directory):
bk_name = self.constellation_directory + get_unique_short_name('_')
print("backup: %s" % bk_name)
os.rename(self.constellation_directory, bk_name)
os.makedirs(self.constellation_directory)
constellation = ConstellationState(self.constellation_name)
constellation.set_value("constellation_directory",
self.constellation_directory)
constellation.set_value('username', self.username)
constellation.set_value('constellation_name', self.constellation_name)
constellation.set_value('gmt', 'gmt')
constellation.set_value('configuration', self.config)
constellation.set_value('constellation_directory',
self.constellation_directory)
constellation.set_value('error', '')
constellation.set_value('current_task', "")
constellation.set_value('tasks', [])
def atest_launch(self):
constellation_prefix = "02"
#launch_stage = None # use the current stage
#launch_stage = "nothing"
#launch_stage = "os_reload"
# "nothing", "os_reload", "init_router", "init_privates", "zip",
# "change_ip", "startup", "reboot", "running"
launch_stage = "running"
self.constellation_name = 'test_vrc_contest_%s' % constellation_prefix
self.username = "******"
self.credentials_softlayer = get_softlayer_path()
CONFIGURATION = 'vrc_contest'
test_name = "test_" + CONFIGURATION
if not self.constellation_name:
self.constellation_name = get_unique_short_name(test_name + "_")
P = os.path.join(get_test_path(test_name), self.constellation_name)
self.constellation_directory = os.path.abspath(P)
# print("creating: %s" % self.constellation_directory )
os.makedirs(self.constellation_directory)
else:
p = os.path.join(get_test_path(test_name), self.constellation_name)
self.constellation_directory = os.path.abspath(p)
constellation = ConstellationState(self.constellation_name)
constellation.set_value("constellation_name", self.constellation_name)
constellation.set_value("constellation_directory",
self.constellation_directory)
constellation.set_value("configuration", 'vrc_contest')
constellation.set_value('current_task', "")
constellation.set_value('tasks', [])
log(self.constellation_directory)
self.tags = {
'TestCase': CONFIGURATION,
'configuration': CONFIGURATION,
'constellation': self.constellation_name,
'user': self.username,
'GMT': "now"
}
if launch_stage:
constellation.set_value("launch_stage", launch_stage)
config = "OSRF VRC Constellation %s" % constellation_prefix
launch(self.username, config, self.constellation_name, self.tags,
self.constellation_directory)
sweep_count = 2
for i in range(sweep_count):
print("monitoring %s/%s" % (i, sweep_count))
monitor(self.username, self.constellation_name, i)
time.sleep(1)
terminate(self.constellation_name, self.credentials_softlayer)