def monitor_gzweb(constellation_name, ssh_client, sim_state):
"""
Detects if the gzweb is running and writes the
url into the "gzweb" dictionary key
"""
constellation = ConstellationState(constellation_name)
simulation_state = constellation.get_value('sim_state')
if machine_states.index(simulation_state) >= \
machine_states.index('running'):
gl_state = constellation.get_value("gazebo")
if gl_state == "running":
try:
# current_state = constellation.get_value(GZWEB_KEY)
out = ssh_client.cmd("bash cloudsim/ping_gzweb.bash")
log("ping_gzweb returned [%s]" % out)
if out == "":
constellation.set_value(GZWEB_KEY, "not running")
return False
else:
constellation.set_value(GZWEB_KEY, "running")
return True
except Exception, e:
log("monitor: cloudsim/ping_gzweb.bash error: %s" % e)
constellation.set_value(GZWEB_KEY, "")
return False
def _change_ip_addresses(constellation_name, partial_deploy,
credentials_softlayer, constellation_directory):
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= launch_sequence.index(
'change_ip'):
return
m = "Setting private IP address"
constellation.set_value('sim_launch_msg', m)
constellation.set_value('router_launch_msg', m)
router_ip = constellation.get_value("router_public_ip")
ssh_router = SshClient(constellation_directory, "key-router", 'ubuntu',
router_ip)
# change ip on sim
ssh_router.cmd("cloudsim/set_sim_ip.bash")
# change ip on router
ssh_router.cmd("nohup sudo bash change_ip.bash %s"
" > ssh_change_ip.out 2> ssh_change_ip.err < /dev/null &" %
ROUTER_IP)
if not partial_deploy:
constellation.set_value('fc1_launch_msg', m)
constellation.set_value('fc2_launch_msg', m)
ssh_router.cmd("cloudsim/set_fc1_ip.bash")
ssh_router.cmd("cloudsim/set_fc2_ip.bash")
constellation.set_value("launch_stage", "change_ip")
def monitor_simulator(constellation_name,
ssh_client,
sim_state_key='sim_state'):
"""
Detects if the simulator is running and writes the
result into the "gazebo" dictionary key
"""
if ssh_client is None:
#constellation.set_value("gazebo", "not running")
return False
constellation = ConstellationState(constellation_name)
simulation_state = constellation.get_value(sim_state_key)
if machine_states.index(simulation_state) >= \
machine_states.index('running'):
gl_state = constellation.get_value("sim_glx_state")
if gl_state == "running":
try:
out = ssh_client.cmd("bash cloudsim/ping_gazebo.bash")
#log("ping_gazebo returned [%s]" % out)
if out == "":
constellation.set_value("gazebo", "not running")
return False
except Exception, e:
log("monitor: cloudsim/ping_gazebo.bash error: %s" % e)
constellation.set_value("gazebo", "not running")
return False
def tearDown(self):
'''
Call the terminate function and
make sure that all resources (floating ip, security group, keypair)
are destroyed.
'''
terminate_openstack_server(self.constellation_name)
constellation = ConstellationState(self.constellation_name)
floating_ip = constellation.get_value("floating_ip")
pingable, ping_str = commands.getstatusoutput("ping -c3 %s" %
floating_ip)
self.assertNotEqual(pingable, 0)
creds = get_nova_creds()
nova = nvclient.Client(**creds)
instance = constellation.get_value("instance")
self.assertRaises(novaclient.exceptions.NotFound,
nova.servers.find,
name=instance)
security_group = constellation.get_value("security_group")
self.assertRaises(novaclient.exceptions.NotFound,
nova.security_groups.find,
name=security_group)
keypair = constellation.get_value("keypair")
self.assertRaises(novaclient.exceptions.NotFound,
nova.keypairs.find,
name=keypair)
def terminate_aws_server(constellation_name, credentials_fname):
log("terminate AWS CloudSim [constellation %s]" % (constellation_name))
constellation = ConstellationState(constellation_name)
ec2conn = None
machine_prefix = constellation.get_value('machine_name')
try:
constellation.set_value('%s_state' % machine_prefix, "terminating")
constellation.set_value('%s_launch_msg' % machine_prefix,
"terminating")
log("Terminate machine_prefix: %s" % machine_prefix)
aws_id = constellation.get_value('%s_aws_id' % machine_prefix)
log("Terminate aws_id: %s" % aws_id)
running_machines = {machine_prefix: aws_id}
ec2conn = aws_connect(credentials_fname)[0]
wait_for_multiple_machines_to_terminate(ec2conn, running_machines,
constellation, max_retries=150)
constellation.set_value('%s_state' % machine_prefix, "terminated")
print ('Waiting after killing instances...')
time.sleep(10.0)
except Exception as e:
log("error killing instances: %s" % e)
constellation.set_value('%s_launch_msg' % machine_prefix, "removing key")
_release_key_pair(constellation_name, machine_prefix, ec2conn)
constellation.set_value('%s_launch_msg' % machine_prefix,
"removing security group")
_release_security_group(constellation_name, machine_prefix, ec2conn)
constellation.set_value('%s_launch_msg' % machine_prefix, "terminated")
def terminate_openstack_server(constellation_name, creds):
'''
Retrieves necessary information from the redis database.
Uses that information to call terminate.
'''
constellation = ConstellationState(constellation_name)
secgroup = constellation.get_value("security_group")
keypair = constellation.get_value("keypair")
instance_name = constellation.get_value("instance")
terminate(instance_name, keypair, secgroup, creds)
def __wait_for_find_file(constellation_name, constellation_directory,
partial_deploy, ls_cmd, end_state):
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= 'running':
return
router_ip = constellation.get_value("router_public_ip")
ssh_router = SshClient(constellation_directory, "key-router", 'ubuntu',
router_ip)
q = []
#
# Wait until machines are online (rebooted?)
#
q.append(
get_ssh_cmd_generator(ssh_router,
"ls %s" % ls_cmd,
ls_cmd,
constellation,
"router_state",
"running",
max_retries=500))
q.append(
get_ssh_cmd_generator(ssh_router,
"cloudsim/find_file_sim.bash %s" % ls_cmd,
ls_cmd,
constellation,
"sim_state",
end_state,
max_retries=500))
if not partial_deploy:
q.append(
get_ssh_cmd_generator(ssh_router,
"cloudsim/find_file_fc1.bash %s" % ls_cmd,
ls_cmd,
constellation,
"fc1_state",
end_state,
max_retries=500))
q.append(
get_ssh_cmd_generator(ssh_router,
"cloudsim/find_file_fc2.bash %s" % ls_cmd,
ls_cmd,
constellation,
"fc2_state",
end_state,
max_retries=500))
empty_ssh_queue(q, sleep=2)
def _release_security_group(constellation_name, machine_prefix, ec2conn):
constellation = ConstellationState(constellation_name)
security_group_id = None
try:
sg_key = '%s_security_group_id' % machine_prefix
security_group_id = constellation.get_value(sg_key)
ec2conn.delete_security_group(group_id=security_group_id)
return True
except Exception, e:
error_msg = constellation.get_value('error')
error_msg += "<b>%s security group</b>: %s<br>" % (machine_prefix, e)
constellation.set_value('error', error_msg)
log("error cleaning up sim security group"
" %s: %s" % (security_group_id, e))
def _release_vpc_elastic_ip(constellation_name, machine_name_prefix, ec2conn):
constellation = ConstellationState(constellation_name)
try:
allocation_id_key = __get_allocation_id_key(machine_name_prefix)
eip_allocation_id = constellation.get_value(allocation_id_key)
log("_release_vpc_elastic_ip %s machine %s id: %s" % (
constellation_name,
machine_name_prefix,
eip_allocation_id))
ec2conn.release_address(allocation_id=eip_allocation_id)
except Exception, e:
error_msg = constellation.get_value('error')
error_msg += "<b>Router IP address</b>: %s<br>" % e
constellation.set_value('error', error_msg)
print("error cleaning up %s elastic ip: %s" % (machine_name_prefix, e))
def terminate_softlayer_constellation(constellation_name, constellation_prefix,
partial_reload, osrf_creds_fname):
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= \
launch_sequence.index('os_reload'):
return
else:
osrf_creds = load_osrf_creds(osrf_creds_fname)
# compute the softlayer machine names
constellation.set_value('sim_launch_msg', 'reload OS')
machine_names = [
x + "-" + constellation_prefix for x in ('router', 'sim')
]
if not partial_reload:
machine_names.append('fc2-' + constellation_prefix)
machine_names.append('fc1-' + constellation_prefix)
constellation.set_value('fc1_launch_msg', 'reload OS')
constellation.set_value('fc2_launch_msg', 'reload OS')
# enable nics on machines with disconnected ones (not the router)
enable_public_ips(osrf_creds, machine_names[1:])
for server in machine_names[1:]:
t = get_active_transaction(osrf_creds, server)
log("Transaction before reload on %s: %s" % (server, t))
reload_servers(osrf_creds, machine_names)
constellation.set_value("launch_stage", "os_reload")
def _acquire_vpc_server(constellation_name,
machine_prefix,
key_pair_name,
machine_data,
startup_script,
subnet_id,
security_group_id,
ec2conn):
constellation = ConstellationState(constellation_name)
try:
constellation.set_value('%s_launch_msg' % machine_prefix, "booting")
aws_image = machine_data['software']
aws_instance = machine_data['hardware']
ip = machine_data['ip']
bdm = __get_block_device_mapping(aws_instance)
constellation_directory = constellation.get_value(
'constellation_directory')
# save local startup script copy
script_fname = os.path.join(constellation_directory,
"%s_startup_script.txt" % machine_prefix)
with open(script_fname, 'w') as f:
f.write(startup_script)
res = ec2conn.run_instances(aws_image,
instance_type=aws_instance,
subnet_id=subnet_id,
private_ip_address=ip,
security_group_ids=[security_group_id],
key_name=key_pair_name,
user_data=startup_script,
block_device_map=bdm)
return res.id
except:
raise
def _shutdown_constellation_public_ips(constellation_name,
constellation_prefix, partial_deploy,
credentials_softlayer,
constellation_directory):
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= launch_sequence.index(
"block_public_ips"):
return
__wait_for_find_file(constellation_name, constellation_directory,
partial_deploy, "cloudsim/setup/done", "running")
m = "Switching off public network interfaces"
constellation.set_value('sim_launch_msg', m)
#constellation.set_value('router_launch_msg', m)
private_machines = ["sim-%s" % constellation_prefix]
if not partial_deploy:
private_machines.append("fc1-%s" % constellation_prefix)
private_machines.append("fc2-%s" % constellation_prefix)
constellation.set_value('fc1_launch_msg', m)
constellation.set_value('fc2_launch_msg', m)
osrf_creds = load_osrf_creds(credentials_softlayer)
shutdown_public_ips(osrf_creds, private_machines)
constellation.set_value("launch_stage", "block_public_ips")
def _monitor_ping(constellation_name, ping_data_key, ping_str):
"""
internal implementation for monitor_cloudsim_ping and monitor_ssh_ping
"""
constellation = ConstellationState(constellation_name)
latency = constellation.get_value(ping_data_key)
latency = record_ping_result(latency, ping_str, LATENCY_TIME_BUFFER)
constellation.set_value(ping_data_key, latency)
def get_ssh_client(constellation_name, machine_state, ip_key, sshkey_key):
"""
Checks to see if machine is ready and creates an ssh client accordingly
"""
ssh_client = None
constellation = ConstellationState(constellation_name)
if machine_states.index(machine_state) >= \
machine_states.index('packages_setup'):
constellation_directory = \
constellation.get_value('constellation_directory')
machine_ip = constellation.get_value(ip_key)
key_pair_name = constellation.get_value(sshkey_key)
ssh_client = SshClient(constellation_directory,
key_pair_name,
'ubuntu',
machine_ip)
return ssh_client
def _release_key_pair(constellation_name, machine_prefix, ec2conn):
constellation = ConstellationState(constellation_name)
key_pair_name = None
try:
key_pair_name = 'key-%s-%s' % (machine_prefix, constellation_name)
ec2conn.delete_key_pair(key_pair_name)
except Exception, e:
error_msg = constellation.get_value('error')
error_msg += "<b>Release key</b>: %s<br>" % e
constellation.set_value('error', error_msg)
log("error cleaning up simulation key %s: %s" % (key_pair_name, e))
def monitor_launch_state(constellation_name, ssh_client,
machine_state,
dpkg_cmd,
launch_msg_key):
if ssh_client is None: # too early to verify
return
try:
constellation = ConstellationState(constellation_name)
constellation_state = constellation.get_value("constellation_state")
if constellation_states.index(constellation_state) >= \
constellation_states.index("launching"):
if machine_state == 'packages_setup':
dpkg_line = ssh_client.cmd(dpkg_cmd)
package_msg = parse_dpkg_line(dpkg_line)
current_value = constellation.get_value(launch_msg_key)
if current_value != package_msg:
constellation.set_value(launch_msg_key, package_msg)
except:
tb = traceback.format_exc()
log("monitor_launch_state traceback: %s" % tb)
def _startup_scripts(constellation_name, partial_deploy):
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= launch_sequence.index('startup'):
return
constellation_directory = constellation.get_value(
'constellation_directory')
# if the change of ip was successful, the script should be in the home
# directory of each machine
__wait_for_find_file(constellation_name, constellation_directory,
partial_deploy, "change_ip.bash", "packages_setup")
m = "Executing startup script"
constellation.set_value('sim_launch_msg', m)
constellation.set_value('router_launch_msg', m)
router_ip = constellation.get_value("router_public_ip")
ssh_router = SshClient(constellation_directory, "key-router", 'ubuntu',
router_ip)
# load packages onto router
ssh_router.cmd("nohup sudo bash cloudsim/router_startup_script.bash "
"> ssh_startup.out 2> ssh_startup.err < /dev/null &")
# load packages onto simulator
ssh_router.cmd("cloudsim/sim_init.bash")
if not partial_deploy:
constellation.set_value('fc1_launch_msg', m)
constellation.set_value('fc2_launch_msg', m)
# load packages onto fc1
ssh_router.cmd("cloudsim/fc1_init.bash")
# load packages onto fc2
ssh_router.cmd("cloudsim/fc2_init.bash")
constellation.set_value("fc1_state", "packages_setup")
constellation.set_value("fc2_state", "packages_setup")
constellation.set_value("sim_state", "packages_setup")
constellation.set_value("router_state", "packages_setup")
constellation.set_value("launch_stage", "startup")
def monitor_cloudsim_ping(constellation_name, ip_address_key, ping_data_key):
"""
Finds the ip of the machine to pind in redis,
pings the machine and integrates
the results with the existing data.
The ping is done from CloudSim
"""
constellation = ConstellationState(constellation_name)
if constellation.has_value(ip_address_key):
ip_address = constellation.get_value(ip_address_key)
o, ping_str = commands.getstatusoutput("ping -c3 %s" % ip_address)
if o == 0:
_monitor_ping(constellation_name, ping_data_key, ping_str)
def acquire_dedicated_sl_server(constellation_name, osrf_creds_fname,
constellation_directory):
"""
Acquire a dedicated SoftLayer machine
"""
constellation = ConstellationState(constellation_name)
constellation_prefix = constellation_name.split("OSRF_CloudSim_")[1]
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= launch_sequence.index('init'):
return
if os.path.exists(constellation_directory):
shutil.rmtree(constellation_directory)
os.makedirs(constellation_directory)
machines_dict = {'cs-%s' % constellation_prefix: 'simulation_launch_msg'}
osrf_creds = load_osrf_creds(osrf_creds_fname)
reload_monitor = ReloadOsCallBack(constellation_name, machines_dict)
# wait
wait_for_server_reloads(osrf_creds, machines_dict.keys(),
reload_monitor.callback)
constellation.set_value('simulation_aws_state', 'running')
constellation.set_value('simulation_state', 'packages_setup')
name = "cs-%s" % constellation_prefix
pub_ip, priv_ip, password = get_machine_login_info(osrf_creds, name)
log("ubuntu user setup for machine cs %s [%s / %s] " %
(name, pub_ip, priv_ip))
# dst_dir = os.path.abspath('.')
log("machine details cs %s %s : %s" % (name, pub_ip, password))
# __add_ubuntu_user_to_router(pub_ip, password, constellation_directory,
# 'key-cs')
key_prefix = 'key-cs'
clean_local_ssh_key_entry(pub_ip)
create_ssh_key(key_prefix, constellation_directory)
# setup a ubuntu sudoer no password user with an ssh key
pub_key_path = os.path.join(constellation_directory,
"%s.pem.pub" % key_prefix)
setup_ssh_key_access(pub_ip, password, pub_key_path)
priv_key_path = os.path.join(constellation_directory,
"%s.pem" % key_prefix)
log("ssh -i %s ubuntu@%s" % (priv_key_path, pub_ip))
constellation.set_value("launch_stage", "init")
return pub_ip, pub_key_path, priv_key_path
def terminate_aws_constellation(constellation_name, credentials_ec2):
"""
Releases a private network, machines and all its resources
"""
boto.config = BotoConfig(credentials_ec2)
ec2conn, vpcconn = aws_connect()
constellation = ConstellationState(constellation_name)
machines = constellation.get_value('machines')
log("machines: %s" % machines.keys())
running_machines = {}
for machine_prefix in machines.keys():
try:
aws_id_key = '%s_aws_id' % machine_prefix
aws_id = constellation.get_value(aws_id_key)
log("%s aws id: %s" % (machine_prefix, aws_id))
running_machines[machine_prefix] = aws_id
m = "terminate machine instance"
constellation.set_value("%s_launch_msg" % machine_prefix, m)
except Exception, e:
error_msg = constellation.get_value('error')
error_msg += " get aws id error %s" % e
constellation.set_value('error', error_msg)
def constellation_is_terminated(constellation_name):
constellation = None
expire = True
try:
constellation = ConstellationState(constellation_name)
constellation_state = constellation.get_value("constellation_state")
expire = constellation_state == "terminated"
except:
log("Can't access constellation %s data" % constellation_name)
if expire:
try:
constellation.expire(1)
except:
pass
return expire
def _acquire_key_pair(constellation_name, machine_prefix, ec2conn):
constellation = ConstellationState(constellation_name)
try:
constellation_directory = constellation.get_value(
'constellation_directory')
key_pair_name = 'key-%s-%s' % (machine_prefix, constellation_name)
key_pair = ec2conn.create_key_pair(key_pair_name)
key_pair.save(constellation_directory)
src = os.path.join(constellation_directory, '%s.pem' % key_pair_name)
dst = os.path.join(constellation_directory,
'key-%s.pem' % machine_prefix)
shutil.copy(src, dst)
return key_pair_name
except Exception, e:
constellation.set_value('error', "key error: %s" % e)
raise
def terminate_dedicated_sl_server(constellation_name, machine_name,
osrf_creds_fname):
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= launch_sequence.index(
'os_reload'):
return
osrf_creds = load_osrf_creds(osrf_creds_fname)
# compute the softlayer machine names
machine_names = [machine_name]
pub_ip, priv_ip, password = get_machine_login_info(osrf_creds,
machine_names[0])
log("reload os for machine %s [%s / %s] password %s " %
(machine_names[0], pub_ip, priv_ip, password))
reload_servers(osrf_creds, machine_names)
constellation.set_value("launch_stage", "os_reload")
def _release_vpc(constellation_name, vpcconn):
constellation = ConstellationState(constellation_name)
error_msg = constellation.get_value('error')
original_error = error_msg
vpc_id = None
subnet_id = None
igw_id = None
route_table_association_id = None
route_table_id = None
try:
vpc_id = constellation.get_value('vpc_id')
subnet_id = constellation.get_value('subnet_id')
igw_id = constellation.get_value('igw_id')
route_table_id = constellation.get_value('route_table_id')
route_table_association_id = constellation.get_value(
'route_table_association_id')
except Exception, e:
error_msg += "%s" % e
log("missing db key %s" % e)
def _notify_portal(constellation, task):
try:
root_log_dir = '/tmp/cloudsim_logs'
# Get metadata (team, competition, ...)
config = get_cloudsim_config()
portal_info_fname = config['cloudsim_portal_json_path']
log("** Portal JSON path: %s ***" % portal_info_fname)
portal_info = None
with open(portal_info_fname, 'r') as f:
portal_info = json.loads(f.read())
log("** Portal JSON file opened ***")
team = portal_info['team']
comp = portal_info['event']
task_num = task['vrc_num']
log("** Team: %s, Event: %s ***" % (team, comp))
if task_num < '1' or task_num > '3':
task_num = '1'
run = task['vrc_id']
if run < '1' or run > '5':
run = '1'
start_time = task['start_time']
start_task = dateutil.parser.parse(start_time)
start_task = start_task.strftime("%d/%m/%y %H:%M:%S")
const = ConstellationState(constellation)
constellation_dict = get_constellation_data(constellation)
constellation_directory = constellation_dict['constellation_directory']
router_ip = const.get_value("router_public_ip")
task_id = task['ros_launch']
task_dirname = task_id.split('.')[0]
# Store in this cloudsim the network and sim logs
router_key = os.path.join(constellation_directory, 'key-router.pem')
new_msg = task['task_message'] + '<B> Getting logs</B>'
const.update_task_value(task['task_id'], 'task_message', new_msg)
cmd = ('bash /var/www/bin/get_logs.bash %s %s %s'
% (task_dirname, router_ip, router_key))
subprocess.check_call(cmd.split())
log("** Log directory created***")
# Get the score and falls
score = '0'
#falls = 'N/A'
runtime = 'N/A'
try:
p = os.path.join(root_log_dir, task_dirname, 'score.log')
with open(p) as f:
log("** score.log found **")
data = f.read()
log("** Reading score.log file **")
lines = data.split('\n')
last_line = lines[-2]
log("** Last line: %s **" % last_line)
score = last_line.split(',')[4]
#falls = last_line.split(',')[5]
# Time when the task stopped
runtime = last_line.split(',')[1]
log("** All sim score fields parsed **")
except Exception:
None
# Create JSON file with the task metadata
data = json.dumps({'team': team, 'event': comp, 'task': task_num,
'start_time': start_task, 'result': 'Terminated',
'runtime': runtime, 'score': score},
sort_keys=True, indent=4, separators=(',', ': '))
log("** JSON data created **")
with open(os.path.join(root_log_dir, task_dirname,
'end_task.json'), 'w') as f:
f.write(str(data))
log("** JSON file created ***")
new_msg = new_msg.replace('Getting logs', 'Creating tar file')
const.update_task_value(task['task_id'], 'task_message', new_msg)
# Tar all the log content
tar_name = (team + '_' + comp + '_' + str(task_num) + '_' + str(run) +
'.tar')
p = os.path.join(root_log_dir, task_dirname)
cmd = 'tar cf /tmp/' + tar_name + ' -C ' + p + ' .'
subprocess.check_call(cmd.split())
log("** Log directory stored in a tar file ***")
new_msg = new_msg.replace('Creating tar file',
'Uploading logs to the portal')
const.update_task_value(task['task_id'], 'task_message', new_msg)
# Send the log to the portal
config = get_cloudsim_config()
portal_info_fname = config['cloudsim_portal_json_path']
portal_info = None
with open(portal_info_fname, 'r') as f:
portal_info = json.loads(f.read())
ssh_portal = SshClient('xxx', 'xxx', portal_info['user'],
portal_info['hostname'])
# this is a hack
ssh_portal.key_fname = config['cloudsim_portal_key_path']
# Upload the file to the Portal temp dir
dest = os.path.join('/tmp', tar_name)
cmd = ('scp -o UserKnownHostsFile=/dev/null'
'-o StrictHostKeyChecking=no'
' -i ' + ssh_portal.key_fname + ' ' + dest + ' ubuntu@' +
portal_info['hostname'] + ':/tmp')
log('cmd: %s' % cmd)
subprocess.check_call(cmd.split())
# Move the file to the final destination into the Portal
final_dest = os.path.join(portal_info['final_destination_dir'],
tar_name)
cmd = 'sudo mv %s %s' % (dest, final_dest)
ssh_portal.cmd(cmd)
new_msg = new_msg.replace('Uploading logs to the portal',
'Logs uploaded to the portal')
const.update_task_value(task['task_id'], 'task_message', new_msg)
except Exception, excep:
log('notify_portal() Exception: %s' % (repr(excep)))
raise
def monitor_task(constellation_name, ssh_router):
"""
Read score and net usage data.
Aborts tasks that timeout
"""
def parse_score_data(score_str):
toks = score_str.split()
keys = [x[6:] for x in toks[-2].split(',')]
keys[0] = 'time'
# keys = ['time', 'wall_time', 'sim_time', 'wall_time_elapsed',
# 'sim_time_elapsed', 'completion_score',
# 'falls', 'message', 'task_type']
values_str = toks[-1]
values = values_str.split(',')
d = dict(zip(keys, values))
score = d['completion_score']
sim_time = float(d['sim_time']) / 1e9
fall_count = d['falls']
msg = d['message']
return (score, sim_time, fall_count, msg)
#log("monitor_task BEGIN")
constellation = ConstellationState(constellation_name)
task_id = constellation.get_value("current_task")
if task_id != "":
sim_time = 0
timeout = 0
task = constellation.get_task(task_id)
timeout = int(task['timeout'])
score_str = None
try:
s = ssh_router.cmd("cloudsim/get_score.bash")
log(s)
score, sim_time, fall_count, msg = parse_score_data(s)
score_str = ""
#score_str += "<b>%s</b>: %s. " % ("score", score)
score_str += "<b>%s</b>: %s. " % ("sim time", sim_time)
score_str += " %s" % (msg)
score_str += "<b>falls:</b> %s." % fall_count
except Exception, e:
#score_str = "No score available."
tb = traceback.format_exc()
log("traceback: %s" % tb)
log("score %s" % score_str)
net_str = None
try:
n = ssh_router.cmd("cloudsim/get_network_usage.bash")
log(n)
toks = n.split()
up_bits = int(toks[2]) * 8
down_bits = int(toks[3]) * 8
up_cap = int(task['uplink_data_cap'])
down_cap = int(task['downlink_data_cap'])
up = 0.0
down = 0.0
if up_cap != 0.0:
up = 100.0 * up_bits / up_cap
if down_cap != 0.0:
down = 100.0 * down_bits / down_cap
net_str = "<b>up/down link (%%)</b>: %0.2f / %0.2f" % (up, down)
except Exception, e:
# net_str = "no network usage available"
log("score monitoring error %s" % e)
tb = traceback.format_exc()
log("traceback: %s" % tb)
def acquire_aws_single_server(constellation_name,
credentials_ec2,
constellation_directory,
machine_prefix, # name of machine, ie "sim"
machine_data,
startup_script,
tags):
sim_machine_name = "%s_%s" % (machine_prefix, constellation_name)
ec2conn, _ = aws_connect(credentials_ec2)
aws_image = machine_data['software']
aws_instance = machine_data['hardware']
bdm = __get_block_device_mapping(aws_instance)
constellation = ConstellationState(constellation_name)
constellation.set_value('%s_launch_msg' % machine_prefix,
"setting up security groups")
constellation_directory = constellation.get_value(
'constellation_directory')
# save local startup script copy
script_fname = os.path.join(constellation_directory,
"%s_startup_script.txt" % machine_prefix)
with open(script_fname, 'w') as f:
f.write(startup_script)
constellation.set_value('machine_name', machine_prefix)
security_group_data = machine_data['security_group']
security_group_name, _ = _acquire_security_group(
constellation_name,
machine_prefix,
security_group_data,
vpc_id=None,
ec2conn=ec2conn)
key_pair_name = _acquire_key_pair(constellation_name,
machine_prefix,
ec2conn)
roles_to_reservations = {}
try:
constellation.set_value('%s_launch_msg' % machine_prefix,
"requesting machine")
res = ec2conn.run_instances(image_id=aws_image,
instance_type=aws_instance,
#subnet_id = subnet_id,
#private_ip_address=SIM_IP,
security_groups=[security_group_name],
key_name=key_pair_name,
user_data=startup_script,
block_device_map=bdm)
roles_to_reservations['simulation_state'] = res.id
except:
log("ouch!")
raise
aws_id = None
count = 200
done = False
while not done:
log("attempt %s" % count)
time.sleep(2)
count -= 1
for r in ec2conn.get_all_instances():
if count < 0:
msg = ("timeout while waiting "
"for EC2 machine(s) %s" % sim_machine_name)
raise LaunchException(msg)
if r.id == res.id:
state = r.instances[0].state
if state == 'running':
aws_id = r.instances[0].id
constellation.set_value('%s_state' % machine_prefix,
'network_setup')
done = True
constellation.set_value('%s_aws_state' % machine_prefix, state)
constellation.set_value('%s_launch_msg' % machine_prefix,
"machine running")
constellation.set_value('%s_aws_id' % machine_prefix, aws_id)
sim_tags = {'Name': sim_machine_name}
sim_tags.update(tags)
ec2conn.create_tags([aws_id], sim_tags)
# ec2conn.associate_address(router_aws_id, allocation_id=eip_allocation_id)
instance = get_ec2_instance(ec2conn, aws_id)
ip = instance.ip_address
clean_local_ssh_key_entry(ip)
constellation.set_value('%s_public_ip' % (machine_prefix), ip)
return ip, aws_id, key_pair_name
def _initialize_private_machines(constellation_name, constellation_prefix,
partial_deploy, credentials_softlayer,
router_script, sim_script, fc1_script,
fc2_script, constellation_directory):
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= \
launch_sequence.index('init_privates'):
return
#
# router machine
#
router_ip = constellation.get_value("router_public_ip")
ssh_router = SshClient(constellation_directory, "key-router", 'ubuntu',
router_ip)
local_fname = os.path.join(constellation_directory, 'router_startup.bash')
with open(local_fname, 'w') as f:
f.write(router_script)
remote_fname = 'cloudsim/router_startup_script.bash'
ssh_router.upload_file(local_fname, remote_fname)
log("upload %s to %s" % (local_fname, remote_fname))
osrf_creds = load_osrf_creds(credentials_softlayer)
#
# sim machine
#
sim_pub_ip, sim_priv_ip, sim_root_password = get_machine_login_info(
osrf_creds, "sim-%s" % constellation_prefix)
log("provision sim [%s / %s] %s" %
(sim_pub_ip, sim_priv_ip, sim_root_password))
_provision_ssh_private_machine(constellation_name, ssh_router, "sim",
sim_priv_ip, sim_root_password, sim_script,
constellation_directory)
if not partial_deploy:
#
# fc1 machine
#
fc1_pub_ip, fc1_priv_ip, fc1_root_password = get_machine_login_info(
osrf_creds, "fc1-%s" % constellation_prefix)
log("provision fc1 [%s / %s] %s" %
(fc1_pub_ip, fc1_priv_ip, fc1_root_password))
_provision_ssh_private_machine(constellation_name, ssh_router, "fc1",
fc1_priv_ip, fc1_root_password,
fc1_script, constellation_directory)
#
# fc2 machine
#
fc2_pub_ip, fc2_priv_ip, fc2_root_password = get_machine_login_info(
osrf_creds, "fc2-%s" % constellation_prefix)
log("provision fc2 [%s / %s] %s" %
(fc2_pub_ip, fc2_priv_ip, fc2_root_password))
_provision_ssh_private_machine(constellation_name, ssh_router, "fc2",
fc2_priv_ip, fc2_root_password,
fc2_script, constellation_directory)
log('configure_machines done')
constellation.set_value("launch_stage", "init_privates")
def _initialize_router(constellation_name, constellation_prefix,
partial_deploy, osrf_creds_fname,
constellation_directory):
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= launch_sequence.index(
'init_router'):
return
osrf_creds = load_osrf_creds(osrf_creds_fname)
router_name = "router-%s" % constellation_prefix
m = "running"
constellation.set_value('fc1_aws_state', m)
constellation.set_value('fc2_aws_state', m)
constellation.set_value('sim_aws_state', m)
constellation.set_value('router_aws_state', m)
constellation.set_value('router_launch_msg', 'user account setup')
router_ip, priv_ip, password = get_machine_login_info(
osrf_creds, router_name)
constellation.set_value("router_public_ip", router_ip)
log("router %s %s" % (router_ip, password))
sim_pub_ip, sim_priv_ip, sim_root_password = get_machine_login_info(
osrf_creds, "sim-%s" % constellation_prefix)
log("sim %s %s" % (sim_pub_ip, sim_root_password))
log("ubuntu user setup for machine router %s [%s / %s] " %
(router_name, router_ip, priv_ip))
log("router %s %s : %s" % (router_name, router_ip, password))
__add_ubuntu_user_to_router(router_ip, password, constellation_directory)
softlayer_scripts_dir = os.path.join(os.path.dirname(softlayer.__file__),
'bash')
ssh_router = SshClient(constellation_directory, "key-router", 'ubuntu',
router_ip)
# create a remote cloudsim directory on the router
ssh_router.cmd("mkdir -p cloudsim")
openvpn_fname = os.path.join(constellation_directory, 'openvpn.key')
create_openvpn_key(openvpn_fname)
remote_fname = 'cloudsim/openvpn.key'
ssh_router.upload_file(openvpn_fname, remote_fname)
local_fname = os.path.join(softlayer_scripts_dir, 'router_init.bash')
remote_fname = 'cloudsim/router_init.bash'
ssh_router.upload_file(local_fname, remote_fname)
log("upload %s to %s" % (local_fname, remote_fname))
# upload ubuntu user setup scripts
local_fname = os.path.join(softlayer_scripts_dir, 'auto_ubuntu.bash')
remote_fname = 'cloudsim/auto_ubuntu.bash'
ssh_router.upload_file(local_fname, remote_fname)
log("upload %s to %s" % (local_fname, remote_fname))
local_fname = os.path.join(softlayer_scripts_dir, 'create_ubuntu_user.exp')
remote_fname = 'cloudsim/create_ubuntu_user.exp'
ssh_router.upload_file(local_fname, remote_fname)
log("upload %s to %s" % (local_fname, remote_fname))
local_fname = os.path.join(softlayer_scripts_dir, 'upload_key.exp')
remote_fname = 'cloudsim/upload_key.exp'
ssh_router.upload_file(local_fname, remote_fname)
log("upload %s to %s" % (local_fname, remote_fname))
local_fname = os.path.join(softlayer_scripts_dir,
'process_remote_ssh_key.exp')
remote_fname = 'cloudsim/process_remote_ssh_key.exp'
ssh_router.upload_file(local_fname, remote_fname)
log("upload %s to %s" % (local_fname, remote_fname))
# avoid ssh error because our server has changed
constellation.set_value("launch_stage", "init_router")
create_ssh_key("key-sim", constellation_directory)
fc1_priv_ip = "0.0.0.0"
fc2_priv_ip = "0.0.0.0"
if not partial_deploy:
fc1_pub_ip, fc1_priv_ip, fc1_root_password = get_machine_login_info(
osrf_creds, "fc1-%s" % constellation_prefix)
log("fc1 %s %s" % (fc1_pub_ip, fc1_root_password))
create_ssh_key("key-fc1", constellation_directory)
fc2_pub_ip, fc2_priv_ip, fc2_root_password = get_machine_login_info(
osrf_creds, "fc2-%s" % constellation_prefix)
log("fc2 %s %s" % (fc2_pub_ip, fc2_root_password))
create_ssh_key("key-fc2", constellation_directory)
arg_str = "%s %s %s" % (sim_priv_ip, fc1_priv_ip, fc2_priv_ip)
cmd = "cd cloudsim; ./router_init.bash /home/ubuntu/cloudsim %s" % arg_str
# run the script
constellation.set_value('router_launch_msg', 'generating bash scripts')
ssh_router.cmd(cmd)
constellation.set_value('router_launch_msg', 'uploading ssh key')
__upload_ssh_keys_to_router(ssh_router, "router", constellation_directory)
constellation.set_value('sim_launch_msg', 'uploading ssh key')
__upload_ssh_keys_to_router(ssh_router, "sim", constellation_directory)
constellation.set_value('fc1_launch_msg', 'uploading ssh key')
__upload_ssh_keys_to_router(ssh_router, "fc1", constellation_directory)
constellation.set_value('fc2_launch_msg', 'uploading ssh key')
__upload_ssh_keys_to_router(ssh_router, "fc2", constellation_directory)