def monitor_gzweb(constellation_name, ssh_client, sim_state): """ Detects if the gzweb is running and writes the url into the "gzweb" dictionary key """ constellation = ConstellationState(constellation_name) simulation_state = constellation.get_value('sim_state') if machine_states.index(simulation_state) >= \ machine_states.index('running'): gl_state = constellation.get_value("gazebo") if gl_state == "running": try: # current_state = constellation.get_value(GZWEB_KEY) out = ssh_client.cmd("bash cloudsim/ping_gzweb.bash") log("ping_gzweb returned [%s]" % out) if out == "": constellation.set_value(GZWEB_KEY, "not running") return False else: constellation.set_value(GZWEB_KEY, "running") return True except Exception, e: log("monitor: cloudsim/ping_gzweb.bash error: %s" % e) constellation.set_value(GZWEB_KEY, "") return False
def _change_ip_addresses(constellation_name, partial_deploy, credentials_softlayer, constellation_directory): constellation = ConstellationState(constellation_name) launch_stage = constellation.get_value("launch_stage") if launch_sequence.index(launch_stage) >= launch_sequence.index( 'change_ip'): return m = "Setting private IP address" constellation.set_value('sim_launch_msg', m) constellation.set_value('router_launch_msg', m) router_ip = constellation.get_value("router_public_ip") ssh_router = SshClient(constellation_directory, "key-router", 'ubuntu', router_ip) # change ip on sim ssh_router.cmd("cloudsim/set_sim_ip.bash") # change ip on router ssh_router.cmd("nohup sudo bash change_ip.bash %s" " > ssh_change_ip.out 2> ssh_change_ip.err < /dev/null &" % ROUTER_IP) if not partial_deploy: constellation.set_value('fc1_launch_msg', m) constellation.set_value('fc2_launch_msg', m) ssh_router.cmd("cloudsim/set_fc1_ip.bash") ssh_router.cmd("cloudsim/set_fc2_ip.bash") constellation.set_value("launch_stage", "change_ip")
def monitor_simulator(constellation_name, ssh_client, sim_state_key='sim_state'): """ Detects if the simulator is running and writes the result into the "gazebo" dictionary key """ if ssh_client is None: #constellation.set_value("gazebo", "not running") return False constellation = ConstellationState(constellation_name) simulation_state = constellation.get_value(sim_state_key) if machine_states.index(simulation_state) >= \ machine_states.index('running'): gl_state = constellation.get_value("sim_glx_state") if gl_state == "running": try: out = ssh_client.cmd("bash cloudsim/ping_gazebo.bash") #log("ping_gazebo returned [%s]" % out) if out == "": constellation.set_value("gazebo", "not running") return False except Exception, e: log("monitor: cloudsim/ping_gazebo.bash error: %s" % e) constellation.set_value("gazebo", "not running") return False
def tearDown(self): ''' Call the terminate function and make sure that all resources (floating ip, security group, keypair) are destroyed. ''' terminate_openstack_server(self.constellation_name) constellation = ConstellationState(self.constellation_name) floating_ip = constellation.get_value("floating_ip") pingable, ping_str = commands.getstatusoutput("ping -c3 %s" % floating_ip) self.assertNotEqual(pingable, 0) creds = get_nova_creds() nova = nvclient.Client(**creds) instance = constellation.get_value("instance") self.assertRaises(novaclient.exceptions.NotFound, nova.servers.find, name=instance) security_group = constellation.get_value("security_group") self.assertRaises(novaclient.exceptions.NotFound, nova.security_groups.find, name=security_group) keypair = constellation.get_value("keypair") self.assertRaises(novaclient.exceptions.NotFound, nova.keypairs.find, name=keypair)
def terminate_aws_server(constellation_name, credentials_fname): log("terminate AWS CloudSim [constellation %s]" % (constellation_name)) constellation = ConstellationState(constellation_name) ec2conn = None machine_prefix = constellation.get_value('machine_name') try: constellation.set_value('%s_state' % machine_prefix, "terminating") constellation.set_value('%s_launch_msg' % machine_prefix, "terminating") log("Terminate machine_prefix: %s" % machine_prefix) aws_id = constellation.get_value('%s_aws_id' % machine_prefix) log("Terminate aws_id: %s" % aws_id) running_machines = {machine_prefix: aws_id} ec2conn = aws_connect(credentials_fname)[0] wait_for_multiple_machines_to_terminate(ec2conn, running_machines, constellation, max_retries=150) constellation.set_value('%s_state' % machine_prefix, "terminated") print ('Waiting after killing instances...') time.sleep(10.0) except Exception as e: log("error killing instances: %s" % e) constellation.set_value('%s_launch_msg' % machine_prefix, "removing key") _release_key_pair(constellation_name, machine_prefix, ec2conn) constellation.set_value('%s_launch_msg' % machine_prefix, "removing security group") _release_security_group(constellation_name, machine_prefix, ec2conn) constellation.set_value('%s_launch_msg' % machine_prefix, "terminated")
def terminate_openstack_server(constellation_name, creds): ''' Retrieves necessary information from the redis database. Uses that information to call terminate. ''' constellation = ConstellationState(constellation_name) secgroup = constellation.get_value("security_group") keypair = constellation.get_value("keypair") instance_name = constellation.get_value("instance") terminate(instance_name, keypair, secgroup, creds)
def __wait_for_find_file(constellation_name, constellation_directory, partial_deploy, ls_cmd, end_state): constellation = ConstellationState(constellation_name) launch_stage = constellation.get_value("launch_stage") if launch_sequence.index(launch_stage) >= 'running': return router_ip = constellation.get_value("router_public_ip") ssh_router = SshClient(constellation_directory, "key-router", 'ubuntu', router_ip) q = [] # # Wait until machines are online (rebooted?) # q.append( get_ssh_cmd_generator(ssh_router, "ls %s" % ls_cmd, ls_cmd, constellation, "router_state", "running", max_retries=500)) q.append( get_ssh_cmd_generator(ssh_router, "cloudsim/find_file_sim.bash %s" % ls_cmd, ls_cmd, constellation, "sim_state", end_state, max_retries=500)) if not partial_deploy: q.append( get_ssh_cmd_generator(ssh_router, "cloudsim/find_file_fc1.bash %s" % ls_cmd, ls_cmd, constellation, "fc1_state", end_state, max_retries=500)) q.append( get_ssh_cmd_generator(ssh_router, "cloudsim/find_file_fc2.bash %s" % ls_cmd, ls_cmd, constellation, "fc2_state", end_state, max_retries=500)) empty_ssh_queue(q, sleep=2)
def _release_security_group(constellation_name, machine_prefix, ec2conn): constellation = ConstellationState(constellation_name) security_group_id = None try: sg_key = '%s_security_group_id' % machine_prefix security_group_id = constellation.get_value(sg_key) ec2conn.delete_security_group(group_id=security_group_id) return True except Exception, e: error_msg = constellation.get_value('error') error_msg += "<b>%s security group</b>: %s<br>" % (machine_prefix, e) constellation.set_value('error', error_msg) log("error cleaning up sim security group" " %s: %s" % (security_group_id, e))
def _release_vpc_elastic_ip(constellation_name, machine_name_prefix, ec2conn): constellation = ConstellationState(constellation_name) try: allocation_id_key = __get_allocation_id_key(machine_name_prefix) eip_allocation_id = constellation.get_value(allocation_id_key) log("_release_vpc_elastic_ip %s machine %s id: %s" % ( constellation_name, machine_name_prefix, eip_allocation_id)) ec2conn.release_address(allocation_id=eip_allocation_id) except Exception, e: error_msg = constellation.get_value('error') error_msg += "<b>Router IP address</b>: %s<br>" % e constellation.set_value('error', error_msg) print("error cleaning up %s elastic ip: %s" % (machine_name_prefix, e))
def terminate_softlayer_constellation(constellation_name, constellation_prefix, partial_reload, osrf_creds_fname): constellation = ConstellationState(constellation_name) launch_stage = constellation.get_value("launch_stage") if launch_sequence.index(launch_stage) >= \ launch_sequence.index('os_reload'): return else: osrf_creds = load_osrf_creds(osrf_creds_fname) # compute the softlayer machine names constellation.set_value('sim_launch_msg', 'reload OS') machine_names = [ x + "-" + constellation_prefix for x in ('router', 'sim') ] if not partial_reload: machine_names.append('fc2-' + constellation_prefix) machine_names.append('fc1-' + constellation_prefix) constellation.set_value('fc1_launch_msg', 'reload OS') constellation.set_value('fc2_launch_msg', 'reload OS') # enable nics on machines with disconnected ones (not the router) enable_public_ips(osrf_creds, machine_names[1:]) for server in machine_names[1:]: t = get_active_transaction(osrf_creds, server) log("Transaction before reload on %s: %s" % (server, t)) reload_servers(osrf_creds, machine_names) constellation.set_value("launch_stage", "os_reload")
def _acquire_vpc_server(constellation_name, machine_prefix, key_pair_name, machine_data, startup_script, subnet_id, security_group_id, ec2conn): constellation = ConstellationState(constellation_name) try: constellation.set_value('%s_launch_msg' % machine_prefix, "booting") aws_image = machine_data['software'] aws_instance = machine_data['hardware'] ip = machine_data['ip'] bdm = __get_block_device_mapping(aws_instance) constellation_directory = constellation.get_value( 'constellation_directory') # save local startup script copy script_fname = os.path.join(constellation_directory, "%s_startup_script.txt" % machine_prefix) with open(script_fname, 'w') as f: f.write(startup_script) res = ec2conn.run_instances(aws_image, instance_type=aws_instance, subnet_id=subnet_id, private_ip_address=ip, security_group_ids=[security_group_id], key_name=key_pair_name, user_data=startup_script, block_device_map=bdm) return res.id except: raise
def _shutdown_constellation_public_ips(constellation_name, constellation_prefix, partial_deploy, credentials_softlayer, constellation_directory): constellation = ConstellationState(constellation_name) launch_stage = constellation.get_value("launch_stage") if launch_sequence.index(launch_stage) >= launch_sequence.index( "block_public_ips"): return __wait_for_find_file(constellation_name, constellation_directory, partial_deploy, "cloudsim/setup/done", "running") m = "Switching off public network interfaces" constellation.set_value('sim_launch_msg', m) #constellation.set_value('router_launch_msg', m) private_machines = ["sim-%s" % constellation_prefix] if not partial_deploy: private_machines.append("fc1-%s" % constellation_prefix) private_machines.append("fc2-%s" % constellation_prefix) constellation.set_value('fc1_launch_msg', m) constellation.set_value('fc2_launch_msg', m) osrf_creds = load_osrf_creds(credentials_softlayer) shutdown_public_ips(osrf_creds, private_machines) constellation.set_value("launch_stage", "block_public_ips")
def _monitor_ping(constellation_name, ping_data_key, ping_str): """ internal implementation for monitor_cloudsim_ping and monitor_ssh_ping """ constellation = ConstellationState(constellation_name) latency = constellation.get_value(ping_data_key) latency = record_ping_result(latency, ping_str, LATENCY_TIME_BUFFER) constellation.set_value(ping_data_key, latency)
def get_ssh_client(constellation_name, machine_state, ip_key, sshkey_key): """ Checks to see if machine is ready and creates an ssh client accordingly """ ssh_client = None constellation = ConstellationState(constellation_name) if machine_states.index(machine_state) >= \ machine_states.index('packages_setup'): constellation_directory = \ constellation.get_value('constellation_directory') machine_ip = constellation.get_value(ip_key) key_pair_name = constellation.get_value(sshkey_key) ssh_client = SshClient(constellation_directory, key_pair_name, 'ubuntu', machine_ip) return ssh_client
def _release_key_pair(constellation_name, machine_prefix, ec2conn): constellation = ConstellationState(constellation_name) key_pair_name = None try: key_pair_name = 'key-%s-%s' % (machine_prefix, constellation_name) ec2conn.delete_key_pair(key_pair_name) except Exception, e: error_msg = constellation.get_value('error') error_msg += "<b>Release key</b>: %s<br>" % e constellation.set_value('error', error_msg) log("error cleaning up simulation key %s: %s" % (key_pair_name, e))
def monitor_launch_state(constellation_name, ssh_client, machine_state, dpkg_cmd, launch_msg_key): if ssh_client is None: # too early to verify return try: constellation = ConstellationState(constellation_name) constellation_state = constellation.get_value("constellation_state") if constellation_states.index(constellation_state) >= \ constellation_states.index("launching"): if machine_state == 'packages_setup': dpkg_line = ssh_client.cmd(dpkg_cmd) package_msg = parse_dpkg_line(dpkg_line) current_value = constellation.get_value(launch_msg_key) if current_value != package_msg: constellation.set_value(launch_msg_key, package_msg) except: tb = traceback.format_exc() log("monitor_launch_state traceback: %s" % tb)
def _startup_scripts(constellation_name, partial_deploy): constellation = ConstellationState(constellation_name) launch_stage = constellation.get_value("launch_stage") if launch_sequence.index(launch_stage) >= launch_sequence.index('startup'): return constellation_directory = constellation.get_value( 'constellation_directory') # if the change of ip was successful, the script should be in the home # directory of each machine __wait_for_find_file(constellation_name, constellation_directory, partial_deploy, "change_ip.bash", "packages_setup") m = "Executing startup script" constellation.set_value('sim_launch_msg', m) constellation.set_value('router_launch_msg', m) router_ip = constellation.get_value("router_public_ip") ssh_router = SshClient(constellation_directory, "key-router", 'ubuntu', router_ip) # load packages onto router ssh_router.cmd("nohup sudo bash cloudsim/router_startup_script.bash " "> ssh_startup.out 2> ssh_startup.err < /dev/null &") # load packages onto simulator ssh_router.cmd("cloudsim/sim_init.bash") if not partial_deploy: constellation.set_value('fc1_launch_msg', m) constellation.set_value('fc2_launch_msg', m) # load packages onto fc1 ssh_router.cmd("cloudsim/fc1_init.bash") # load packages onto fc2 ssh_router.cmd("cloudsim/fc2_init.bash") constellation.set_value("fc1_state", "packages_setup") constellation.set_value("fc2_state", "packages_setup") constellation.set_value("sim_state", "packages_setup") constellation.set_value("router_state", "packages_setup") constellation.set_value("launch_stage", "startup")
def monitor_cloudsim_ping(constellation_name, ip_address_key, ping_data_key): """ Finds the ip of the machine to pind in redis, pings the machine and integrates the results with the existing data. The ping is done from CloudSim """ constellation = ConstellationState(constellation_name) if constellation.has_value(ip_address_key): ip_address = constellation.get_value(ip_address_key) o, ping_str = commands.getstatusoutput("ping -c3 %s" % ip_address) if o == 0: _monitor_ping(constellation_name, ping_data_key, ping_str)
def acquire_dedicated_sl_server(constellation_name, osrf_creds_fname, constellation_directory): """ Acquire a dedicated SoftLayer machine """ constellation = ConstellationState(constellation_name) constellation_prefix = constellation_name.split("OSRF_CloudSim_")[1] launch_stage = constellation.get_value("launch_stage") if launch_sequence.index(launch_stage) >= launch_sequence.index('init'): return if os.path.exists(constellation_directory): shutil.rmtree(constellation_directory) os.makedirs(constellation_directory) machines_dict = {'cs-%s' % constellation_prefix: 'simulation_launch_msg'} osrf_creds = load_osrf_creds(osrf_creds_fname) reload_monitor = ReloadOsCallBack(constellation_name, machines_dict) # wait wait_for_server_reloads(osrf_creds, machines_dict.keys(), reload_monitor.callback) constellation.set_value('simulation_aws_state', 'running') constellation.set_value('simulation_state', 'packages_setup') name = "cs-%s" % constellation_prefix pub_ip, priv_ip, password = get_machine_login_info(osrf_creds, name) log("ubuntu user setup for machine cs %s [%s / %s] " % (name, pub_ip, priv_ip)) # dst_dir = os.path.abspath('.') log("machine details cs %s %s : %s" % (name, pub_ip, password)) # __add_ubuntu_user_to_router(pub_ip, password, constellation_directory, # 'key-cs') key_prefix = 'key-cs' clean_local_ssh_key_entry(pub_ip) create_ssh_key(key_prefix, constellation_directory) # setup a ubuntu sudoer no password user with an ssh key pub_key_path = os.path.join(constellation_directory, "%s.pem.pub" % key_prefix) setup_ssh_key_access(pub_ip, password, pub_key_path) priv_key_path = os.path.join(constellation_directory, "%s.pem" % key_prefix) log("ssh -i %s ubuntu@%s" % (priv_key_path, pub_ip)) constellation.set_value("launch_stage", "init") return pub_ip, pub_key_path, priv_key_path
def terminate_aws_constellation(constellation_name, credentials_ec2): """ Releases a private network, machines and all its resources """ boto.config = BotoConfig(credentials_ec2) ec2conn, vpcconn = aws_connect() constellation = ConstellationState(constellation_name) machines = constellation.get_value('machines') log("machines: %s" % machines.keys()) running_machines = {} for machine_prefix in machines.keys(): try: aws_id_key = '%s_aws_id' % machine_prefix aws_id = constellation.get_value(aws_id_key) log("%s aws id: %s" % (machine_prefix, aws_id)) running_machines[machine_prefix] = aws_id m = "terminate machine instance" constellation.set_value("%s_launch_msg" % machine_prefix, m) except Exception, e: error_msg = constellation.get_value('error') error_msg += " get aws id error %s" % e constellation.set_value('error', error_msg)
def constellation_is_terminated(constellation_name): constellation = None expire = True try: constellation = ConstellationState(constellation_name) constellation_state = constellation.get_value("constellation_state") expire = constellation_state == "terminated" except: log("Can't access constellation %s data" % constellation_name) if expire: try: constellation.expire(1) except: pass return expire
def _acquire_key_pair(constellation_name, machine_prefix, ec2conn): constellation = ConstellationState(constellation_name) try: constellation_directory = constellation.get_value( 'constellation_directory') key_pair_name = 'key-%s-%s' % (machine_prefix, constellation_name) key_pair = ec2conn.create_key_pair(key_pair_name) key_pair.save(constellation_directory) src = os.path.join(constellation_directory, '%s.pem' % key_pair_name) dst = os.path.join(constellation_directory, 'key-%s.pem' % machine_prefix) shutil.copy(src, dst) return key_pair_name except Exception, e: constellation.set_value('error', "key error: %s" % e) raise
def terminate_dedicated_sl_server(constellation_name, machine_name, osrf_creds_fname): constellation = ConstellationState(constellation_name) launch_stage = constellation.get_value("launch_stage") if launch_sequence.index(launch_stage) >= launch_sequence.index( 'os_reload'): return osrf_creds = load_osrf_creds(osrf_creds_fname) # compute the softlayer machine names machine_names = [machine_name] pub_ip, priv_ip, password = get_machine_login_info(osrf_creds, machine_names[0]) log("reload os for machine %s [%s / %s] password %s " % (machine_names[0], pub_ip, priv_ip, password)) reload_servers(osrf_creds, machine_names) constellation.set_value("launch_stage", "os_reload")
def _release_vpc(constellation_name, vpcconn): constellation = ConstellationState(constellation_name) error_msg = constellation.get_value('error') original_error = error_msg vpc_id = None subnet_id = None igw_id = None route_table_association_id = None route_table_id = None try: vpc_id = constellation.get_value('vpc_id') subnet_id = constellation.get_value('subnet_id') igw_id = constellation.get_value('igw_id') route_table_id = constellation.get_value('route_table_id') route_table_association_id = constellation.get_value( 'route_table_association_id') except Exception, e: error_msg += "%s" % e log("missing db key %s" % e)
def _notify_portal(constellation, task): try: root_log_dir = '/tmp/cloudsim_logs' # Get metadata (team, competition, ...) config = get_cloudsim_config() portal_info_fname = config['cloudsim_portal_json_path'] log("** Portal JSON path: %s ***" % portal_info_fname) portal_info = None with open(portal_info_fname, 'r') as f: portal_info = json.loads(f.read()) log("** Portal JSON file opened ***") team = portal_info['team'] comp = portal_info['event'] task_num = task['vrc_num'] log("** Team: %s, Event: %s ***" % (team, comp)) if task_num < '1' or task_num > '3': task_num = '1' run = task['vrc_id'] if run < '1' or run > '5': run = '1' start_time = task['start_time'] start_task = dateutil.parser.parse(start_time) start_task = start_task.strftime("%d/%m/%y %H:%M:%S") const = ConstellationState(constellation) constellation_dict = get_constellation_data(constellation) constellation_directory = constellation_dict['constellation_directory'] router_ip = const.get_value("router_public_ip") task_id = task['ros_launch'] task_dirname = task_id.split('.')[0] # Store in this cloudsim the network and sim logs router_key = os.path.join(constellation_directory, 'key-router.pem') new_msg = task['task_message'] + '<B> Getting logs</B>' const.update_task_value(task['task_id'], 'task_message', new_msg) cmd = ('bash /var/www/bin/get_logs.bash %s %s %s' % (task_dirname, router_ip, router_key)) subprocess.check_call(cmd.split()) log("** Log directory created***") # Get the score and falls score = '0' #falls = 'N/A' runtime = 'N/A' try: p = os.path.join(root_log_dir, task_dirname, 'score.log') with open(p) as f: log("** score.log found **") data = f.read() log("** Reading score.log file **") lines = data.split('\n') last_line = lines[-2] log("** Last line: %s **" % last_line) score = last_line.split(',')[4] #falls = last_line.split(',')[5] # Time when the task stopped runtime = last_line.split(',')[1] log("** All sim score fields parsed **") except Exception: None # Create JSON file with the task metadata data = json.dumps({'team': team, 'event': comp, 'task': task_num, 'start_time': start_task, 'result': 'Terminated', 'runtime': runtime, 'score': score}, sort_keys=True, indent=4, separators=(',', ': ')) log("** JSON data created **") with open(os.path.join(root_log_dir, task_dirname, 'end_task.json'), 'w') as f: f.write(str(data)) log("** JSON file created ***") new_msg = new_msg.replace('Getting logs', 'Creating tar file') const.update_task_value(task['task_id'], 'task_message', new_msg) # Tar all the log content tar_name = (team + '_' + comp + '_' + str(task_num) + '_' + str(run) + '.tar') p = os.path.join(root_log_dir, task_dirname) cmd = 'tar cf /tmp/' + tar_name + ' -C ' + p + ' .' subprocess.check_call(cmd.split()) log("** Log directory stored in a tar file ***") new_msg = new_msg.replace('Creating tar file', 'Uploading logs to the portal') const.update_task_value(task['task_id'], 'task_message', new_msg) # Send the log to the portal config = get_cloudsim_config() portal_info_fname = config['cloudsim_portal_json_path'] portal_info = None with open(portal_info_fname, 'r') as f: portal_info = json.loads(f.read()) ssh_portal = SshClient('xxx', 'xxx', portal_info['user'], portal_info['hostname']) # this is a hack ssh_portal.key_fname = config['cloudsim_portal_key_path'] # Upload the file to the Portal temp dir dest = os.path.join('/tmp', tar_name) cmd = ('scp -o UserKnownHostsFile=/dev/null' '-o StrictHostKeyChecking=no' ' -i ' + ssh_portal.key_fname + ' ' + dest + ' ubuntu@' + portal_info['hostname'] + ':/tmp') log('cmd: %s' % cmd) subprocess.check_call(cmd.split()) # Move the file to the final destination into the Portal final_dest = os.path.join(portal_info['final_destination_dir'], tar_name) cmd = 'sudo mv %s %s' % (dest, final_dest) ssh_portal.cmd(cmd) new_msg = new_msg.replace('Uploading logs to the portal', 'Logs uploaded to the portal') const.update_task_value(task['task_id'], 'task_message', new_msg) except Exception, excep: log('notify_portal() Exception: %s' % (repr(excep))) raise
def monitor_task(constellation_name, ssh_router): """ Read score and net usage data. Aborts tasks that timeout """ def parse_score_data(score_str): toks = score_str.split() keys = [x[6:] for x in toks[-2].split(',')] keys[0] = 'time' # keys = ['time', 'wall_time', 'sim_time', 'wall_time_elapsed', # 'sim_time_elapsed', 'completion_score', # 'falls', 'message', 'task_type'] values_str = toks[-1] values = values_str.split(',') d = dict(zip(keys, values)) score = d['completion_score'] sim_time = float(d['sim_time']) / 1e9 fall_count = d['falls'] msg = d['message'] return (score, sim_time, fall_count, msg) #log("monitor_task BEGIN") constellation = ConstellationState(constellation_name) task_id = constellation.get_value("current_task") if task_id != "": sim_time = 0 timeout = 0 task = constellation.get_task(task_id) timeout = int(task['timeout']) score_str = None try: s = ssh_router.cmd("cloudsim/get_score.bash") log(s) score, sim_time, fall_count, msg = parse_score_data(s) score_str = "" #score_str += "<b>%s</b>: %s. " % ("score", score) score_str += "<b>%s</b>: %s. " % ("sim time", sim_time) score_str += " %s" % (msg) score_str += "<b>falls:</b> %s." % fall_count except Exception, e: #score_str = "No score available." tb = traceback.format_exc() log("traceback: %s" % tb) log("score %s" % score_str) net_str = None try: n = ssh_router.cmd("cloudsim/get_network_usage.bash") log(n) toks = n.split() up_bits = int(toks[2]) * 8 down_bits = int(toks[3]) * 8 up_cap = int(task['uplink_data_cap']) down_cap = int(task['downlink_data_cap']) up = 0.0 down = 0.0 if up_cap != 0.0: up = 100.0 * up_bits / up_cap if down_cap != 0.0: down = 100.0 * down_bits / down_cap net_str = "<b>up/down link (%%)</b>: %0.2f / %0.2f" % (up, down) except Exception, e: # net_str = "no network usage available" log("score monitoring error %s" % e) tb = traceback.format_exc() log("traceback: %s" % tb)
def acquire_aws_single_server(constellation_name, credentials_ec2, constellation_directory, machine_prefix, # name of machine, ie "sim" machine_data, startup_script, tags): sim_machine_name = "%s_%s" % (machine_prefix, constellation_name) ec2conn, _ = aws_connect(credentials_ec2) aws_image = machine_data['software'] aws_instance = machine_data['hardware'] bdm = __get_block_device_mapping(aws_instance) constellation = ConstellationState(constellation_name) constellation.set_value('%s_launch_msg' % machine_prefix, "setting up security groups") constellation_directory = constellation.get_value( 'constellation_directory') # save local startup script copy script_fname = os.path.join(constellation_directory, "%s_startup_script.txt" % machine_prefix) with open(script_fname, 'w') as f: f.write(startup_script) constellation.set_value('machine_name', machine_prefix) security_group_data = machine_data['security_group'] security_group_name, _ = _acquire_security_group( constellation_name, machine_prefix, security_group_data, vpc_id=None, ec2conn=ec2conn) key_pair_name = _acquire_key_pair(constellation_name, machine_prefix, ec2conn) roles_to_reservations = {} try: constellation.set_value('%s_launch_msg' % machine_prefix, "requesting machine") res = ec2conn.run_instances(image_id=aws_image, instance_type=aws_instance, #subnet_id = subnet_id, #private_ip_address=SIM_IP, security_groups=[security_group_name], key_name=key_pair_name, user_data=startup_script, block_device_map=bdm) roles_to_reservations['simulation_state'] = res.id except: log("ouch!") raise aws_id = None count = 200 done = False while not done: log("attempt %s" % count) time.sleep(2) count -= 1 for r in ec2conn.get_all_instances(): if count < 0: msg = ("timeout while waiting " "for EC2 machine(s) %s" % sim_machine_name) raise LaunchException(msg) if r.id == res.id: state = r.instances[0].state if state == 'running': aws_id = r.instances[0].id constellation.set_value('%s_state' % machine_prefix, 'network_setup') done = True constellation.set_value('%s_aws_state' % machine_prefix, state) constellation.set_value('%s_launch_msg' % machine_prefix, "machine running") constellation.set_value('%s_aws_id' % machine_prefix, aws_id) sim_tags = {'Name': sim_machine_name} sim_tags.update(tags) ec2conn.create_tags([aws_id], sim_tags) # ec2conn.associate_address(router_aws_id, allocation_id=eip_allocation_id) instance = get_ec2_instance(ec2conn, aws_id) ip = instance.ip_address clean_local_ssh_key_entry(ip) constellation.set_value('%s_public_ip' % (machine_prefix), ip) return ip, aws_id, key_pair_name
def _initialize_private_machines(constellation_name, constellation_prefix, partial_deploy, credentials_softlayer, router_script, sim_script, fc1_script, fc2_script, constellation_directory): constellation = ConstellationState(constellation_name) launch_stage = constellation.get_value("launch_stage") if launch_sequence.index(launch_stage) >= \ launch_sequence.index('init_privates'): return # # router machine # router_ip = constellation.get_value("router_public_ip") ssh_router = SshClient(constellation_directory, "key-router", 'ubuntu', router_ip) local_fname = os.path.join(constellation_directory, 'router_startup.bash') with open(local_fname, 'w') as f: f.write(router_script) remote_fname = 'cloudsim/router_startup_script.bash' ssh_router.upload_file(local_fname, remote_fname) log("upload %s to %s" % (local_fname, remote_fname)) osrf_creds = load_osrf_creds(credentials_softlayer) # # sim machine # sim_pub_ip, sim_priv_ip, sim_root_password = get_machine_login_info( osrf_creds, "sim-%s" % constellation_prefix) log("provision sim [%s / %s] %s" % (sim_pub_ip, sim_priv_ip, sim_root_password)) _provision_ssh_private_machine(constellation_name, ssh_router, "sim", sim_priv_ip, sim_root_password, sim_script, constellation_directory) if not partial_deploy: # # fc1 machine # fc1_pub_ip, fc1_priv_ip, fc1_root_password = get_machine_login_info( osrf_creds, "fc1-%s" % constellation_prefix) log("provision fc1 [%s / %s] %s" % (fc1_pub_ip, fc1_priv_ip, fc1_root_password)) _provision_ssh_private_machine(constellation_name, ssh_router, "fc1", fc1_priv_ip, fc1_root_password, fc1_script, constellation_directory) # # fc2 machine # fc2_pub_ip, fc2_priv_ip, fc2_root_password = get_machine_login_info( osrf_creds, "fc2-%s" % constellation_prefix) log("provision fc2 [%s / %s] %s" % (fc2_pub_ip, fc2_priv_ip, fc2_root_password)) _provision_ssh_private_machine(constellation_name, ssh_router, "fc2", fc2_priv_ip, fc2_root_password, fc2_script, constellation_directory) log('configure_machines done') constellation.set_value("launch_stage", "init_privates")
def _initialize_router(constellation_name, constellation_prefix, partial_deploy, osrf_creds_fname, constellation_directory): constellation = ConstellationState(constellation_name) launch_stage = constellation.get_value("launch_stage") if launch_sequence.index(launch_stage) >= launch_sequence.index( 'init_router'): return osrf_creds = load_osrf_creds(osrf_creds_fname) router_name = "router-%s" % constellation_prefix m = "running" constellation.set_value('fc1_aws_state', m) constellation.set_value('fc2_aws_state', m) constellation.set_value('sim_aws_state', m) constellation.set_value('router_aws_state', m) constellation.set_value('router_launch_msg', 'user account setup') router_ip, priv_ip, password = get_machine_login_info( osrf_creds, router_name) constellation.set_value("router_public_ip", router_ip) log("router %s %s" % (router_ip, password)) sim_pub_ip, sim_priv_ip, sim_root_password = get_machine_login_info( osrf_creds, "sim-%s" % constellation_prefix) log("sim %s %s" % (sim_pub_ip, sim_root_password)) log("ubuntu user setup for machine router %s [%s / %s] " % (router_name, router_ip, priv_ip)) log("router %s %s : %s" % (router_name, router_ip, password)) __add_ubuntu_user_to_router(router_ip, password, constellation_directory) softlayer_scripts_dir = os.path.join(os.path.dirname(softlayer.__file__), 'bash') ssh_router = SshClient(constellation_directory, "key-router", 'ubuntu', router_ip) # create a remote cloudsim directory on the router ssh_router.cmd("mkdir -p cloudsim") openvpn_fname = os.path.join(constellation_directory, 'openvpn.key') create_openvpn_key(openvpn_fname) remote_fname = 'cloudsim/openvpn.key' ssh_router.upload_file(openvpn_fname, remote_fname) local_fname = os.path.join(softlayer_scripts_dir, 'router_init.bash') remote_fname = 'cloudsim/router_init.bash' ssh_router.upload_file(local_fname, remote_fname) log("upload %s to %s" % (local_fname, remote_fname)) # upload ubuntu user setup scripts local_fname = os.path.join(softlayer_scripts_dir, 'auto_ubuntu.bash') remote_fname = 'cloudsim/auto_ubuntu.bash' ssh_router.upload_file(local_fname, remote_fname) log("upload %s to %s" % (local_fname, remote_fname)) local_fname = os.path.join(softlayer_scripts_dir, 'create_ubuntu_user.exp') remote_fname = 'cloudsim/create_ubuntu_user.exp' ssh_router.upload_file(local_fname, remote_fname) log("upload %s to %s" % (local_fname, remote_fname)) local_fname = os.path.join(softlayer_scripts_dir, 'upload_key.exp') remote_fname = 'cloudsim/upload_key.exp' ssh_router.upload_file(local_fname, remote_fname) log("upload %s to %s" % (local_fname, remote_fname)) local_fname = os.path.join(softlayer_scripts_dir, 'process_remote_ssh_key.exp') remote_fname = 'cloudsim/process_remote_ssh_key.exp' ssh_router.upload_file(local_fname, remote_fname) log("upload %s to %s" % (local_fname, remote_fname)) # avoid ssh error because our server has changed constellation.set_value("launch_stage", "init_router") create_ssh_key("key-sim", constellation_directory) fc1_priv_ip = "0.0.0.0" fc2_priv_ip = "0.0.0.0" if not partial_deploy: fc1_pub_ip, fc1_priv_ip, fc1_root_password = get_machine_login_info( osrf_creds, "fc1-%s" % constellation_prefix) log("fc1 %s %s" % (fc1_pub_ip, fc1_root_password)) create_ssh_key("key-fc1", constellation_directory) fc2_pub_ip, fc2_priv_ip, fc2_root_password = get_machine_login_info( osrf_creds, "fc2-%s" % constellation_prefix) log("fc2 %s %s" % (fc2_pub_ip, fc2_root_password)) create_ssh_key("key-fc2", constellation_directory) arg_str = "%s %s %s" % (sim_priv_ip, fc1_priv_ip, fc2_priv_ip) cmd = "cd cloudsim; ./router_init.bash /home/ubuntu/cloudsim %s" % arg_str # run the script constellation.set_value('router_launch_msg', 'generating bash scripts') ssh_router.cmd(cmd) constellation.set_value('router_launch_msg', 'uploading ssh key') __upload_ssh_keys_to_router(ssh_router, "router", constellation_directory) constellation.set_value('sim_launch_msg', 'uploading ssh key') __upload_ssh_keys_to_router(ssh_router, "sim", constellation_directory) constellation.set_value('fc1_launch_msg', 'uploading ssh key') __upload_ssh_keys_to_router(ssh_router, "fc1", constellation_directory) constellation.set_value('fc2_launch_msg', 'uploading ssh key') __upload_ssh_keys_to_router(ssh_router, "fc2", constellation_directory)