def _acquire_vpc_server(constellation_name, machine_prefix, key_pair_name, machine_data, startup_script, subnet_id, security_group_id, ec2conn): constellation = ConstellationState(constellation_name) try: constellation.set_value('%s_launch_msg' % machine_prefix, "booting") aws_image = machine_data['software'] aws_instance = machine_data['hardware'] ip = machine_data['ip'] bdm = __get_block_device_mapping(aws_instance) constellation_directory = constellation.get_value( 'constellation_directory') # save local startup script copy script_fname = os.path.join(constellation_directory, "%s_startup_script.txt" % machine_prefix) with open(script_fname, 'w') as f: f.write(startup_script) res = ec2conn.run_instances(aws_image, instance_type=aws_instance, subnet_id=subnet_id, private_ip_address=ip, security_group_ids=[security_group_id], key_name=key_pair_name, user_data=startup_script, block_device_map=bdm) return res.id except: raise
def monitor_gzweb(constellation_name, ssh_client, sim_state): """ Detects if the gzweb is running and writes the url into the "gzweb" dictionary key """ constellation = ConstellationState(constellation_name) simulation_state = constellation.get_value('sim_state') if machine_states.index(simulation_state) >= \ machine_states.index('running'): gl_state = constellation.get_value("gazebo") if gl_state == "running": try: # current_state = constellation.get_value(GZWEB_KEY) out = ssh_client.cmd("bash cloudsim/ping_gzweb.bash") log("ping_gzweb returned [%s]" % out) if out == "": constellation.set_value(GZWEB_KEY, "not running") return False else: constellation.set_value(GZWEB_KEY, "running") return True except Exception, e: log("monitor: cloudsim/ping_gzweb.bash error: %s" % e) constellation.set_value(GZWEB_KEY, "") return False
def _monitor_ping(constellation_name, ping_data_key, ping_str): """ internal implementation for monitor_cloudsim_ping and monitor_ssh_ping """ constellation = ConstellationState(constellation_name) latency = constellation.get_value(ping_data_key) latency = record_ping_result(latency, ping_str, LATENCY_TIME_BUFFER) constellation.set_value(ping_data_key, latency)
def terminate_openstack_server(constellation_name, creds): ''' Retrieves necessary information from the redis database. Uses that information to call terminate. ''' constellation = ConstellationState(constellation_name) secgroup = constellation.get_value("security_group") keypair = constellation.get_value("keypair") instance_name = constellation.get_value("instance") terminate(instance_name, keypair, secgroup, creds)
def terminate_softlayer_constellation(constellation_name, constellation_prefix, partial_reload, osrf_creds_fname): constellation = ConstellationState(constellation_name) launch_stage = constellation.get_value("launch_stage") if launch_sequence.index(launch_stage) >= \ launch_sequence.index('os_reload'): return else: osrf_creds = load_osrf_creds(osrf_creds_fname) # compute the softlayer machine names constellation.set_value('sim_launch_msg', 'reload OS') machine_names = [ x + "-" + constellation_prefix for x in ('router', 'sim') ] if not partial_reload: machine_names.append('fc2-' + constellation_prefix) machine_names.append('fc1-' + constellation_prefix) constellation.set_value('fc1_launch_msg', 'reload OS') constellation.set_value('fc2_launch_msg', 'reload OS') # enable nics on machines with disconnected ones (not the router) enable_public_ips(osrf_creds, machine_names[1:]) for server in machine_names[1:]: t = get_active_transaction(osrf_creds, server) log("Transaction before reload on %s: %s" % (server, t)) reload_servers(osrf_creds, machine_names) constellation.set_value("launch_stage", "os_reload")
def _shutdown_constellation_public_ips(constellation_name, constellation_prefix, partial_deploy, credentials_softlayer, constellation_directory): constellation = ConstellationState(constellation_name) launch_stage = constellation.get_value("launch_stage") if launch_sequence.index(launch_stage) >= launch_sequence.index( "block_public_ips"): return __wait_for_find_file(constellation_name, constellation_directory, partial_deploy, "cloudsim/setup/done", "running") m = "Switching off public network interfaces" constellation.set_value('sim_launch_msg', m) #constellation.set_value('router_launch_msg', m) private_machines = ["sim-%s" % constellation_prefix] if not partial_deploy: private_machines.append("fc1-%s" % constellation_prefix) private_machines.append("fc2-%s" % constellation_prefix) constellation.set_value('fc1_launch_msg', m) constellation.set_value('fc2_launch_msg', m) osrf_creds = load_osrf_creds(credentials_softlayer) shutdown_public_ips(osrf_creds, private_machines) constellation.set_value("launch_stage", "block_public_ips")
def _release_key_pair(constellation_name, machine_prefix, ec2conn): constellation = ConstellationState(constellation_name) key_pair_name = None try: key_pair_name = 'key-%s-%s' % (machine_prefix, constellation_name) ec2conn.delete_key_pair(key_pair_name) except Exception, e: error_msg = constellation.get_value('error') error_msg += "<b>Release key</b>: %s<br>" % e constellation.set_value('error', error_msg) log("error cleaning up simulation key %s: %s" % (key_pair_name, e))
class ReloadOsCallBack(object): def __init__(self, constellation_name, machines_dict): self.constellation_name = constellation_name self.machines_dict = machines_dict self.constellation_state = ConstellationState(constellation_name) def callback(self, machine_name, state): msg_key = self.machines_dict[machine_name] log("[%s] %s [%s] %s" % (self.constellation_name, machine_name, msg_key, state)) self.constellation_state.set_value(msg_key, state)
def __wait_for_find_file(constellation_name, constellation_directory, partial_deploy, ls_cmd, end_state): constellation = ConstellationState(constellation_name) launch_stage = constellation.get_value("launch_stage") if launch_sequence.index(launch_stage) >= 'running': return router_ip = constellation.get_value("router_public_ip") ssh_router = SshClient(constellation_directory, "key-router", 'ubuntu', router_ip) q = [] # # Wait until machines are online (rebooted?) # q.append( get_ssh_cmd_generator(ssh_router, "ls %s" % ls_cmd, ls_cmd, constellation, "router_state", "running", max_retries=500)) q.append( get_ssh_cmd_generator(ssh_router, "cloudsim/find_file_sim.bash %s" % ls_cmd, ls_cmd, constellation, "sim_state", end_state, max_retries=500)) if not partial_deploy: q.append( get_ssh_cmd_generator(ssh_router, "cloudsim/find_file_fc1.bash %s" % ls_cmd, ls_cmd, constellation, "fc1_state", end_state, max_retries=500)) q.append( get_ssh_cmd_generator(ssh_router, "cloudsim/find_file_fc2.bash %s" % ls_cmd, ls_cmd, constellation, "fc2_state", end_state, max_retries=500)) empty_ssh_queue(q, sleep=2)
def monitor_cloudsim_ping(constellation_name, ip_address_key, ping_data_key): """ Finds the ip of the machine to pind in redis, pings the machine and integrates the results with the existing data. The ping is done from CloudSim """ constellation = ConstellationState(constellation_name) if constellation.has_value(ip_address_key): ip_address = constellation.get_value(ip_address_key) o, ping_str = commands.getstatusoutput("ping -c3 %s" % ip_address) if o == 0: _monitor_ping(constellation_name, ping_data_key, ping_str)
def monitor_simulator(constellation_name, ssh_client, sim_state_key='sim_state'): """ Detects if the simulator is running and writes the result into the "gazebo" dictionary key """ if ssh_client is None: #constellation.set_value("gazebo", "not running") return False constellation = ConstellationState(constellation_name) simulation_state = constellation.get_value(sim_state_key) if machine_states.index(simulation_state) >= \ machine_states.index('running'): gl_state = constellation.get_value("sim_glx_state") if gl_state == "running": try: out = ssh_client.cmd("bash cloudsim/ping_gazebo.bash") #log("ping_gazebo returned [%s]" % out) if out == "": constellation.set_value("gazebo", "not running") return False except Exception, e: log("monitor: cloudsim/ping_gazebo.bash error: %s" % e) constellation.set_value("gazebo", "not running") return False
def _acquire_key_pair(constellation_name, machine_prefix, ec2conn): constellation = ConstellationState(constellation_name) try: constellation_directory = constellation.get_value( 'constellation_directory') key_pair_name = 'key-%s-%s' % (machine_prefix, constellation_name) key_pair = ec2conn.create_key_pair(key_pair_name) key_pair.save(constellation_directory) src = os.path.join(constellation_directory, '%s.pem' % key_pair_name) dst = os.path.join(constellation_directory, 'key-%s.pem' % machine_prefix) shutil.copy(src, dst) return key_pair_name except Exception, e: constellation.set_value('error', "key error: %s" % e) raise
def constellation_is_terminated(constellation_name): constellation = None expire = True try: constellation = ConstellationState(constellation_name) constellation_state = constellation.get_value("constellation_state") expire = constellation_state == "terminated" except: log("Can't access constellation %s data" % constellation_name) if expire: try: constellation.expire(1) except: pass return expire
def get_ssh_client(constellation_name, machine_state, ip_key, sshkey_key): """ Checks to see if machine is ready and creates an ssh client accordingly """ ssh_client = None constellation = ConstellationState(constellation_name) if machine_states.index(machine_state) >= \ machine_states.index('packages_setup'): constellation_directory = \ constellation.get_value('constellation_directory') machine_ip = constellation.get_value(ip_key) key_pair_name = constellation.get_value(sshkey_key) ssh_client = SshClient(constellation_directory, key_pair_name, 'ubuntu', machine_ip) return ssh_client
def monitor_ssh_ping(constellation_name, ssh_client, ip_address, ping_data_key): """ Pings a machine and integrates the results with the existing data into the database. The ping is done from the ssh client (i.e router computer) """ if ssh_client is None: return try: ping_str = ssh_client.cmd("ping -c3 %s" % ip_address) _monitor_ping(constellation_name, ping_data_key, ping_str) except: tb = traceback.format_exc() log("monitor_ssh_ping traceback: %s" % tb) constellation = ConstellationState(constellation_name) constellation.set_value(ping_data_key, "[]")
def acquire_dedicated_sl_server(constellation_name, osrf_creds_fname, constellation_directory): """ Acquire a dedicated SoftLayer machine """ constellation = ConstellationState(constellation_name) constellation_prefix = constellation_name.split("OSRF_CloudSim_")[1] launch_stage = constellation.get_value("launch_stage") if launch_sequence.index(launch_stage) >= launch_sequence.index('init'): return if os.path.exists(constellation_directory): shutil.rmtree(constellation_directory) os.makedirs(constellation_directory) machines_dict = {'cs-%s' % constellation_prefix: 'simulation_launch_msg'} osrf_creds = load_osrf_creds(osrf_creds_fname) reload_monitor = ReloadOsCallBack(constellation_name, machines_dict) # wait wait_for_server_reloads(osrf_creds, machines_dict.keys(), reload_monitor.callback) constellation.set_value('simulation_aws_state', 'running') constellation.set_value('simulation_state', 'packages_setup') name = "cs-%s" % constellation_prefix pub_ip, priv_ip, password = get_machine_login_info(osrf_creds, name) log("ubuntu user setup for machine cs %s [%s / %s] " % (name, pub_ip, priv_ip)) # dst_dir = os.path.abspath('.') log("machine details cs %s %s : %s" % (name, pub_ip, password)) # __add_ubuntu_user_to_router(pub_ip, password, constellation_directory, # 'key-cs') key_prefix = 'key-cs' clean_local_ssh_key_entry(pub_ip) create_ssh_key(key_prefix, constellation_directory) # setup a ubuntu sudoer no password user with an ssh key pub_key_path = os.path.join(constellation_directory, "%s.pem.pub" % key_prefix) setup_ssh_key_access(pub_ip, password, pub_key_path) priv_key_path = os.path.join(constellation_directory, "%s.pem" % key_prefix) log("ssh -i %s ubuntu@%s" % (priv_key_path, pub_ip)) constellation.set_value("launch_stage", "init") return pub_ip, pub_key_path, priv_key_path
def terminate_dedicated_sl_server(constellation_name, machine_name, osrf_creds_fname): constellation = ConstellationState(constellation_name) launch_stage = constellation.get_value("launch_stage") if launch_sequence.index(launch_stage) >= launch_sequence.index( 'os_reload'): return osrf_creds = load_osrf_creds(osrf_creds_fname) # compute the softlayer machine names machine_names = [machine_name] pub_ip, priv_ip, password = get_machine_login_info(osrf_creds, machine_names[0]) log("reload os for machine %s [%s / %s] password %s " % (machine_names[0], pub_ip, priv_ip, password)) reload_servers(osrf_creds, machine_names) constellation.set_value("launch_stage", "os_reload")
def acquire_openstack_server(constellation_name, creds, constellation_directory, machine_name, script): ''' Calls the launch function. Stores the returned values in a redis database ''' floating_ip, instance_name, keypair_name, security_group_name = \ openstack_launch(constellation_name, machine_name, constellation_directory, script, creds) constellation = ConstellationState(constellation_name) constellation.set_value("security_group", security_group_name) constellation.set_value("keypair", keypair_name) constellation.set_value("instance", instance_name) constellation.set_value("floating_ip", floating_ip) return floating_ip, instance_name, keypair_name
def acquire_aws_constellation(constellation_name, credentials_ec2, machines, scripts, tags): """ Creates a virtual network with machines inside. Each machine has - an elastic IP - a security group - a key (for the ubuntu user) """ constellation = ConstellationState(constellation_name) constellation.set_value('machines', machines) ec2conn, vpcconn = aws_connect(credentials_ec2) vpc_id, subnet_id = _acquire_vpc(constellation_name, vpcconn) log("VPC %s" % vpc_id) roles_to_reservations = {} for machine_name, machine_data in machines.iteritems(): aws_key_name = _acquire_key_pair(constellation_name, machine_name, ec2conn) security_group_data = machines[machine_name]['security_group'] _, security_group_id = _acquire_security_group( constellation_name, machine_name, security_group_data, vpc_id, ec2conn) startup_srcript = scripts[machine_name] reservation_id = _acquire_vpc_server(constellation_name, machine_name, aws_key_name, machine_data, startup_srcript, subnet_id, security_group_id, ec2conn) roles_to_reservations[machine_name] = reservation_id machines_to_awsid = wait_for_multiple_machines_to_run(ec2conn, roles_to_reservations, tags, constellation, max_retries=500, final_state='packages_setup') for machine_name, aws_id in machines_to_awsid.iteritems(): m = "acquiring public Internet IP" constellation.set_value("%s_launch_msg" % machine_name, m) _acquire_vpc_elastic_ip(constellation_name, machine_name, aws_id, ec2conn) if machine_name == "router": router_instance = get_ec2_instance(ec2conn, aws_id) router_instance.modify_attribute('sourceDestCheck', False) log("running machines %s" % machines_to_awsid)
def tearDown(self): ''' Call the terminate function and make sure that all resources (floating ip, security group, keypair) are destroyed. ''' terminate_openstack_server(self.constellation_name) constellation = ConstellationState(self.constellation_name) floating_ip = constellation.get_value("floating_ip") pingable, ping_str = commands.getstatusoutput("ping -c3 %s" % floating_ip) self.assertNotEqual(pingable, 0) creds = get_nova_creds() nova = nvclient.Client(**creds) instance = constellation.get_value("instance") self.assertRaises(novaclient.exceptions.NotFound, nova.servers.find, name=instance) security_group = constellation.get_value("security_group") self.assertRaises(novaclient.exceptions.NotFound, nova.security_groups.find, name=security_group) keypair = constellation.get_value("keypair") self.assertRaises(novaclient.exceptions.NotFound, nova.keypairs.find, name=keypair)
def _provision_ssh_private_machine(constellation_name, ssh_router, machine_name_prefix, private_machine_ip, machine_password, startup_script, constellation_directory): constellation = ConstellationState(constellation_name) constellation.set_value('%s_launch_msg' % machine_name_prefix, 'User account setup') # execute script on router to add ubuntu user on the private machine cmd = "cd cloudsim; ./auto_ubuntu.bash %s %s ./key-%s.pem.pub" % ( private_machine_ip, machine_password, machine_name_prefix) log(cmd) ssh_router.cmd(cmd) local_fname = os.path.join(constellation_directory, '%s_startup.bash' % machine_name_prefix) with open(local_fname, 'w') as f: f.write(startup_script) remote_fname = 'cloudsim/%s_startup_script.bash' % machine_name_prefix # send startup script to router ssh_router.upload_file(local_fname, remote_fname)
def monitor_cloudsim_notebook(constellation_name, ssh_client): constellation = ConstellationState(constellation_name) try: out = ssh_client.cmd("bash cloudsim/ping_cloudsim_notebook.bash") constellation.set_value(CLOUDSIM_NOTEBOOK_KEY, "running") except Exception, e: log("monitor: cloudsim/ping_cloudsim_notebook.bash error: %s" % e) constellation.set_value(CLOUDSIM_NOTEBOOK_KEY, "")
def _release_vpc(constellation_name, vpcconn): constellation = ConstellationState(constellation_name) error_msg = constellation.get_value('error') original_error = error_msg vpc_id = None subnet_id = None igw_id = None route_table_association_id = None route_table_id = None try: vpc_id = constellation.get_value('vpc_id') subnet_id = constellation.get_value('subnet_id') igw_id = constellation.get_value('igw_id') route_table_id = constellation.get_value('route_table_id') route_table_association_id = constellation.get_value( 'route_table_association_id') except Exception, e: error_msg += "%s" % e log("missing db key %s" % e)
def terminate_aws_constellation(constellation_name, credentials_ec2): """ Releases a private network, machines and all its resources """ boto.config = BotoConfig(credentials_ec2) ec2conn, vpcconn = aws_connect() constellation = ConstellationState(constellation_name) machines = constellation.get_value('machines') log("machines: %s" % machines.keys()) running_machines = {} for machine_prefix in machines.keys(): try: aws_id_key = '%s_aws_id' % machine_prefix aws_id = constellation.get_value(aws_id_key) log("%s aws id: %s" % (machine_prefix, aws_id)) running_machines[machine_prefix] = aws_id m = "terminate machine instance" constellation.set_value("%s_launch_msg" % machine_prefix, m) except Exception, e: error_msg = constellation.get_value('error') error_msg += " get aws id error %s" % e constellation.set_value('error', error_msg)
def test_acquire_server(self): ''' Tests than an openstack server can be launched. Checks ping and ssh abilities as well as checking to make sure the startup script was run. ''' creds = get_nova_creds() machine_name = "cloudsim_server" script = '''#!/bin/bash touch /home/ubuntu/new_file.txt''' # startup script floating_ip, instance_name, keypair_name = acquire_openstack_server( self.constellation_name, creds, self.constellation_directory, machine_name, script) constellation = ConstellationState(self.constellation_name) #uname = 'cirros' uname = 'ubuntu' ssh = SshClient(self.constellation_directory, keypair_name, uname, floating_ip) cmd = 'ls /home/ubuntu/new_file.txt' #expected_output = '/home/cirros' expected_output = '/home/ubuntu/new_file.txt' ssh_command = get_ssh_cmd_generator(ssh, cmd, expected_output, constellation, "can_ssh", 1, max_retries=100) ctr = 30 done = False while not done: time.sleep(2) ctr -= 1 if ctr < 0: msg = ("timeout while waiting for floating ip for %s" % machine_name) raise Exception(msg) pingable, ping_str = commands.getstatusoutput("ping -c3 %s" % floating_ip) if pingable == 0: done = True empty_ssh_queue([ssh_command], 2) self.assertEqual(pingable, 0, ping_str)
def _release_security_group(constellation_name, machine_prefix, ec2conn): constellation = ConstellationState(constellation_name) security_group_id = None try: sg_key = '%s_security_group_id' % machine_prefix security_group_id = constellation.get_value(sg_key) ec2conn.delete_security_group(group_id=security_group_id) return True except Exception, e: error_msg = constellation.get_value('error') error_msg += "<b>%s security group</b>: %s<br>" % (machine_prefix, e) constellation.set_value('error', error_msg) log("error cleaning up sim security group" " %s: %s" % (security_group_id, e))
def _release_vpc_elastic_ip(constellation_name, machine_name_prefix, ec2conn): constellation = ConstellationState(constellation_name) try: allocation_id_key = __get_allocation_id_key(machine_name_prefix) eip_allocation_id = constellation.get_value(allocation_id_key) log("_release_vpc_elastic_ip %s machine %s id: %s" % ( constellation_name, machine_name_prefix, eip_allocation_id)) ec2conn.release_address(allocation_id=eip_allocation_id) except Exception, e: error_msg = constellation.get_value('error') error_msg += "<b>Router IP address</b>: %s<br>" % e constellation.set_value('error', error_msg) print("error cleaning up %s elastic ip: %s" % (machine_name_prefix, e))
def _acquire_vpc_elastic_ip(constellation_name, machine_name_prefix, aws_id, ec2conn): constellation = ConstellationState(constellation_name) try: aws_elastic_ip = ec2conn.allocate_address('vpc') allocation_id = aws_elastic_ip.allocation_id allocation_id_key = __get_allocation_id_key(machine_name_prefix) constellation.set_value(allocation_id_key, allocation_id) public_ip = aws_elastic_ip.public_ip log("%s elastic ip %s" % (machine_name_prefix, aws_elastic_ip.public_ip)) ip_key = '%s_public_ip' % machine_name_prefix constellation.set_value(ip_key, public_ip) # # <Errors><Error><Code>InvalidAllocationID.NotFound</Code> # time.sleep(5) max_ = 20 i = 0 while i < max_: try: time.sleep(i * 2) ec2conn.associate_address(aws_id, allocation_id=allocation_id) i = max_ # leave the loop except: i += 1 if i == max_: raise clean_local_ssh_key_entry(public_ip) return public_ip except Exception, e: constellation.set_value('error', "Elastic IP error: %s" % e) raise
def _acquire_security_group(constellation_name, machine_prefix, security_group_data, vpc_id, ec2conn): constellation = ConstellationState(constellation_name) sg = None try: sg_name = '%s-sg-%s' % (machine_prefix, constellation_name) dsc = 'machine %s CloudSim constellation %s' % (machine_prefix, constellation_name, ) sg = ec2conn.create_security_group(sg_name, dsc, vpc_id) max_try = 10 i = 0 while i < max_try: log("adding tag to %s/%s security group" % (constellation_name, machine_prefix)) try: sg.add_tag('constellation', constellation_name) log("tag added") i = max_try except Exception, e: log("%s / %s: error: %s" % (i, max_try, e)) i += 1 time.sleep(i * 2) if i == max_try: raise for rule in security_group_data: log("authorize %s" % (rule)) sg.authorize(rule['protocol'], rule['from_port'], rule['to_port'], rule['cidr']) security_group_id = sg.id security_group_name = sg.name constellation.set_value('%s_security_group_id' % machine_prefix, security_group_id) constellation.set_value('%s_security_group_name' % machine_prefix, security_group_name)
def monitor_launch_state(constellation_name, ssh_client, machine_state, dpkg_cmd, launch_msg_key): if ssh_client is None: # too early to verify return try: constellation = ConstellationState(constellation_name) constellation_state = constellation.get_value("constellation_state") if constellation_states.index(constellation_state) >= \ constellation_states.index("launching"): if machine_state == 'packages_setup': dpkg_line = ssh_client.cmd(dpkg_cmd) package_msg = parse_dpkg_line(dpkg_line) current_value = constellation.get_value(launch_msg_key) if current_value != package_msg: constellation.set_value(launch_msg_key, package_msg) except: tb = traceback.format_exc() log("monitor_launch_state traceback: %s" % tb)