def _acquire_vpc_server(constellation_name,
machine_prefix,
key_pair_name,
machine_data,
startup_script,
subnet_id,
security_group_id,
ec2conn):
constellation = ConstellationState(constellation_name)
try:
constellation.set_value('%s_launch_msg' % machine_prefix, "booting")
aws_image = machine_data['software']
aws_instance = machine_data['hardware']
ip = machine_data['ip']
bdm = __get_block_device_mapping(aws_instance)
constellation_directory = constellation.get_value(
'constellation_directory')
# save local startup script copy
script_fname = os.path.join(constellation_directory,
"%s_startup_script.txt" % machine_prefix)
with open(script_fname, 'w') as f:
f.write(startup_script)
res = ec2conn.run_instances(aws_image,
instance_type=aws_instance,
subnet_id=subnet_id,
private_ip_address=ip,
security_group_ids=[security_group_id],
key_name=key_pair_name,
user_data=startup_script,
block_device_map=bdm)
return res.id
except:
raise
def monitor_gzweb(constellation_name, ssh_client, sim_state):
"""
Detects if the gzweb is running and writes the
url into the "gzweb" dictionary key
"""
constellation = ConstellationState(constellation_name)
simulation_state = constellation.get_value('sim_state')
if machine_states.index(simulation_state) >= \
machine_states.index('running'):
gl_state = constellation.get_value("gazebo")
if gl_state == "running":
try:
# current_state = constellation.get_value(GZWEB_KEY)
out = ssh_client.cmd("bash cloudsim/ping_gzweb.bash")
log("ping_gzweb returned [%s]" % out)
if out == "":
constellation.set_value(GZWEB_KEY, "not running")
return False
else:
constellation.set_value(GZWEB_KEY, "running")
return True
except Exception, e:
log("monitor: cloudsim/ping_gzweb.bash error: %s" % e)
constellation.set_value(GZWEB_KEY, "")
return False
def _monitor_ping(constellation_name, ping_data_key, ping_str):
"""
internal implementation for monitor_cloudsim_ping and monitor_ssh_ping
"""
constellation = ConstellationState(constellation_name)
latency = constellation.get_value(ping_data_key)
latency = record_ping_result(latency, ping_str, LATENCY_TIME_BUFFER)
constellation.set_value(ping_data_key, latency)
def terminate_openstack_server(constellation_name, creds):
'''
Retrieves necessary information from the redis database.
Uses that information to call terminate.
'''
constellation = ConstellationState(constellation_name)
secgroup = constellation.get_value("security_group")
keypair = constellation.get_value("keypair")
instance_name = constellation.get_value("instance")
terminate(instance_name, keypair, secgroup, creds)
def terminate_softlayer_constellation(constellation_name, constellation_prefix,
partial_reload, osrf_creds_fname):
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= \
launch_sequence.index('os_reload'):
return
else:
osrf_creds = load_osrf_creds(osrf_creds_fname)
# compute the softlayer machine names
constellation.set_value('sim_launch_msg', 'reload OS')
machine_names = [
x + "-" + constellation_prefix for x in ('router', 'sim')
]
if not partial_reload:
machine_names.append('fc2-' + constellation_prefix)
machine_names.append('fc1-' + constellation_prefix)
constellation.set_value('fc1_launch_msg', 'reload OS')
constellation.set_value('fc2_launch_msg', 'reload OS')
# enable nics on machines with disconnected ones (not the router)
enable_public_ips(osrf_creds, machine_names[1:])
for server in machine_names[1:]:
t = get_active_transaction(osrf_creds, server)
log("Transaction before reload on %s: %s" % (server, t))
reload_servers(osrf_creds, machine_names)
constellation.set_value("launch_stage", "os_reload")
def _shutdown_constellation_public_ips(constellation_name,
constellation_prefix, partial_deploy,
credentials_softlayer,
constellation_directory):
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= launch_sequence.index(
"block_public_ips"):
return
__wait_for_find_file(constellation_name, constellation_directory,
partial_deploy, "cloudsim/setup/done", "running")
m = "Switching off public network interfaces"
constellation.set_value('sim_launch_msg', m)
#constellation.set_value('router_launch_msg', m)
private_machines = ["sim-%s" % constellation_prefix]
if not partial_deploy:
private_machines.append("fc1-%s" % constellation_prefix)
private_machines.append("fc2-%s" % constellation_prefix)
constellation.set_value('fc1_launch_msg', m)
constellation.set_value('fc2_launch_msg', m)
osrf_creds = load_osrf_creds(credentials_softlayer)
shutdown_public_ips(osrf_creds, private_machines)
constellation.set_value("launch_stage", "block_public_ips")
def _release_key_pair(constellation_name, machine_prefix, ec2conn):
constellation = ConstellationState(constellation_name)
key_pair_name = None
try:
key_pair_name = 'key-%s-%s' % (machine_prefix, constellation_name)
ec2conn.delete_key_pair(key_pair_name)
except Exception, e:
error_msg = constellation.get_value('error')
error_msg += "<b>Release key</b>: %s<br>" % e
constellation.set_value('error', error_msg)
log("error cleaning up simulation key %s: %s" % (key_pair_name, e))
class ReloadOsCallBack(object):
def __init__(self, constellation_name, machines_dict):
self.constellation_name = constellation_name
self.machines_dict = machines_dict
self.constellation_state = ConstellationState(constellation_name)
def callback(self, machine_name, state):
msg_key = self.machines_dict[machine_name]
log("[%s] %s [%s] %s" %
(self.constellation_name, machine_name, msg_key, state))
self.constellation_state.set_value(msg_key, state)
def __wait_for_find_file(constellation_name, constellation_directory,
partial_deploy, ls_cmd, end_state):
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= 'running':
return
router_ip = constellation.get_value("router_public_ip")
ssh_router = SshClient(constellation_directory, "key-router", 'ubuntu',
router_ip)
q = []
#
# Wait until machines are online (rebooted?)
#
q.append(
get_ssh_cmd_generator(ssh_router,
"ls %s" % ls_cmd,
ls_cmd,
constellation,
"router_state",
"running",
max_retries=500))
q.append(
get_ssh_cmd_generator(ssh_router,
"cloudsim/find_file_sim.bash %s" % ls_cmd,
ls_cmd,
constellation,
"sim_state",
end_state,
max_retries=500))
if not partial_deploy:
q.append(
get_ssh_cmd_generator(ssh_router,
"cloudsim/find_file_fc1.bash %s" % ls_cmd,
ls_cmd,
constellation,
"fc1_state",
end_state,
max_retries=500))
q.append(
get_ssh_cmd_generator(ssh_router,
"cloudsim/find_file_fc2.bash %s" % ls_cmd,
ls_cmd,
constellation,
"fc2_state",
end_state,
max_retries=500))
empty_ssh_queue(q, sleep=2)
def monitor_cloudsim_ping(constellation_name, ip_address_key, ping_data_key):
"""
Finds the ip of the machine to pind in redis,
pings the machine and integrates
the results with the existing data.
The ping is done from CloudSim
"""
constellation = ConstellationState(constellation_name)
if constellation.has_value(ip_address_key):
ip_address = constellation.get_value(ip_address_key)
o, ping_str = commands.getstatusoutput("ping -c3 %s" % ip_address)
if o == 0:
_monitor_ping(constellation_name, ping_data_key, ping_str)
def monitor_simulator(constellation_name,
ssh_client,
sim_state_key='sim_state'):
"""
Detects if the simulator is running and writes the
result into the "gazebo" dictionary key
"""
if ssh_client is None:
#constellation.set_value("gazebo", "not running")
return False
constellation = ConstellationState(constellation_name)
simulation_state = constellation.get_value(sim_state_key)
if machine_states.index(simulation_state) >= \
machine_states.index('running'):
gl_state = constellation.get_value("sim_glx_state")
if gl_state == "running":
try:
out = ssh_client.cmd("bash cloudsim/ping_gazebo.bash")
#log("ping_gazebo returned [%s]" % out)
if out == "":
constellation.set_value("gazebo", "not running")
return False
except Exception, e:
log("monitor: cloudsim/ping_gazebo.bash error: %s" % e)
constellation.set_value("gazebo", "not running")
return False
def _acquire_key_pair(constellation_name, machine_prefix, ec2conn):
constellation = ConstellationState(constellation_name)
try:
constellation_directory = constellation.get_value(
'constellation_directory')
key_pair_name = 'key-%s-%s' % (machine_prefix, constellation_name)
key_pair = ec2conn.create_key_pair(key_pair_name)
key_pair.save(constellation_directory)
src = os.path.join(constellation_directory, '%s.pem' % key_pair_name)
dst = os.path.join(constellation_directory,
'key-%s.pem' % machine_prefix)
shutil.copy(src, dst)
return key_pair_name
except Exception, e:
constellation.set_value('error', "key error: %s" % e)
raise
def constellation_is_terminated(constellation_name):
constellation = None
expire = True
try:
constellation = ConstellationState(constellation_name)
constellation_state = constellation.get_value("constellation_state")
expire = constellation_state == "terminated"
except:
log("Can't access constellation %s data" % constellation_name)
if expire:
try:
constellation.expire(1)
except:
pass
return expire
def get_ssh_client(constellation_name, machine_state, ip_key, sshkey_key):
"""
Checks to see if machine is ready and creates an ssh client accordingly
"""
ssh_client = None
constellation = ConstellationState(constellation_name)
if machine_states.index(machine_state) >= \
machine_states.index('packages_setup'):
constellation_directory = \
constellation.get_value('constellation_directory')
machine_ip = constellation.get_value(ip_key)
key_pair_name = constellation.get_value(sshkey_key)
ssh_client = SshClient(constellation_directory,
key_pair_name,
'ubuntu',
machine_ip)
return ssh_client
def monitor_ssh_ping(constellation_name,
ssh_client,
ip_address,
ping_data_key):
"""
Pings a machine and integrates the results with the existing data into the
database. The ping is done from the ssh client (i.e router computer)
"""
if ssh_client is None:
return
try:
ping_str = ssh_client.cmd("ping -c3 %s" % ip_address)
_monitor_ping(constellation_name, ping_data_key, ping_str)
except:
tb = traceback.format_exc()
log("monitor_ssh_ping traceback: %s" % tb)
constellation = ConstellationState(constellation_name)
constellation.set_value(ping_data_key, "[]")
def acquire_dedicated_sl_server(constellation_name, osrf_creds_fname,
constellation_directory):
"""
Acquire a dedicated SoftLayer machine
"""
constellation = ConstellationState(constellation_name)
constellation_prefix = constellation_name.split("OSRF_CloudSim_")[1]
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= launch_sequence.index('init'):
return
if os.path.exists(constellation_directory):
shutil.rmtree(constellation_directory)
os.makedirs(constellation_directory)
machines_dict = {'cs-%s' % constellation_prefix: 'simulation_launch_msg'}
osrf_creds = load_osrf_creds(osrf_creds_fname)
reload_monitor = ReloadOsCallBack(constellation_name, machines_dict)
# wait
wait_for_server_reloads(osrf_creds, machines_dict.keys(),
reload_monitor.callback)
constellation.set_value('simulation_aws_state', 'running')
constellation.set_value('simulation_state', 'packages_setup')
name = "cs-%s" % constellation_prefix
pub_ip, priv_ip, password = get_machine_login_info(osrf_creds, name)
log("ubuntu user setup for machine cs %s [%s / %s] " %
(name, pub_ip, priv_ip))
# dst_dir = os.path.abspath('.')
log("machine details cs %s %s : %s" % (name, pub_ip, password))
# __add_ubuntu_user_to_router(pub_ip, password, constellation_directory,
# 'key-cs')
key_prefix = 'key-cs'
clean_local_ssh_key_entry(pub_ip)
create_ssh_key(key_prefix, constellation_directory)
# setup a ubuntu sudoer no password user with an ssh key
pub_key_path = os.path.join(constellation_directory,
"%s.pem.pub" % key_prefix)
setup_ssh_key_access(pub_ip, password, pub_key_path)
priv_key_path = os.path.join(constellation_directory,
"%s.pem" % key_prefix)
log("ssh -i %s ubuntu@%s" % (priv_key_path, pub_ip))
constellation.set_value("launch_stage", "init")
return pub_ip, pub_key_path, priv_key_path
def terminate_dedicated_sl_server(constellation_name, machine_name,
osrf_creds_fname):
constellation = ConstellationState(constellation_name)
launch_stage = constellation.get_value("launch_stage")
if launch_sequence.index(launch_stage) >= launch_sequence.index(
'os_reload'):
return
osrf_creds = load_osrf_creds(osrf_creds_fname)
# compute the softlayer machine names
machine_names = [machine_name]
pub_ip, priv_ip, password = get_machine_login_info(osrf_creds,
machine_names[0])
log("reload os for machine %s [%s / %s] password %s " %
(machine_names[0], pub_ip, priv_ip, password))
reload_servers(osrf_creds, machine_names)
constellation.set_value("launch_stage", "os_reload")
def acquire_openstack_server(constellation_name, creds,
constellation_directory, machine_name, script):
'''
Calls the launch function.
Stores the returned values in a redis database
'''
floating_ip, instance_name, keypair_name, security_group_name = \
openstack_launch(constellation_name,
machine_name,
constellation_directory,
script,
creds)
constellation = ConstellationState(constellation_name)
constellation.set_value("security_group", security_group_name)
constellation.set_value("keypair", keypair_name)
constellation.set_value("instance", instance_name)
constellation.set_value("floating_ip", floating_ip)
return floating_ip, instance_name, keypair_name
def acquire_aws_constellation(constellation_name,
credentials_ec2,
machines,
scripts,
tags):
"""
Creates a virtual network with machines inside. Each machine has
- an elastic IP
- a security group
- a key (for the ubuntu user)
"""
constellation = ConstellationState(constellation_name)
constellation.set_value('machines', machines)
ec2conn, vpcconn = aws_connect(credentials_ec2)
vpc_id, subnet_id = _acquire_vpc(constellation_name,
vpcconn)
log("VPC %s" % vpc_id)
roles_to_reservations = {}
for machine_name, machine_data in machines.iteritems():
aws_key_name = _acquire_key_pair(constellation_name,
machine_name, ec2conn)
security_group_data = machines[machine_name]['security_group']
_, security_group_id = _acquire_security_group(
constellation_name,
machine_name,
security_group_data,
vpc_id,
ec2conn)
startup_srcript = scripts[machine_name]
reservation_id = _acquire_vpc_server(constellation_name,
machine_name,
aws_key_name,
machine_data,
startup_srcript,
subnet_id,
security_group_id,
ec2conn)
roles_to_reservations[machine_name] = reservation_id
machines_to_awsid = wait_for_multiple_machines_to_run(ec2conn,
roles_to_reservations,
tags,
constellation,
max_retries=500,
final_state='packages_setup')
for machine_name, aws_id in machines_to_awsid.iteritems():
m = "acquiring public Internet IP"
constellation.set_value("%s_launch_msg" % machine_name, m)
_acquire_vpc_elastic_ip(constellation_name,
machine_name,
aws_id,
ec2conn)
if machine_name == "router":
router_instance = get_ec2_instance(ec2conn, aws_id)
router_instance.modify_attribute('sourceDestCheck', False)
log("running machines %s" % machines_to_awsid)
def tearDown(self):
'''
Call the terminate function and
make sure that all resources (floating ip, security group, keypair)
are destroyed.
'''
terminate_openstack_server(self.constellation_name)
constellation = ConstellationState(self.constellation_name)
floating_ip = constellation.get_value("floating_ip")
pingable, ping_str = commands.getstatusoutput("ping -c3 %s" %
floating_ip)
self.assertNotEqual(pingable, 0)
creds = get_nova_creds()
nova = nvclient.Client(**creds)
instance = constellation.get_value("instance")
self.assertRaises(novaclient.exceptions.NotFound,
nova.servers.find,
name=instance)
security_group = constellation.get_value("security_group")
self.assertRaises(novaclient.exceptions.NotFound,
nova.security_groups.find,
name=security_group)
keypair = constellation.get_value("keypair")
self.assertRaises(novaclient.exceptions.NotFound,
nova.keypairs.find,
name=keypair)
def _provision_ssh_private_machine(constellation_name, ssh_router,
machine_name_prefix, private_machine_ip,
machine_password, startup_script,
constellation_directory):
constellation = ConstellationState(constellation_name)
constellation.set_value('%s_launch_msg' % machine_name_prefix,
'User account setup')
# execute script on router to add ubuntu user on the private machine
cmd = "cd cloudsim; ./auto_ubuntu.bash %s %s ./key-%s.pem.pub" % (
private_machine_ip, machine_password, machine_name_prefix)
log(cmd)
ssh_router.cmd(cmd)
local_fname = os.path.join(constellation_directory,
'%s_startup.bash' % machine_name_prefix)
with open(local_fname, 'w') as f:
f.write(startup_script)
remote_fname = 'cloudsim/%s_startup_script.bash' % machine_name_prefix
# send startup script to router
ssh_router.upload_file(local_fname, remote_fname)
def monitor_cloudsim_notebook(constellation_name, ssh_client):
constellation = ConstellationState(constellation_name)
try:
out = ssh_client.cmd("bash cloudsim/ping_cloudsim_notebook.bash")
constellation.set_value(CLOUDSIM_NOTEBOOK_KEY, "running")
except Exception, e:
log("monitor: cloudsim/ping_cloudsim_notebook.bash error: %s" % e)
constellation.set_value(CLOUDSIM_NOTEBOOK_KEY, "")
def _release_vpc(constellation_name, vpcconn):
constellation = ConstellationState(constellation_name)
error_msg = constellation.get_value('error')
original_error = error_msg
vpc_id = None
subnet_id = None
igw_id = None
route_table_association_id = None
route_table_id = None
try:
vpc_id = constellation.get_value('vpc_id')
subnet_id = constellation.get_value('subnet_id')
igw_id = constellation.get_value('igw_id')
route_table_id = constellation.get_value('route_table_id')
route_table_association_id = constellation.get_value(
'route_table_association_id')
except Exception, e:
error_msg += "%s" % e
log("missing db key %s" % e)
def terminate_aws_constellation(constellation_name, credentials_ec2):
"""
Releases a private network, machines and all its resources
"""
boto.config = BotoConfig(credentials_ec2)
ec2conn, vpcconn = aws_connect()
constellation = ConstellationState(constellation_name)
machines = constellation.get_value('machines')
log("machines: %s" % machines.keys())
running_machines = {}
for machine_prefix in machines.keys():
try:
aws_id_key = '%s_aws_id' % machine_prefix
aws_id = constellation.get_value(aws_id_key)
log("%s aws id: %s" % (machine_prefix, aws_id))
running_machines[machine_prefix] = aws_id
m = "terminate machine instance"
constellation.set_value("%s_launch_msg" % machine_prefix, m)
except Exception, e:
error_msg = constellation.get_value('error')
error_msg += " get aws id error %s" % e
constellation.set_value('error', error_msg)
def test_acquire_server(self):
'''
Tests than an openstack server can be launched.
Checks ping and ssh abilities
as well as checking to make sure the startup script was run.
'''
creds = get_nova_creds()
machine_name = "cloudsim_server"
script = '''#!/bin/bash
touch /home/ubuntu/new_file.txt''' # startup script
floating_ip, instance_name, keypair_name = acquire_openstack_server(
self.constellation_name, creds, self.constellation_directory,
machine_name, script)
constellation = ConstellationState(self.constellation_name)
#uname = 'cirros'
uname = 'ubuntu'
ssh = SshClient(self.constellation_directory, keypair_name, uname,
floating_ip)
cmd = 'ls /home/ubuntu/new_file.txt'
#expected_output = '/home/cirros'
expected_output = '/home/ubuntu/new_file.txt'
ssh_command = get_ssh_cmd_generator(ssh,
cmd,
expected_output,
constellation,
"can_ssh",
1,
max_retries=100)
ctr = 30
done = False
while not done:
time.sleep(2)
ctr -= 1
if ctr < 0:
msg = ("timeout while waiting for floating ip for %s" %
machine_name)
raise Exception(msg)
pingable, ping_str = commands.getstatusoutput("ping -c3 %s" %
floating_ip)
if pingable == 0:
done = True
empty_ssh_queue([ssh_command], 2)
self.assertEqual(pingable, 0, ping_str)
def _release_security_group(constellation_name, machine_prefix, ec2conn):
constellation = ConstellationState(constellation_name)
security_group_id = None
try:
sg_key = '%s_security_group_id' % machine_prefix
security_group_id = constellation.get_value(sg_key)
ec2conn.delete_security_group(group_id=security_group_id)
return True
except Exception, e:
error_msg = constellation.get_value('error')
error_msg += "<b>%s security group</b>: %s<br>" % (machine_prefix, e)
constellation.set_value('error', error_msg)
log("error cleaning up sim security group"
" %s: %s" % (security_group_id, e))
def _release_vpc_elastic_ip(constellation_name, machine_name_prefix, ec2conn):
constellation = ConstellationState(constellation_name)
try:
allocation_id_key = __get_allocation_id_key(machine_name_prefix)
eip_allocation_id = constellation.get_value(allocation_id_key)
log("_release_vpc_elastic_ip %s machine %s id: %s" % (
constellation_name,
machine_name_prefix,
eip_allocation_id))
ec2conn.release_address(allocation_id=eip_allocation_id)
except Exception, e:
error_msg = constellation.get_value('error')
error_msg += "<b>Router IP address</b>: %s<br>" % e
constellation.set_value('error', error_msg)
print("error cleaning up %s elastic ip: %s" % (machine_name_prefix, e))
def _acquire_vpc_elastic_ip(constellation_name,
machine_name_prefix,
aws_id,
ec2conn):
constellation = ConstellationState(constellation_name)
try:
aws_elastic_ip = ec2conn.allocate_address('vpc')
allocation_id = aws_elastic_ip.allocation_id
allocation_id_key = __get_allocation_id_key(machine_name_prefix)
constellation.set_value(allocation_id_key, allocation_id)
public_ip = aws_elastic_ip.public_ip
log("%s elastic ip %s" % (machine_name_prefix,
aws_elastic_ip.public_ip))
ip_key = '%s_public_ip' % machine_name_prefix
constellation.set_value(ip_key, public_ip)
#
# <Errors><Error><Code>InvalidAllocationID.NotFound</Code>
#
time.sleep(5)
max_ = 20
i = 0
while i < max_:
try:
time.sleep(i * 2)
ec2conn.associate_address(aws_id, allocation_id=allocation_id)
i = max_ # leave the loop
except:
i += 1
if i == max_:
raise
clean_local_ssh_key_entry(public_ip)
return public_ip
except Exception, e:
constellation.set_value('error', "Elastic IP error: %s" % e)
raise
def _acquire_security_group(constellation_name,
machine_prefix,
security_group_data,
vpc_id,
ec2conn):
constellation = ConstellationState(constellation_name)
sg = None
try:
sg_name = '%s-sg-%s' % (machine_prefix, constellation_name)
dsc = 'machine %s CloudSim constellation %s' % (machine_prefix,
constellation_name,
)
sg = ec2conn.create_security_group(sg_name, dsc, vpc_id)
max_try = 10
i = 0
while i < max_try:
log("adding tag to %s/%s security group" % (constellation_name,
machine_prefix))
try:
sg.add_tag('constellation', constellation_name)
log("tag added")
i = max_try
except Exception, e:
log("%s / %s: error: %s" % (i, max_try, e))
i += 1
time.sleep(i * 2)
if i == max_try:
raise
for rule in security_group_data:
log("authorize %s" % (rule))
sg.authorize(rule['protocol'],
rule['from_port'],
rule['to_port'],
rule['cidr'])
security_group_id = sg.id
security_group_name = sg.name
constellation.set_value('%s_security_group_id' % machine_prefix,
security_group_id)
constellation.set_value('%s_security_group_name' % machine_prefix,
security_group_name)
def monitor_launch_state(constellation_name, ssh_client,
machine_state,
dpkg_cmd,
launch_msg_key):
if ssh_client is None: # too early to verify
return
try:
constellation = ConstellationState(constellation_name)
constellation_state = constellation.get_value("constellation_state")
if constellation_states.index(constellation_state) >= \
constellation_states.index("launching"):
if machine_state == 'packages_setup':
dpkg_line = ssh_client.cmd(dpkg_cmd)
package_msg = parse_dpkg_line(dpkg_line)
current_value = constellation.get_value(launch_msg_key)
if current_value != package_msg:
constellation.set_value(launch_msg_key, package_msg)
except:
tb = traceback.format_exc()
log("monitor_launch_state traceback: %s" % tb)
