def reboot(options, args):
"""
Reboot an already booted plan. You must supply the run name of the booted plan.
"""
if len(args) < 2:
print "The reboot command requires a run name. See --help"
return 1
dbname = args[1]
cb = CloudInitD(options.database, db_name=dbname, log_level=options.loglevel, level_callback=level_callback, service_callback=service_callback, logdir=options.logdir, terminate=True, boot=False, ready=False, continue_on_error=True)
print_chars(1, "Rebooting %s\n" % (cb.run_name))
cb.shutdown()
try:
try:
print_chars(1, "Terminating all services %s\n" % (cb.run_name))
options.logger.info("Terminating all services")
cb.block_until_complete(poll_period=0.1)
options.logger.info("Starting services back up")
cb = CloudInitD(options.database, db_name=dbname, log_level=options.loglevel, level_callback=level_callback, service_callback=service_callback, logdir=options.logdir, terminate=False, boot=True, ready=True, continue_on_error=False)
print_chars(1, "Booting all services %s\n" % (cb.run_name))
cb.start()
cb.block_until_complete(poll_period=0.1)
return 0
except CloudServiceException, svcex:
print svcex
except MultilevelException, mex:
print mex
def iceage(options, args):
if len(args) < 2:
print "The iceage command requires a run name. See --help"
return 1
dbname = args[1]
cb = CloudInitD(options.database, db_name=dbname, log_level=options.loglevel, logdir=options.logdir, terminate=False, boot=False, ready=True)
ha = cb.get_iaas_history()
print_chars(0, "ID \t:\tstate:\tassociated service\n")
for h in ha:
print_chars(1, "%s\t:\t%s\t:\t" % (h.get_id(), h.get_service_name()))
state = h.get_state()
clean = False
color = None
if state == "running":
color = "green"
clean = True
elif state == "terminated" or state == "shutting-down":
color="red"
elif state == "pending":
color="yellow"
clean = True
print_chars(1, ": %s\n" % (state), color=color)
if options.kill and clean:
print_chars(1, "Terminating %s\n" % (h.get_id()), bold=True)
h.terminate()
return 0
def _start_one(self, conf_file):
dir = tempfile.mkdtemp()
conf_file = self.plan_basedir + "/" + conf_file + "/top.conf"
cb = CloudInitD(dir, conf_file, terminate=False, boot=True, ready=True)
cb.start()
cb.block_until_complete(poll_period=1.0)
return (dir, cb)
def launch_new(options, args):
"""
Boot a new launch plan. You must supply the path to a top level configuration file. A run name will be displayed in the output. See --help for more information.
"""
if len(args) < 2:
print "The boot command requires a top level file. See --help"
return 1
config_file = args[1]
print_chars(1, "Starting up run ")
print_chars(1, "%s\n" % (options.name), inverse=True, color="green", bold=True)
cb = CloudInitD(options.database, log_level=options.loglevel, db_name=options.name, config_file=config_file, level_callback=level_callback, service_callback=service_callback, logdir=options.logdir, terminate=False, boot=True, ready=True, fail_if_db_present=True)
print_chars(3, "Logging to: %s%s.log\n" % (options.logdir, options.name))
if options.validate:
print_chars(1, "Validating the launch plan.\n")
errors = cb.boot_validate()
if len(errors) > 0:
print_chars(0, "The boot plan is not valid.\n", color = "red")
for (svc, ex) in errors:
print_chars(1, "Service %s had the error:\n" % (svc.name))
print_chars(1, "\t%s" %(str(ex)))
return 1
if options.dryrun:
test_env = _getenv_or_none('CLOUDINITD_TESTENV')
host_time_env = _getenv_or_none('CLOUDINITD_CBIAAS_TEST_HOSTNAME_TIME')
fab_env = _getenv_or_none('CLOUDINITD_FAB')
ssh_env = _getenv_or_none('CLOUDINITD_SSH')
print_chars(1, "Performing a dry run...\n", bold=True)
os.environ['CLOUDINITD_TESTENV'] = "2"
os.environ['CLOUDINITD_CBIAAS_TEST_HOSTNAME_TIME'] = "0.0"
os.environ['CLOUDINITD_FAB'] = cloudinitd.find_true()
os.environ['CLOUDINITD_SSH'] = cloudinitd.find_true()
try:
(rc, cb) = _launch_new(options, args, cb)
print_chars(1, "Dry run successful\n", bold=True, color="green")
finally:
_setenv_or_none('CLOUDINITD_TESTENV', test_env)
_setenv_or_none('CLOUDINITD_CBIAAS_TEST_HOSTNAME_TIME', host_time_env)
_setenv_or_none('CLOUDINITD_FAB', fab_env)
_setenv_or_none('CLOUDINITD_SSH', ssh_env)
if not options.noclean:
path = "%s/cloudinitd-%s.db" % (options.database, cb.run_name)
if not os.path.exists(path):
raise Exception("That DB does not seem to exist: %s" % (path))
os.remove(path)
return rc
(rc, cb) = _launch_new(options, args, cb)
return rc
def test_poll_to_soon_error(self):
self.plan_basedir = self.plan_basedir = cloudinitd.nosetests.g_plans_dir
dir = tempfile.mkdtemp()
conf_file = self.plan_basedir + "/simplelevels/top.conf"
cb = CloudInitD(dir, conf_file, terminate=False, boot=True, ready=True)
try:
cb.block_until_complete(poll_period=1.0)
self.fail("exception should have been thrown")
except APIUsageException, ex:
pass
def test_manyservices_one_vm_simple(self):
self.plan_basedir = cloudinitd.nosetests.g_plans_dir
dir = tempfile.mkdtemp()
conf_file = self.plan_basedir + "/singlevmmanyservice/top.conf"
cb = CloudInitD(dir, conf_file, terminate=False, boot=True, ready=True)
cb.start()
cb.block_until_complete(poll_period=1.0)
cb = CloudInitD(dir, db_name=cb.run_name, terminate=True, boot=False, ready=False)
cb.shutdown()
cb.block_until_complete(poll_period=1.0)
def load(p, c, m, run_name, cloudinitd_dbdir, silent=False, terminate=False, wholerun=True):
"""Load any EPU related instances from a local cloudinit.d launch with the same run name.
"""
try:
cb = CloudInitD(cloudinitd_dbdir, db_name=run_name, terminate=terminate, boot=False, ready=False)
cb.start()
cb.block_until_complete()
except APIUsageException, e:
raise IncompatibleEnvironment("Problem loading records from cloudinit.d: %s" % str(e))
def dep_not_found_test(self):
self.plan_basedir = cloudinitd.nosetests.g_plans_dir
dir = tempfile.mkdtemp()
conf_file = self.plan_basedir + "/oneservice/top.conf"
cb = CloudInitD(dir, conf_file, terminate=False, boot=True, ready=True)
cb.start()
ok = False
try:
cb.find_dep("notaservice", "whatever")
except APIUsageException, ex:
ok = True
def test_badlevel_bootpgm(self):
ilist_1 = self._get_running_vms()
count1 = len(ilist_1)
self.plan_basedir = cloudinitd.nosetests.g_plans_dir
dir = tempfile.mkdtemp()
conf_file = self.plan_basedir + "/badlevel2/top.conf"
pass_ex = True
cb = CloudInitD(dir, conf_file, terminate=False, boot=True, ready=True)
try:
cb.start()
cb.block_until_complete(poll_period=1.0)
except Exception, ex:
pass_ex = True
def connect_cloudinitd(self, must_exist=False):
config = os.path.join(os.path.abspath(self.plan), 'launch.conf')
home = os.environ['HOME']
db_file = os.path.join(home, '.cloudinitd', 'cloudinitd-'+self.name+'.db')
print '*** ' + db_file + ' must exist? ' + repr(must_exist)
if must_exist and not os.path.exists(db_file):
raise ApeException('cannot reconnect to cloudinitd -- launch does not exist')
elif os.path.exists(db_file):
self.util = CloudInitD(home + '/.cloudinitd', config_file=config, db_name=self.name, boot=False, ready=False, fail_if_db_present=False)
def _status(options, args):
global g_repair
dbname = args[1]
c_on_e = not g_repair
options.name = dbname
cb = CloudInitD(options.database, db_name=dbname, log_level=options.loglevel, level_callback=level_callback, service_callback=service_callback, logdir=options.logdir, terminate=False, boot=False, ready=True, continue_on_error=c_on_e)
print_chars(1, "Checking status on %s\n" % (cb.run_name))
cb.start()
try:
try:
cb.block_until_complete(poll_period=0.1)
except CloudServiceException, svcex:
print svcex
return 1
except MultilevelException, mex:
print mex
return 1
def test_only_one_launched(self):
if 'CLOUDINITD_CLEAN_ACCOUNT' not in os.environ:
raise SkipTest()
ilist_1 = self._get_running_vms()
count1 = len(ilist_1)
self.plan_basedir = cloudinitd.nosetests.g_plans_dir
dir = tempfile.mkdtemp()
conf_file = self.plan_basedir + "/singlevmmanyservice/top.conf"
cb = CloudInitD(dir, conf_file, terminate=False, boot=True, ready=True)
cb.start()
cb.block_until_complete(poll_period=1.0)
ilist_2 = self._get_running_vms()
count2 = len(ilist_2)
self.assertEqual(count1, count2 - 1, "there should be exactly 1 more VM in count 2: %d %d" % (count1, count2))
cb = CloudInitD(dir, db_name=cb.run_name, terminate=True, boot=False, ready=False)
cb.shutdown()
cb.block_until_complete(poll_period=1.0)
def check_status_shutdown_error_test(self):
(osf, outfile) = tempfile.mkstemp()
os.close(osf)
dir = os.path.expanduser("~/.cloudinitd/")
conf_file = self.plan_basedir + "/terminate/top.conf"
cb = CloudInitD(dir, conf_file, terminate=False, boot=True, ready=True)
cb.start()
cb.block_until_complete(poll_period=1.0)
runname = cb.run_name
svc = cb.get_service("sampleservice")
p = svc.shutdown()
rc = p.poll()
while not rc:
rc = p.poll()
time.sleep(0.1)
if 'CLOUDINITD_TESTENV' in os.environ:
bkfab = os.environ['CLOUDINITD_FAB']
bkssh = os.environ['CLOUDINITD_SSH']
os.environ['CLOUDINITD_FAB'] = "/bin/false"
os.environ['CLOUDINITD_SSH'] = "/bin/false"
rc = cloudinitd.cli.boot.main(["-O", outfile, "-v","-v","-v","-v", "status", runname])
if 'CLOUDINITD_TESTENV' in os.environ:
os.environ['CLOUDINITD_FAB'] = bkfab
os.environ['CLOUDINITD_SSH'] = bkssh
self._dump_output(outfile)
n = "ERROR"
line = self._find_str(outfile, n)
self.assertNotEqual(line, None)
rc = cloudinitd.cli.boot.main(["-O", outfile, "terminate", "%s" % (runname)])
self.assertEqual(rc, 0)
def check_repair_error_test(self):
if 'CLOUDINITD_TESTENV' in os.environ:
# we cannot run this one in fake mode yet
return
(osf, outfile) = tempfile.mkstemp()
os.close(osf)
dir = os.path.expanduser("~/.cloudinitd/")
conf_file = "%s/outputdep/top.conf" % (cloudinitd.nosetests.g_plans_dir)
cb = CloudInitD(dir, conf_file, terminate=False, boot=True, ready=True)
cb.start()
cb.block_until_complete(poll_period=1.0)
runname = cb.run_name
svc = cb.get_service("onelvl1")
secret = svc.get_attr_from_bag('iaas_secret')
key = svc.get_attr_from_bag('iaas_key')
iaas_url= svc.get_attr_from_bag('iaas_url')
instance_id = svc.get_attr_from_bag('instance_id')
con = iaas_get_con(svc._svc, key=key, secret=secret, iaasurl=iaas_url)
instance = con.find_instance(instance_id)
instance.terminate()
print "start repair"
rc = cloudinitd.cli.boot.main(["-O", outfile, "-v","-v","-v","repair", runname])
self._dump_output(outfile)
n = "ERROR"
line = self._find_str(outfile, n)
self.assertNotEqual(line, None)
print "start terminate"
rc = cloudinitd.cli.boot.main(["terminate", "%s" % (runname)])
self.assertEqual(rc, 0)
def reload_conf(options, args):
"""
Reload an updated launch plan configuration into the database of the run name supplied with --name. This is typically followed by a repair of the same run name.
"""
if len(args) < 2:
print "The reload command requires a top level file. See --help"
return 1
config_file = args[1]
print_chars(1, "Loading the launch plan for run ")
print_chars(1, "%s\n" % (options.name), inverse=True, color="green", bold=True)
cb = CloudInitD(options.database, log_level=options.loglevel, db_name=options.name, config_file=config_file, level_callback=level_callback, service_callback=service_callback, logdir=options.logdir, fail_if_db_present=False, terminate=False, boot=False, ready=False)
if options.validate:
print_chars(1, "Validating the launch plan.\n")
errors = cb.boot_validate()
if len(errors) > 0:
print_chars(0, "The boot plan is not valid.\n", color = "red")
for (svc, ex) in errors:
print_chars(1, "Service %s had the error:\n" % (svc.name))
print_chars(1, "\t%s" %(str(ex)))
return 1
return 0
def clean_ice(options, args):
"""
Clean all orphaned VMs
"""
if len(args) < 2:
print "The iceage command requires a run name. See --help"
return 1
dbname = args[1]
cb = CloudInitD(options.database, db_name=dbname, log_level=options.loglevel, logdir=options.logdir, terminate=False, boot=False, ready=True)
ha = cb.get_iaas_history()
for h in ha:
state = h.get_state()
handle = h.get_service_iaas_handle()
if state == "running":
if handle != h.get_id():
print_chars(2, "Terminating an orphaned VM %s\n" % (h.get_id()), bold=True)
h.terminate()
elif h.get_context_state() == cloudinitd.service_state_initial:
print_chars(2, "Terminating pre-staged VM %s\n" % (h.get_id()), bold=True)
h.terminate()
return 0
def terminate(options, args):
"""
Terminate an already booted plan. You must supply the run name of the booted plan.
"""
if len(args) < 2:
print "The terminate command requires a run name. See --help"
return 1
for dbname in args[1:]:
options.name = dbname
rc = 0
try:
cb = CloudInitD(options.database, log_level=options.loglevel, db_name=dbname, level_callback=level_callback, service_callback=service_callback, logdir=options.logdir, terminate=True, boot=False, ready=False, continue_on_error=True)
print_chars(1, "Terminating %s\n" % (cb.run_name))
cb.shutdown()
cb.block_until_complete(poll_period=0.1)
if not options.noclean:
path = "%s/cloudinitd-%s.db" % (options.database, dbname)
if not os.path.exists(path):
raise Exception("That DB does not seem to exist: %s" % (path))
if not options.safeclean or (cb.get_exception() is None and not cb.get_all_exceptions()):
print_chars(1, "Deleting the db file %s\n" % (path))
os.remove(path)
else:
print_chars(4, "There were errors when terminating %s, keeping db\n" % (cb.run_name))
ex = cb.get_exception()
if ex is None:
ex_list = cb.get_all_exceptions()
if ex_list:
ex = ex_list[-1]
if ex is not None:
print_chars(4, "An error occured %s" % (str(ex)))
raise ex
except CloudServiceException, svcex:
print svcex
rc = 1
except Exception, mex:
rc = 1
def setup(self):
"""
Build a fake test environment, with the sleepers cloudinit.d plan.
We can grab all logged messages from c.log.transcript.
"""
self.test_run_name = "TESTRUN"
self.config = ConfigParser.RawConfigParser()
self.config.add_section("events")
self.runlogdir = tempfile.mkdtemp()
self.config.set("events", "runlogdir", self.runlogdir)
self.vmlogdir = tempfile.mkdtemp()
self.config.set("events", "vmlogdir", self.vmlogdir)
self.optdict = {}
self.optdict[em_args.NAME.name] = self.test_run_name
self.params = DefaultParameters(self.config, None)
self.params.optdict = self.optdict
remote_svc_adapter = FakeRemoteSvcAdapter()
self.common = FakeCommon()
self.modules = FakeModules(remote_svc_adapter=remote_svc_adapter)
# Note that we monkey-patch the get_scp_command_str function
# to prepend "echo" to it. That way we can still allow the
# command to be run, but we can still see how it actually gets
# constructed
runlogs = DefaultRunlogs(self.params, self.common)
runlogs.validate()
self.modules.runlogs = runlogs
new_get_scp = make_fake_scp_command_str(runlogs, runlogs.get_scp_command_str)
self.modules.runlogs.get_scp_command_str = types.MethodType(new_get_scp, self.modules.runlogs)
self.test_dir = os.path.dirname(__file__)
self.test_db_dir = tempfile.mkdtemp()
self.test_cd_config = os.path.join(self.test_dir, "configs/main.conf")
self.cloudinitd = CloudInitD(self.test_db_dir, self.test_cd_config, self.test_run_name)
def check_status_error_test(self):
(osf, outfile) = tempfile.mkstemp()
os.close(osf)
dir = os.path.expanduser("~/.cloudinitd/")
conf_file = self.plan_basedir + "/terminate/top.conf"
cb = CloudInitD(dir, conf_file, terminate=False, boot=True, ready=True)
cb.start()
cb.block_until_complete(poll_period=1.0)
runname = cb.run_name
svc = cb.get_service("sampleservice")
secret = svc.get_attr_from_bag('iaas_secret')
key = svc.get_attr_from_bag('iaas_key')
iaas_url = svc.get_attr_from_bag('iaas_url')
instance_id = svc.get_attr_from_bag('instance_id')
con = iaas_get_con(None, key=key, secret=secret, iaasurl=iaas_url)
instance = con.find_instance(instance_id)
instance.terminate()
if 'CLOUDINITD_TESTENV' in os.environ:
bkfab = os.environ['CLOUDINITD_FAB']
bkssh = os.environ['CLOUDINITD_SSH']
os.environ['CLOUDINITD_FAB'] = "/bin/false"
os.environ['CLOUDINITD_SSH'] = "/bin/false"
rc = cloudinitd.cli.boot.main(["-O", outfile, "-v","-v","-v","-v", "status", runname])
if 'CLOUDINITD_TESTENV' in os.environ:
os.environ['CLOUDINITD_FAB'] = bkfab
os.environ['CLOUDINITD_SSH'] = bkssh
self._dump_output(outfile)
n = "ERROR"
line = self._find_str(outfile, n)
self.assertNotEqual(line, None)
rc = cloudinitd.cli.boot.main(["-O", outfile, "terminate", "%s" % (runname)])
if 'CLOUDINITD_TESTENV' in os.environ:
# in fake mode we cannot detect that an instance was killed
self.assertEqual(rc, 0)
else:
self.assertNotEqual(rc, 0)
class TestLogfetch:
def setup(self):
"""
Build a fake test environment, with the sleepers cloudinit.d plan.
We can grab all logged messages from c.log.transcript.
"""
self.test_run_name = "TESTRUN"
self.config = ConfigParser.RawConfigParser()
self.config.add_section("events")
self.runlogdir = tempfile.mkdtemp()
self.config.set("events", "runlogdir", self.runlogdir)
self.vmlogdir = tempfile.mkdtemp()
self.config.set("events", "vmlogdir", self.vmlogdir)
self.optdict = {}
self.optdict[em_args.NAME.name] = self.test_run_name
self.params = DefaultParameters(self.config, None)
self.params.optdict = self.optdict
remote_svc_adapter = FakeRemoteSvcAdapter()
self.common = FakeCommon()
self.modules = FakeModules(remote_svc_adapter=remote_svc_adapter)
# Note that we monkey-patch the get_scp_command_str function
# to prepend "echo" to it. That way we can still allow the
# command to be run, but we can still see how it actually gets
# constructed
runlogs = DefaultRunlogs(self.params, self.common)
runlogs.validate()
self.modules.runlogs = runlogs
new_get_scp = make_fake_scp_command_str(runlogs, runlogs.get_scp_command_str)
self.modules.runlogs.get_scp_command_str = types.MethodType(new_get_scp, self.modules.runlogs)
self.test_dir = os.path.dirname(__file__)
self.test_db_dir = tempfile.mkdtemp()
self.test_cd_config = os.path.join(self.test_dir, "configs/main.conf")
self.cloudinitd = CloudInitD(self.test_db_dir, self.test_cd_config, self.test_run_name)
def teardown(self):
shutil.rmtree(self.test_db_dir)
shutil.rmtree(self.vmlogdir)
shutil.rmtree(self.runlogdir)
def test_fetch_one_vm(self):
from epumgmt.main.em_core_logfetch import _fetch_one_vm
test_vm = epumgmt.api.RunVM()
_fetch_one_vm(self.params, self.common, self.modules,
self.test_run_name, test_vm, cloudinitd=self.cloudinitd)
def test_fetch_by_service_name(self):
"""
This test constructs a RunVM instance, and then asks
logfetch to grab its logs. We confirm that the correct
scp call was made indirectly by examining the transcript
of the log files.
We also neuter the scp call by prefixing it with echo, since
we're not trying to scp from a real host.
"""
from epumgmt.main.em_core_logfetch import fetch_by_service_name
test_service_name = "provisioner"
test_provisioner = epumgmt.api.RunVM()
test_provisioner.service_type = test_service_name
test_provisioner_hostname = "test.hostname.example.com"
test_provisioner.hostname = test_provisioner_hostname
test_provisioner_vmlogdir = "/some/fake/logdir"
test_provisioner.vmlogdir = test_provisioner_vmlogdir
test_provisioner_runlogdir = "/some/fake/local/runlogdir"
test_provisioner.runlogdir = test_provisioner_runlogdir
test_provisioner_instanceid = "i-TEST"
test_provisioner.instanceid = test_provisioner_instanceid
test_run_vms = []
test_run_vms.append(test_provisioner)
self.modules.persistence.store_run_vms(self.test_run_name, test_run_vms)
# Be tricky and patch in our hostname
self.cloudinitd.get_service("provisioner")._svc._s.hostname = test_provisioner_hostname
fetch_by_service_name(self.params, self.common, self.modules,
self.test_run_name, test_service_name, self.cloudinitd)
run_commands = [message for (level, message)
in self.common.log.transcript
if level == "DEBUG"
and "command =" in message]
# confirm that scp command gets called for our service
expected_scp_pattern = ".*@%s:%s %s" % (test_provisioner_hostname,
test_provisioner_vmlogdir,
test_provisioner_runlogdir)
# only expect one command to be run
assert len(run_commands) == 1
assert re.search(expected_scp_pattern, run_commands[0])
def test_fetch_all(self):
from epumgmt.main.em_core_logfetch import fetch_all
test_service_name = "provisioner"
test_provisioner = epumgmt.api.RunVM()
test_provisioner.service_type = test_service_name
test_provisioner_hostname = "test.hostname.example.com"
test_provisioner.hostname = test_provisioner_hostname
test_provisioner_vmlogdir = "/some/fake/logdir"
test_provisioner.vmlogdir = test_provisioner_vmlogdir
test_provisioner_runlogdir = "/some/fake/local/runlogdir"
test_provisioner.runlogdir = test_provisioner_runlogdir
test_provisioner_instanceid = "i-TEST"
test_provisioner.instanceid = test_provisioner_instanceid
# Be tricky and patch in our hostname
self.cloudinitd.get_service("provisioner")._svc._s.hostname = test_provisioner_hostname
# Two workers. Note that they have the same hostname
# to simulate the issue where we have a terminated worker
# and the second one was booted with the same hostname as
# the first
test_worker_0 = epumgmt.api.RunVM()
test_worker_0_service_type = "iamaworker"
test_worker_0.service_type = test_worker_0_service_type
test_worker_0_hostname = "worker0.example.com"
test_worker_0.hostname = test_worker_0_hostname
test_worker_0_instanceid = "i-TESTWORKER0"
test_worker_0.instanceid = test_worker_0_instanceid
test_worker_0_vmlogdir = "/some/fake/logdir"
test_worker_0.vmlogdir = test_worker_0_vmlogdir
test_worker_0_runlogdir = "/some/fake/%s/runlogdir" % test_worker_0_instanceid
test_worker_0.runlogdir = test_worker_0_runlogdir
test_worker_0_iaas_state = epustates.TERMINATED
test_worker_0_events = [Event(name="iaas_state", timestamp=1000, state=test_worker_0_iaas_state)]
test_worker_0.events = test_worker_0_events
test_worker_1 = epumgmt.api.RunVM()
test_worker_1_service_type = "iamaworker"
test_worker_1.service_type = test_worker_0_service_type
test_worker_1.hostname = test_worker_0_hostname
test_worker_1_instanceid = "i-TESTWORKER1"
test_worker_1.instanceid = test_worker_1_instanceid
test_worker_1_vmlogdir = "/some/fake/logdir"
test_worker_1.vmlogdir = test_worker_1_vmlogdir
test_worker_1_runlogdir = "/some/fake/%s/runlogdir/" % test_worker_1_instanceid
test_worker_1.runlogdir = test_worker_1_runlogdir
test_worker_1_iaas_state = epustates.RUNNING
test_worker_1_events = [Event(name="iaas_state", timestamp=1000, state=test_worker_1_iaas_state)]
test_worker_1.events = test_worker_1_events
test_run_vms = []
test_run_vms.append(test_provisioner)
test_run_vms.append(test_worker_0)
test_run_vms.append(test_worker_1)
self.modules.persistence.store_run_vms(self.test_run_name, test_run_vms)
fetch_all(self.params, self.common, self.modules, self.test_run_name, self.cloudinitd)
run_commands = [message for (level, message)
in self.common.log.transcript
if level == "DEBUG"
and "command =" in message]
# We have two VMs we should fetch from
assert len(run_commands) == 2
# Confirm that we never scp log files from the TERMINATED VM
for scp_command in run_commands:
assert scp_command.find(test_worker_0_instanceid) == -1
def test_prelaunch(self):
key = None
secret = None
url = None
try:
key = os.environ['CLOUDINITD_IAAS_ACCESS_KEY']
secret = os.environ['CLOUDINITD_IAAS_SECRET_KEY']
url = os.environ['CLOUDINITD_IAAS_URL']
except:
pass
# XXX this test may fail for nimbus
con = cloudinitd.cb_iaas.iaas_get_con(None, key=key, secret=secret, iaasurl=url)
i_list = con.get_all_instances()
conf_file = "multilevelsimple"
self.plan_basedir = cloudinitd.nosetests.g_plans_dir
dir = tempfile.mkdtemp()
conf_file = self.plan_basedir + "/" + conf_file + "/top.conf"
cb = CloudInitD(dir, conf_file, terminate=False, boot=True, ready=True)
cb.pre_start_iaas()
post_boot_list = con.get_all_instances()
self.assertNotEqual(len(i_list), len(post_boot_list), "The list of instances should have grown")
self.assertTrue(len(i_list)+3 < len(post_boot_list), "The list of instances should have grown by more than the number of services in the first level")
cb.start()
post_start_list = con.get_all_instances()
self.assertEqual(len(post_boot_list), len(post_start_list), "The list should not have grown")
cb.block_until_complete(poll_period=1.0)
cb = CloudInitD(dir, db_name=cb.run_name, terminate=True, boot=False, ready=False)
cb.shutdown()
cb.block_until_complete(poll_period=1.0)
fname = cb.get_db_file()
os.remove(fname)
def get_status_test(self):
self.plan_basedir = cloudinitd.nosetests.g_plans_dir
dir = tempfile.mkdtemp()
conf_file = self.plan_basedir + "/oneservice/top.conf"
cb = CloudInitD(dir, conf_file, terminate=False, boot=True, ready=True)
cb.start()
cb.block_until_complete(poll_period=1.0)
svc = cb.get_service("sampleservice")
status = svc.get_iaas_status()
self.assertEqual("running", status, "status is %s" % (status))
cb = CloudInitD(dir, db_name=cb.run_name, terminate=True, boot=False, ready=False)
cb.shutdown()
cb.block_until_complete(poll_period=1.0)
fname = cb.get_db_file()
os.remove(fname)
def dep_keys_test(self):
self.plan_basedir = cloudinitd.nosetests.g_plans_dir
dir = tempfile.mkdtemp()
conf_file = self.plan_basedir + "/oneservice/top.conf"
cb = CloudInitD(dir, conf_file, terminate=False, boot=True, ready=True)
cb.start()
cb.block_until_complete(poll_period=1.0)
svc = cb.get_service("sampleservice")
attr_keys = svc.get_keys_from_bag()
# until we figure out how to get __dict__ values from the sqlalchemy objects this will be in complete
expectations = [
"hostname",
"instance_id",
# "name",
# "level_id",
# "image",
# "iaas",
# "allocation",
# "keyname",
# "localkey",
# "username",
# "scp_username",
# "readypgm",
# "hostname",
# "bootconf",
# "bootpgm",
# "instance_id",
# "iaas_url",
# "iaas_key",
# "iaas_secret",
# "contextualized",
# "securitygroups",
"webmessage"
]
for e in expectations:
self.assertTrue(e in attr_keys, "The key %s should exist in %s" % (e, str(attr_keys)))
cb = CloudInitD(dir, db_name=cb.run_name, terminate=True, boot=False, ready=False)
cb.shutdown()
cb.block_until_complete(poll_period=1.0)
fname = cb.get_db_file()
os.remove(fname)
class Containers(object):
""" a set of ExecutionEngines
this object wraps the launch-plans and cloudinitd portions of a container launch
tmp/launch-plan/<pid>/plan/ result of generate-plan
cloud.yml modified copy of profiles/<template>.yml.example
launch.yml modified copy of res/launch/<template>.yml
"""
def __init__(self, config, couch=None, rabbit=None, graylog=None, elasticsearch=None):
self.config = config
self.target_dir = os.path.join('tmp', 'launch-plan', str(os.getpid()))
self.plan = os.path.join(self.target_dir, 'plan')
self.name = config.get('containers.name')
self.source_directory = config.get('containers.launch-plan')
self.couch = couch
self.rabbit = rabbit
self.graylog = graylog
self.elasticsearch = elasticsearch
# zookeeper
self.proc = None
def create_launch_plan(self):
os.makedirs(self.target_dir, mode=0755)
# make private copy of coi-services tarball
self._prepare_install_software()
# copy and modify configuration of cloud resources
self._modify_cloud_config()
# copy and modify configuration of services and execution engines
self._modify_launch()
# copy deploy.yml and modify
self._modify_deploy()
self._generate_plan()
def _prepare_install_software(self):
cmd = self.config.get('containers.software.copy-command')
if cmd:
log.debug('executing: %s', cmd)
subprocess.check_call(cmd, shell=True)
def _modify_cloud_config(self):
""" copy template cloud config file and modify """
file = os.path.join(self.source_directory, self.config.get('containers.resource-config'))
log.debug('configuring: %s', file)
with open(file, 'r') as f:
data = yaml.load(f)
# configure appropriately
data['iaas']['url'] = os.environ['EC2_URL']
data['iaas']['key'] = os.environ['EC2_ACCESS_KEY']
data['iaas']['secret'] = os.environ['EC2_SECRET_KEY']
data['iaas']['base-image'] = self.config.get('containers.image')
data['iaas']['base-allocation'] = self.config.get('containers.allocation')
data['rabbitmq']['host'] = self.config.get('rabbit.hostname')
data['rabbitmq']['username'] = self.config.get('rabbit.username')
data['rabbitmq']['password'] = self.config.get('rabbit.password')
data['couchdb']['host'] = self.config.get('couch.hostname')
data['couchdb']['username'] = self.config.get('couch.username')
data['couchdb']['password'] = self.config.get('couch.password')
data['graylog']['host'] = self.config.get('graylog.hostname')
# TODO: get from config file
data['zookeeper'] = {
'enabled': True,
'hosts': [ 'zk01.s.oceanobservatories.org', 'zk02.s.oceanobservatories.org', 'zk03.s.oceanobservatories.org']
}
url = self.config.get('containers.software.url')
if url:
if 'packages' not in data or not data['packages']:
data['packages'] = {}
data['packages']['coi_services'] = url
if self.config.get('containers.recipes'):
if 'packages' not in data or not data['packages']:
data['packages'] = {}
data['packages']['dt_data'] = self.config.get('containers.recipes')
self.cloud_config = os.path.join(self.target_dir, 'resources.yml')
with open(self.cloud_config, 'w') as f:
yaml.dump(data, f, default_flow_style=False)
def _modify_launch(self):
file = self.config.get('containers.cloud-config')
log.debug('configuring: %s', file)
with open(file, 'r') as f:
data = yaml.load(f)
all_engines_config = self.config.get('containers.execution-engines')
for name,engine_config in all_engines_config.iteritems():
if name not in data['execution_engines']:
# add new execution engine
data['execution_engines'][name] = {}
cfg = data['execution_engines'][name]
for key in 'slots', 'base_need', 'replicas':
value = engine_config.get(key)
if value:
cfg[key] = value
self.launch_config = os.path.join(self.target_dir, 'launch.yml')
with open(self.launch_config, 'w') as f:
yaml.dump(data, f, default_flow_style=False)
def _modify_deploy(self):
# read template
src = self.config.get('services.deploy-file')
log.debug('configuring: %s', src)
with open(src, 'r') as f:
data = yaml.load(f)
# keep subset of existing apps from deploy file
config_list = self.config.get('services.deploy-list')
config_apps = self.config.get('services.deploy-apps')
if config_list == '*' or (not config_list and not config_apps):
pass # use all apps from config file (default)
elif not config_list:
data['apps'] = []
elif isinstance(config_list, list):
orig_apps = data['apps']
data['apps'] = []
for name in config_list:
for app in orig_apps:
if app['name']==name:
data['apps'].append(app)
break
# add explicitly configured apps
if config_apps:
for app in config_apps:
data['apps'].append(app.as_dict())
# save
self.deploy_config = os.path.join(self.target_dir, 'deploy.yml')
with open(self.deploy_config, 'w') as f:
yaml.dump(data, f, default_flow_style=False)
# OBSOLETE: no longer run rel2levels, now run generate-plan
# # run rel2levels
# # HACK: have to execute with PYTHONPATH that includes YAML, so pass whatever version I'm using
# yaml_dir = os.path.dirname(os.path.dirname(yaml.__file__))
## cmd = 'export PYTHONPATH=$PYTHONPATH:' + yaml_dir + ' ; ./rel2levels.py -c ' + self.cloud_config + ' deploy.yml -f --ignore-bootlevels > /dev/null 2>&1'
# cmd = 'export PYTHONPATH=$PYTHONPATH:' + yaml_dir + ' ; ./rel2levels.py -c ' + self.cloud_config + ' deploy.yml -f > /dev/null 2>&1'
# code = subprocess.call(cmd, shell=True, cwd=self.launch_plan)
# if code<0:
# raise Exception('failed to execute ' + cmd)
def _generate_plan(self):
logconfig = self.config.get('containers.logging-config')
if logconfig:
cmd = 'bin/generate-plan --profile %s --rel %s --launch %s --logconfig %s --ignore-bootlevels %s' % (
os.path.abspath(self.cloud_config), os.path.abspath(self.deploy_config),
os.path.abspath(self.launch_config), os.path.abspath(logconfig), os.path.abspath(self.plan))
else:
cmd = 'bin/generate-plan --profile %s --rel %s --launch %s --ignore-bootlevels %s' % (
os.path.abspath(self.cloud_config), os.path.abspath(self.deploy_config),
os.path.abspath(self.launch_config), os.path.abspath(self.plan))
log.debug('executing: %s', cmd)
code = subprocess.call(cmd, shell=True, cwd=self.source_directory)
if code!=0:
raise ApeException('failed to execute ' + cmd)
def launch_containers(self):
# home = os.environ['HOME']
# file = os.path.join(self.launch_plan, self.cloud_config)
cmd='cloudinitd boot -vvv -n %s launch.conf' % self.name
# print '====>>> about to execute: ' + cmd
# HACK: cannot launch into BG and then connect -- CloudInitD will throw exception. so must always wait until launch completes instead
#self.proc = subprocess.Popen(cmd, shell=True, cwd=self.plan)
#status = self.proc.wait()
log.debug('executing: %s', cmd)
subprocess.check_call(cmd, shell=True, cwd=self.plan)
file = os.path.join(os.path.abspath(self.plan), 'launch.conf')
self.connect_cloudinitd()
def connect_cloudinitd(self, must_exist=False):
config = os.path.join(os.path.abspath(self.plan), 'launch.conf')
home = os.environ['HOME']
db_file = os.path.join(home, '.cloudinitd', 'cloudinitd-'+self.name+'.db')
print '*** ' + db_file + ' must exist? ' + repr(must_exist)
if must_exist and not os.path.exists(db_file):
raise ApeException('cannot reconnect to cloudinitd -- launch does not exist')
elif os.path.exists(db_file):
self.util = CloudInitD(home + '/.cloudinitd', config_file=config, db_name=self.name, boot=False, ready=False, fail_if_db_present=False)
def wait_for_containers(self):
print 'waiting...'
if self.proc:
self.proc.wait()
self.proc = None
# self.util.block_until_complete(poll_period=2)
def get_manager(self):
return SimpleManager(**self.get_nodes_broker())
def get_nodes_broker(self):
""" interrogate cloudinitd for rabbitmq parameters """
vars = {}
svc_list = self.util.get_all_services()
for svc in svc_list:
if svc.name == 'basenode':
vars['broker_hostname'] = svc.get_attr_from_bag("hostname")
vars['broker_username'] = svc.get_attr_from_bag("rabbitmq_username")
vars['broker_password'] = svc.get_attr_from_bag("rabbitmq_password")
return vars
def get_process_list(self):
cmd='ceictl -n %s -t 60 process list' % self.name
# print '====>>> about to execute: ' + cmd
# HACK: cannot launch into BG and then connect -- CloudInitD will throw exception. so must always wait until launch completes instead
#self.proc = subprocess.Popen(cmd, shell=True, cwd=self.plan)
#status = self.proc.wait()
processes = [ ]
current = { }
log.debug('executing: %s', cmd)
lines = iter(subprocess.check_output(cmd, shell=True).split('\n'))
try:
while True:
line = lines.next()
if line:
fields = line.split('=')
name = fields[0].strip()
value = fields[1].strip()
if name=='Process ID':
if current:
processes.append(current)
current = { }
elif name=='Process Name':
idlen = len(value.split('-')[-1])
type = value[0:-idlen-1]
current['type'] = type
current[name] = value
except StopIteration:
pass
return processes
def _status(self, dir, run_name):
cb = CloudInitD(dir, db_name=run_name, terminate=False, boot=False, ready=True, continue_on_error=True)
cb.start()
cb.block_until_complete(poll_period=1.0)
def test_validateiaas(self):
self.plan_basedir = cloudinitd.nosetests.g_plans_dir
dir = tempfile.mkdtemp()
conf_file = self.plan_basedir + "/iaastypevalidate/top.conf"
cb = CloudInitD(dir, conf_file, terminate=False, boot=True, ready=True)
cb.start()
cb.block_until_complete(poll_period=1.0)
# check the log for a warning
fname = os.path.expanduser("~/.cloudinitd/%s/badsvc.log" % (cb.run_name))
print fname
self.assertTrue(os.path.exists(fname), "The path %s should exist" % (fname))
f = open(fname, "r")
found = False
for l in f.readlines():
print l
ndx = l.find("WARN")
if ndx >= 0:
ndx = l.find("2.7")
if ndx >= 0:
found = True
self.assertTrue(found, "a warning with the key 2.7 should be in the logfile %s" %(fname))
f.close()
cb = CloudInitD(dir, db_name=cb.run_name, terminate=True, boot=False, ready=False)
cb.shutdown()
cb.block_until_complete(poll_period=1.0)
fname = cb.get_db_file()
os.remove(fname)
def _terminate(self, dir, run_name):
cb = CloudInitD(dir, db_name=run_name, terminate=True, boot=False, ready=False, continue_on_error=True)
cb.shutdown()
cb.block_until_complete(poll_period=1.0)
fname = cb.get_db_file()
os.remove(fname)
def test_badlevel_creds(self):
ilist_1 = self._get_running_vms()
count1 = len(ilist_1)
self.plan_basedir = cloudinitd.nosetests.g_plans_dir
dir = tempfile.mkdtemp()
conf_file = self.plan_basedir + "/badlevel2.2/top.conf"
pass_ex = True
cb = CloudInitD(dir, conf_file, terminate=False, boot=True, ready=True)
try:
cb.start()
cb.block_until_complete(poll_period=1.0)
except Exception, ex:
pass_ex = True
self.assertTrue(pass_ex, "An exception should have happened and didn't")
cb = CloudInitD(dir, db_name=cb.run_name, terminate=True, boot=False, ready=False)
cb.shutdown()
cb.block_until_complete(poll_period=1.0)
time.sleep(5)
ilist_2 = self._get_running_vms()
count2 = len(ilist_2)
self.assertEqual(count1, count2, "the vm count before and after should be the same: %d %d" % (count1, count2))
if __name__ == '__main__':
unittest.main()
def _start_one(self, conf_file):
self.plan_basedir = cloudinitd.nosetests.g_plans_dir
dir = tempfile.mkdtemp()
conf_file = self.plan_basedir + "/" + conf_file + "/top.conf"
cb = CloudInitD(dir, conf_file, terminate=False, boot=True, ready=True)
cb.start()
cb.block_until_complete(poll_period=1.0)
cb = CloudInitD(dir, db_name=cb.run_name, terminate=True, boot=False, ready=False)
cb.shutdown()
cb.block_until_complete(poll_period=1.0)
fname = cb.get_db_file()
os.remove(fname)
def test_env_set(self):
if cloudinitd.nosetests.is_a_test():
raise SkipTest()
dir = tempfile.mkdtemp()
conf_file = self.plan_basedir + "/oneservice/top.conf"
cb = CloudInitD(dir, conf_file, terminate=False, boot=True, ready=True)
cb.start()
cb.block_until_complete(poll_period=1.0)
svc = cb.get_service("sampleservice")
ssh_cmd = svc.get_ssh_command() + " ls -l %s" % (self.remote_dir)
# form test directory command
print ssh_cmd
rc = os.system(ssh_cmd)
self.assertEquals(rc, 0)
cb = CloudInitD(dir, db_name=cb.run_name, terminate=True, boot=False, ready=False)
cb.shutdown()
cb.block_until_complete(poll_period=1.0)
