def create_instance_from_saved_state():
saved_state = load_state()
cloud = saved_state['cloud']
specified_cloud = env.config.get('cloud')
if specified_cloud and specified_cloud != cloud:
log_red("The specified cloud: {} does not match the cloud "
"specified in the saved state file: {}".format(
specified_cloud, cloud))
sys.exit(1)
distro = Distribution(saved_state['distro'])
region = saved_state['region']
config = _get_platform_config(cloud, region, distro)
log_green('Reusing instance from saved state...')
instance_factory = _get_cloud_instance_factory(cloud)
instance = instance_factory.create_from_saved_state(
config, saved_state['state'])
log_green('...Done')
_setup_fab_for_instance(instance)
_save_state_from_instance(instance)
specified_distribution = env.config.get('distribution')
if (specified_distribution and
specified_distribution != instance.distro.value):
log_red("The specified distribution: {} does not match the distro "
"specified in the saved state file: {}".format(
specified_distribution, instance.distro.value))
sys.exit(1)
return instance
def _ensure_instance_running(self, instance_name):
"""
If an instance is terminated but still exists (hasn't been deleted
calling this will start the instance up again. Raises an error
if the instance no longer exists.
"""
try:
instance_info = (
self._compute.instances().get(project=self.project, zone=self.zone, instance=instance_name).execute()
)
if instance_info["status"] == "RUNNING":
pass
elif instance_info["status"] == "TERMINATED":
self._start_terminated_server(instance_name)
else:
msg = ("Instance {} is in state {}, " "please start it from the console").format(
instance_name, instance_info["status"]
)
raise Exception(msg)
# if we've started a terminated server, re-save
# the networking info, if we have
except HttpError as e:
if e.resp.status == 404:
log_red("Instance {} does not exist".format(instance_name))
log_yellow("you might need to remove state file.")
else:
log_red("Unknown error querying for instance {}".format(instance_name))
raise e
def _ensure_instance_running(self, instance_name):
"""
If an instance is terminated but still exists (hasn't been deleted
calling this will start the instance up again. Raises an error
if the instance no longer exists.
"""
try:
instance_info = self._compute.instances().get(
project=self.project, zone=self.zone,
instance=instance_name).execute()
if instance_info['status'] == 'RUNNING':
pass
elif instance_info['status'] == 'TERMINATED':
self._start_terminated_server(instance_name)
else:
msg = ("Instance {} is in state {}, "
"please start it from the console").format(
instance_name, instance_info['status'])
raise Exception(msg)
# if we've started a terminated server, re-save
# the networking info, if we have
except HttpError as e:
if e.resp.status == 404:
log_red("Instance {} does not exist".format(instance_name))
log_yellow("you might need to remove state file.")
else:
log_red("Unknown error querying for instance {}".format(
instance_name))
raise e
def create_ami(connection,
region,
instance_id,
name,
description,
block_device_mapping=None,
log=False):
ami = connection.create_image(instance_id,
name,
description,
block_device_mapping)
image_status = connection.get_image(ami)
while (image_status.state != "available" and
image_status.state != "failed"):
if log:
log_yellow('creating ami...')
sleep_for_one_minute()
image_status = connection.get_image(ami)
if image_status.state == "available":
if log:
log_green("ami %s %s" % (ami, image_status))
return(ami)
else:
if log:
log_red("ami %s %s" % (ami, image_status))
return False
def up_rackspace(region, access_key_id, secret_access_key, instance_id,
username):
""" boots an existing rackspace instance, or creates a new one if needed """
# if we don't have a state file, then its likely we need to create a new
# rackspace instance.
log_red('not implemented')
exit(1)
def down(self):
"""
Not raising an exception here because I'm a pushover, settle for
just yelling loudly at the user.
"""
log_red("Rackspace instances can't be downed, just keeping it running")
log_red("/Tableflip :( ")
def down(self):
"""
Not raising an exception here because I'm a pushover, settle for
just yelling loudly at the user.
"""
log_red("Rackspace instances can't be downed, just keeping it running")
log_red("/Tableflip :( ")
def does_container_exist(container):
with settings(warn_only=True):
result = sudo('docker inspect %s' % container)
print('*********************************************')
log_red(result.return_code)
if result.return_code is 0:
return True
else:
return False
def does_container_exist(container):
with settings(warn_only=True):
result = sudo('docker inspect %s' % container)
print('*********************************************')
log_red(result.return_code)
if result.return_code is 0:
return True
else:
return False
def _set_instance_networking(self, server):
ip_address = server.accessIPv4
if ip_address is None:
log_red('No IP address assigned')
exit(1)
self.state = self.state.transform(['ip_address'], ip_address)
wait_for_ssh(ip_address)
log_green(
'Connected to server with IP address {0}.'.format(ip_address))
def _set_instance_networking(self, server):
ip_address = server.accessIPv4
if ip_address is None:
log_red('No IP address assigned')
exit(1)
self.state = self.state.transform(['ip_address'], ip_address)
wait_for_ssh(ip_address)
log_green('Connected to server with IP address {0}.'.format(
ip_address)
)
def up_rackspace(region,
access_key_id,
secret_access_key,
instance_id,
username):
""" boots an existing rackspace instance, or creates a new one if needed """
# if we don't have a state file, then its likely we need to create a new
# rackspace instance.
log_red('not implemented')
exit(1)
def delete_image(self, image_id):
images = self.connection.get_all_images(owners='self')
found = False
for image in images:
if image.id == image_id:
log_yellow("Deleting image {}".format(image_id))
image.deregister(delete_snapshot=True)
found = True
break
if not found:
log_red("Could not find image {}".format(image_id))
def delete_image(self, image_id):
images = self.connection.get_all_images(owners='self')
found = False
for image in images:
if image.id == image_id:
log_yellow("Deleting image {}".format(image_id))
image.deregister(delete_snapshot=True)
found = True
break
if not found:
log_red("Could not find image {}".format(image_id))
def create_server_rackspace(connection,
distribution,
disk_name,
disk_size,
ami,
region,
key_pair,
instance_type,
instance_name,
tags={},
security_groups=None):
"""
Creates Rackspace Instance and saves it state in a local json file
"""
log_yellow("Creating Rackspace instance...")
flavor = connection.flavors.find(name=instance_type)
image = connection.images.find(name=ami)
server = connection.servers.create(name=instance_name,
flavor=flavor.id,
image=image.id,
region=region,
availability_zone=region,
key_name=key_pair)
while server.status == 'BUILD':
log_yellow("Waiting for build to finish...")
sleep(5)
server = connection.servers.get(server.id)
# check for errors
if server.status != 'ACTIVE':
log_red("Error creating rackspace instance")
exit(1)
# the server was assigned IPv4 and IPv6 addresses, locate the IPv4 address
ip_address = server.accessIPv4
if ip_address is None:
log_red('No IP address assigned')
exit(1)
wait_for_ssh(ip_address)
log_green('New server with IP address {0}.'.format(ip_address))
return server
def create_server_rackspace(connection,
distribution,
disk_name,
disk_size,
ami,
region,
key_pair,
instance_type,
instance_name,
tags={},
security_groups=None):
"""
Creates Rackspace Instance and saves it state in a local json file
"""
log_yellow("Creating Rackspace instance...")
flavor = connection.flavors.find(name=instance_type)
image = connection.images.find(name=ami)
server = connection.servers.create(name=instance_name,
flavor=flavor.id,
image=image.id,
region=region,
availability_zone=region,
key_name=key_pair)
while server.status == 'BUILD':
log_yellow("Waiting for build to finish...")
sleep(5)
server = connection.servers.get(server.id)
# check for errors
if server.status != 'ACTIVE':
log_red("Error creating rackspace instance")
exit(1)
# the server was assigned IPv4 and IPv6 addresses, locate the IPv4 address
ip_address = server.accessIPv4
if ip_address is None:
log_red('No IP address assigned')
exit(1)
wait_for_ssh(ip_address)
log_green('New server with IP address {0}.'.format(ip_address))
return server
def _connect_to_ec2(region, credentials):
"""
:param region: The region of AWS to connect to.
:param EC2Credentials credentials: The credentials to use to authenticate
with EC2.
:return: a connection object to AWS EC2
"""
conn = boto.ec2.connect_to_region(
region,
aws_access_key_id=credentials.access_key_id,
aws_secret_access_key=credentials.secret_access_key)
if conn:
return conn
else:
log_red('Failure to authenticate to EC2.')
return False
def _connect_to_ec2(region, credentials):
"""
:param region: The region of AWS to connect to.
:param EC2Credentials credentials: The credentials to use to authenticate
with EC2.
:return: a connection object to AWS EC2
"""
conn = boto.ec2.connect_to_region(
region,
aws_access_key_id=credentials.access_key_id,
aws_secret_access_key=credentials.secret_access_key
)
if conn:
return conn
else:
log_red('Failure to authenticate to EC2.')
return False
def get_ip_address_from_rackspace_server(connection, server_id):
"""
returns an ipaddress for a rackspace instance
"""
server = connection.servers.get(server_id)
# the server was assigned IPv4 and IPv6 addresses, locate the IPv4 address
ip_address = None
for network in server.networks['public']:
if re.match('\d+\.\d+\.\d+\.\d+', network):
ip_address = network
break
# find out if we have an ip address
if ip_address is None:
log_red('No IP address assigned')
return False
else:
return ip_address
def get_ip_address_from_rackspace_server(connection, server_id):
"""
returns an ipaddress for a rackspace instance
"""
server = connection.servers.get(server_id)
# the server was assigned IPv4 and IPv6 addresses, locate the IPv4 address
ip_address = None
for network in server.networks['public']:
if re.match('\d+\.\d+\.\d+\.\d+', network):
ip_address = network
break
# find out if we have an ip address
if ip_address is None:
log_red('No IP address assigned')
return False
else:
return ip_address
def create_image(self, image_name):
ami = self.connection.create_image(
self.state.instance_id,
image_name,
description=self.config.image_description,
)
image_status = self.connection.get_image(ami)
while (image_status.state != "available" and
image_status.state != "failed"):
log_yellow('creating ami...')
sleep(60)
image_status = self.connection.get_image(ami)
if image_status.state == "available":
log_green("ami %s %s" % (ami, image_status))
return(ami)
else:
log_red("ami %s %s" % (ami, image_status))
return False
def create_image(self, image_name):
ami = self.connection.create_image(
self.state.instance_id,
image_name,
description=self.config.image_description,
)
image_status = self.connection.get_image(ami)
while (image_status.state != "available"
and image_status.state != "failed"):
log_yellow('creating ami...')
sleep(60)
image_status = self.connection.get_image(ami)
if image_status.state == "available":
log_green("ami %s %s" % (ami, image_status))
return (ami)
else:
log_red("ami %s %s" % (ami, image_status))
return False
def create_rackspace_image(connection,
server_id,
name,
description,
block_device_mapping=None):
image_id = connection.servers.create_image(server_id, name)
image = connection.images.get(image_id).status.lower()
log_green('creating rackspace image...')
while connection.images.get(image_id).status.lower() not in ['active',
'error']:
log_green('building rackspace image...')
sleep_for_one_minute()
if image == 'error':
log_red('error creating image')
sys.exit(1)
log_green('finished image: %s' % image_id)
return image_id
def create_image(self, image_name):
server = self._nova.servers.find(name=self.state.instance_name)
image_id = self._nova.servers.create_image(server.id,
image_name=image_name)
image = self._nova.images.get(image_id).status.lower()
log_green('creating rackspace image...')
sleep_time = 20
elapsed = 0
while self._nova.images.get(image_id).status.lower() not in ['active',
'error']:
log_green('building rackspace image, '
'this could take a bit: elapsed {}s.'.format(elapsed))
sleep(20)
elapsed += sleep_time
if image == 'error':
log_red('error creating image')
exit(1)
log_green('finished image: %s' % image_id)
return image_id
def _create_server(self):
log_yellow("Creating Rackspace instance...")
flavor = self._nova.flavors.find(name=self.config.instance_type)
image = self._nova.images.find(name=self.config.ami)
server = self._nova.servers.create(name=self.state.instance_name,
flavor=flavor.id,
image=image.id,
region=self.state.region,
availability_zone=self.state.region,
key_name=self.config.key_pair)
while server.status == 'BUILD':
log_yellow("Waiting for build to finish...")
sleep(10)
server = self._nova.servers.get(server.id)
# check for errors
if server.status != 'ACTIVE':
log_red("Error creating rackspace instance")
exit(1)
self._set_instance_networking(server)
def create_image(self, image_name):
server = self._nova.servers.find(name=self.state.instance_name)
image_id = self._nova.servers.create_image(server.id,
image_name=image_name)
image = self._nova.images.get(image_id).status.lower()
log_green('creating rackspace image...')
sleep_time = 20
elapsed = 0
while self._nova.images.get(image_id).status.lower() not in [
'active', 'error'
]:
log_green('building rackspace image, '
'this could take a bit: elapsed {}s.'.format(elapsed))
sleep(20)
elapsed += sleep_time
if image == 'error':
log_red('error creating image')
exit(1)
log_green('finished image: %s' % image_id)
return image_id
def create_rackspace_image(connection,
server_id,
name,
description,
block_device_mapping=None):
image_id = connection.servers.create_image(server_id, name)
image = connection.images.get(image_id).status.lower()
log_green('creating rackspace image...')
while connection.images.get(image_id).status.lower() not in [
'active', 'error'
]:
log_green('building rackspace image...')
sleep_for_one_minute()
if image == 'error':
log_red('error creating image')
sys.exit(1)
log_green('finished image: %s' % image_id)
return image_id
def _create_server(self):
log_yellow("Creating Rackspace instance...")
flavor = self._nova.flavors.find(name=self.config.instance_type)
image = self._nova.images.find(name=self.config.ami)
server = self._nova.servers.create(
name=self.state.instance_name,
flavor=flavor.id,
image=image.id,
region=self.state.region,
availability_zone=self.state.region,
key_name=self.config.key_pair
)
while server.status == 'BUILD':
log_yellow("Waiting for build to finish...")
sleep(10)
server = self._nova.servers.get(server.id)
# check for errors
if server.status != 'ACTIVE':
log_red("Error creating rackspace instance")
exit(1)
self._set_instance_networking(server)
def jenkins_build(mesos_masters=[
('[email protected]', 'nixos-vagrant-configs/mesos-zk-01'),
('[email protected]', 'nixos-vagrant-configs/mesos-zk-02'),
('[email protected]', 'nixos-vagrant-configs/mesos-zk-03'),
],
mesos_slaves=[('[email protected]',
'nixos-vagrant-configs/slave')],
cleanup=True):
""" runs a jenkins build """
nodes = mesos_masters + mesos_slaves
# clean previous build logs
local('rm -f log/*')
def _provision_obor(nodes=nodes):
log_green('running _provision_obor')
local('chmod 600 nixos-vagrant-configs/vagrant.priv')
count = 1
while True or count > 3:
jobs = []
for node, hostdir in nodes:
jobs.append(
mp(target=local,
args=("fab -i nixos-vagrant-configs/vagrant.priv " +
"-H %s update:" % node +
"host_dir=%s," % hostdir + "rsync='yes'," +
"nix_gc='no'," + "nix_release='18.09'," +
"switch='no'" + "> log/`date '+%Y%m%d%H%M%S'`." +
"%s.provision.log 2>&1" % node, )))
for job in jobs:
job.start()
exit_code = 0
for job in jobs:
job.join()
exit_code = exit_code + job.exitcode
if exit_code == 0:
break
count = count + 1
log_green('_provision_obor completed')
def _test_obor(mesos_masters=mesos_masters, mesos_slaves=mesos_slaves):
log_green('running _test_obor')
obor_env = [
"eval `ssh-agent`",
"ssh-add $PWD/nixos-vagrant-configs/*.priv",
]
# local() doesn't support most context managers
# so let's bake a local environment file and consume as a prefix()
with open('shell_env', 'w') as shell_env:
for line in obor_env:
shell_env.write(line + '\n')
local('chmod +x shell_env')
local('chmod 600 nixos-vagrant-configs/vagrant.priv')
with settings(shell='/run/current-system/sw/bin/bash -l -c'):
with prefix(". ./shell_env"): # pylint: disable=not-context-manager
for target, _ in mesos_masters:
local(
"fab -i nixos-vagrant-configs/vagrant.priv " +
"-H {} acceptance_tests_mesos_master ".format(target) +
"> log/`date '+%Y%m%d%H%M%S'`."
"{}.test_obor.log 2>&1".format(target))
for target, _ in mesos_slaves:
local(
"fab -i nixos-vagrant-configs/vagrant.priv " +
"-H {} acceptance_tests_mesos_slave ".format(target) +
"> log/`date '+%Y%m%d%H%M%S'`."
"{}.test_obor.log 2>&1".format(target))
log_green('_test_obor completed')
def _flow1():
# spin up and provision the Cluster
local('vagrant up')
sleep(45) # allow VMs to boot up
_provision_obor()
def _flow2():
# spin up Railtrack, which is required for OBOR
spin_up_railtrack()
sleep(45) # allow VMs to boot up
provision_railtrack()
try:
p_flow1 = mp(target=_flow1)
p_flow2 = mp(target=_flow2)
p_flow1.start()
p_flow2.start()
p_flow1.join()
p_flow2.join()
# reload after initial provision
local('vagrant reload')
sleep(240) # allow the start services
# test all the things
_test_obor()
# and now destroy Railtrack and mesos VMs
if cleanup in [True, 'yes', 'y', 'Y', 'YES']:
clean()
except: # noqa: E722 pylint: disable=bare-except
log_red("jenkins_build() FAILED, aborting...")
if cleanup in [True, 'yes', 'y', 'Y', 'YES']:
clean()
sys.exit(1)
