def test_file_deployment(self):
# use this file (__file__) for obtaining permissions
target = os.path.join("/tmp", os.path.basename(__file__))
fd = FileDeployment(__file__, target)
self.assertEqual(target, fd.target)
self.assertEqual(__file__, fd.source)
self.assertEqual(self.node, fd.run(node=self.node, client=MockClient(hostname="localhost")))
def test_file_deployment(self):
# use this file (__file__) for obtaining permissions
target = os.path.join('/tmp', os.path.basename(__file__))
fd = FileDeployment(__file__, target)
self.assertEqual(target, fd.target)
self.assertEqual(__file__, fd.source)
self.assertEqual(self.node, fd.run(
node=self.node, client=MockClient(hostname='localhost')))
def generate_salt_key_for_minion(minion_hostname):
"""
Install key in pki and return FileDeployment
"""
command = "cd /tmp/ && salt-key --gen-keys=%s" % minion_hostname
run_local_command(command)
pem_key = FileDeployment("/tmp/%s.pem" % minion_hostname,
target="/etc/salt/pki/minion/%s.pem" %
minion_hostname)
public_key = FileDeployment("/tmp/%s.pub" % minion_hostname,
target="/etc/salt/pki/minion/%s.pub" %
minion_hostname)
return (pem_key, public_key)
def _create_config_file_deployment_step(file_path):
# type: (str) -> FileDeployment
"""
Create a FileDeployment class instance for a step which uploads a config file to the remote
server.
This function works by copying source config file to a temporary location, adding in Scalyr
API key and uploading this file to a remote server.
"""
temp_dir = tempfile.mkdtemp(prefix="ami-tests-configs")
file_name = os.path.basename(file_path)
temp_file_path = os.path.join(temp_dir, file_name)
# Copy file to a temporary directory
shutil.copyfile(file_path, temp_file_path)
# Add in Scalyr API key
with open(temp_file_path, "r") as fp:
content = fp.read()
content = content.replace("REPLACE_THIS", SCALYR_API_KEY)
with open(temp_file_path, "w") as fp:
fp.write("// AUTO GENERATED BY AMI TESTS SCRIPT\n")
fp.write(content)
target_path = "./{0}".format(file_name)
step = FileDeployment(temp_file_path, target_path)
return step
def _create_file_deployment_step(file_path, remote_file_name):
# type: (str, str) -> FileDeployment
"""
Create Libcloud file deployment step object.
"""
file_name = os.path.basename(file_path)
extension = os.path.splitext(file_name)[1]
target_path = "./{0}{1}".format(remote_file_name, extension)
step = FileDeployment(file_path, target_path)
return step
def water_machines(seed_profile, uuids):
"""
Bootstrap with salt
"""
nodes = []
if seed_profile.driver == 'aws':
driver = obtain_driver(seed_profile)
nodes = [i for i in driver.list_nodes() if i.name == seed_profile.name]
for libcloud_node in nodes:
logger = logging.getLogger('*'.join([__name__, libcloud_node.name]))
libcloud_node, private_ips = libcloud_node.driver.wait_until_running(
nodes=[libcloud_node], ssh_interface="private_ips")[0]
scripts = []
for script in seed_profile.init_scripts:
logger.warn("SCRIPT: %s" % script)
_file = FileDeployment(find_script(script),
target="/home/%s/%s" % (seed_profile.ami_user, script), )
scripts.append(_file)
msd = MultiStepDeployment(scripts)
deploy_msd_to_node(libcloud_node, msd, seed_profile.keypair['local_path'])
def deploy_msd_to_node(libcloud_node, msd, private_key_path=None):
##msd = MultiStepDeployment(Scripts from water_machines above)
logger.warn("TODO: REFACTOR AND TAKE OUT ec2-user literal")
seed_profile = settings.operation_profile
seed_profile = get_profile(seed_profile)
pkey = seed_profile.keypair['local_path']
ssh_client = SSHClient(hostname=libcloud_node.private_ip[0],
port=settings.SSH_PORT,
username='******',
password=None,
key=pkey,
timeout=int(settings.NETWORK_TIMEOUT),)
attempts = 0
dns_attempts = 0
##This begins a series of file placements for the masters subsequent deployment tasks in the init script.
while True:
time.sleep(5)
if seed_profile.profile_name == "salt_master":
dns_attempts += 1
logger.info("Number of attempts to connect: %s" % dns_attempts)
try:
logger.info("Attemping to connect to new node.")
ssh_client.connect()
logger.info("DNS SSH connection successful")
except Exception as error:
logger.info("DNS register ssh connection failed, trying again")
dns_attempts += 1
if dns_attempts > 10:
logger.error("DNS process failed to make a connection. Exiting.")
break
continue
# salt-cloud files necessary for deployment
for f in seed_profile.salt_cloud_vpc_files:
try:
cloud_files = FileDeployment(find_script(f),
target="/home/%s/%s" % (seed_profile.ami_user, os.path.basename(f)))
cloud_files.run(libcloud_node, ssh_client)
logger.info("salt-cloud file %s placed in home directory" % f)
except Exception as e:
logger.error("could not place salt-cloud file: %s" % e)
# places private key from path specified in keys.sh
try:
git_key = seed_profile.git_rsa_key
git_key_file = FileDeployment(git_key,
target= "/home/%s/%s" % (seed_profile.ami_user, os.path.basename(git_key)))
git_key_file.run(libcloud_node, ssh_client)
logger.info("Placed %s." % git_key_file.target)
except Exception as e:
logger.error("Could not place file: %s" % e)
# places DNS registration files for the master to add itself to route 53
try:
try_script = find_script(seed_profile.DNS_script)
dns_file = FileDeployment(try_script,
target="/home/%s/%s" % (seed_profile.ami_user, os.path.basename(try_script)) )
dns_file.run(libcloud_node, ssh_client)
logger.info("Placed %s ." % dns_file.target)
except Exception as e:
logger.error("Could not place file: %s" % e)
try:
dns_command = find_script(seed_profile.DNS_command)
domain = seed_profile.r53_domain
r53_key = seed_profile.r53_key
r53_secret = seed_profile.r53_secret
w_command = open(dns_command, 'w')
w_command.write("sudo python register_master_DNS.py '%(domain)s' '%(r53_key)s' '%(r53_secret)s'" %
{'domain': domain,
'r53_key': r53_key,
'r53_secret': r53_secret})
w_command.close()
c_deploy = FileDeployment(dns_command,
target="/home/%s/%s" % (seed_profile.ami_user,
os.path.basename(dns_command)) )
c_deploy.run(libcloud_node, ssh_client)
r_command = open(dns_command, 'w')
r_command.write("""
#This file get's blanked by the code to keep the keys out.\n
echo 'The DNS register command did not make it to this file.'""" )
r_command.close()
logger.info("The command file is in place")
break
except Exception as error:
logger.error("Deployment of the DNS register file failed: %s", error)
break
else:
print "%s isn't a master." % seed_profile.profile_name
logger.warn("%s isn't a master." % seed_profile.profile_name)
break
##This beings the deployment of init_scripts from water_machines
while True:
time.sleep(5)
try:
if ssh_client.connect() is True:
# Deploy files to libcloud_node
msd.run(libcloud_node, ssh_client)
pubkey_file = find_script("master_public_keys.sh")
ssh_key = get_public_key_from_file(pubkey_file)
ssh_key.run(libcloud_node, ssh_client)
for failed_step in msd.steps:
try:
execute_files_on_minion([failed_step], libcloud_node, ssh_client)
except socket_timeout, timeout:
logger.debug(timeout)
# We'll have to have the minion ping the master
# when it's alive and kicking so that we can confirm it's alive.
# maybe via a webhook.
# This happens when scripts you've implemented take to
# long to complete.
break
except Exception, error:
logger.error(error.message)
def _get_rubs(self, node, settings):
"""
Defines the set of actions to be done on a node
:param node: the node to be polished
:type node: :class:`libcloud.compute.base.Node`
:param settings: the fittings plan for this node
:type settings: ``dict``
:return: a list of actions to be performed, and related descriptions
:rtype: a ``list`` of `{ 'description': ..., 'genius': ... }``
"""
if not isinstance(settings, dict) or 'rub' not in settings:
return []
rubs = []
if self.key is not None:
rubs.append({
'description': 'deploy SSH public key',
'genius': SSHKeyDeployment(self.key)})
if settings['rub'] is not None:
for script in settings['rub']:
tokens = script.split(' ')
if len(tokens) == 1:
tokens.insert(0, 'run')
if tokens[0] == 'run':
script = tokens[1]
if len(tokens) > 2:
args = tokens[2:]
else:
args = None
try:
with open(os.path.dirname(__file__)+'/'+script) as stream:
text = stream.read()
if text:
rubs.append({
'description': ' '.join(tokens),
'genius': ScriptDeployment(script=text,
args=args,
name=script)})
except IOError:
raise PlumberyException("Error: cannot read '{}'"
.format(script))
elif tokens[0] == 'put':
file = tokens[1]
if len(tokens) > 2:
destination = tokens[2]
else:
destination = './'+file
try:
source = os.path.dirname(__file__)+'/'+file
with open(source) as stream:
text = stream.read()
if text:
rubs.append({
'description': ' '.join(tokens),
'genius': FileDeployment(source=source,
target=destination)})
except IOError:
raise PlumberyException("Error: cannot read '{}'"
.format(file))
else:
raise PlumberyException("Error: unknown directive '{}'"
.format(' '.join(tokens)))
return rubs
