def execute(self, email, cloud_id):
log.warn('Running list images for user %s cloud %s' % (email, cloud_id))
from mist.io import methods
user = user_from_email(email)
images = methods.list_images(user, cloud_id)
log.warn('Returning list images for user %s cloud %s' % (email, cloud_id))
return {'cloud_id': cloud_id, 'images': images}
def error_rerun_handler(self, exc, errors, email, backend_id):
from mist.io.methods import notify_user
if len(errors) < 6:
return self.result_fresh # Retry when the result is no longer fresh
user = user_from_email(email)
if len(errors) == 6: # If does not respond for a minute
notify_user(user, 'Backend %s does not respond' % user.backends[backend_id].title,
email_notify=False, backend_id=backend_id)
# Keep retrying for 30 minutes
times = [60, 60, 120, 300, 600, 600]
index = len(errors) - 6
if index < len(times):
return times[index]
else: # If backend still unresponsive disable it & notify user
with user.lock_n_load():
user.backends[backend_id].enabled = False
user.save()
notify_user(user, "Backend %s disabled after not responding for 30mins" % user.backends[backend_id].title,
email_notify=True, backend_id=backend_id)
log_event(user.email, 'incident', action='disable_backend',
backend_id=backend_id, error="Backend unresponsive")
def execute(self, email, cloud_id):
log.warn('Running list projects for user %s cloud %s' % (email, cloud_id))
from mist.io import methods
user = user_from_email(email)
projects = methods.list_projects(user, cloud_id)
log.warn('Returning list projects for user %s cloud %s' % (email, cloud_id))
return {'cloud_id': cloud_id, 'projects': projects}
def execute(self, email, backend_id):
log.warn('Running list networks for user %s backend %s' % (email, backend_id))
from mist.io import methods
user = user_from_email(email)
networks = methods.list_networks(user, backend_id)
log.warn('Returning list networks for user %s backend %s' % (email, backend_id))
return {'backend_id': backend_id, 'networks': networks}
def error_rerun_handler(self, exc, errors, email, backend_id):
if len(errors) < 6:
return self.result_fresh # Retry when the result is no longer fresh
user = user_from_email(email)
with user.lock_n_load():
user.backends[backend_id].enabled = False
user.save()
def execute(self, email, backend_id):
log.warn('Running list machines for user %s backend %s' % (email, backend_id))
from mist.io import methods
user = user_from_email(email)
machines = methods.list_machines(user, backend_id)
log.warn('Returning list machines for user %s backend %s' % (email, backend_id))
return {'backend_id': backend_id, 'machines': machines}
def deploy_collectd(email, backend_id, machine_id, extra_vars):
if not multi_user:
from mist.io.methods import deploy_collectd as deploy_collectd_method
else:
from mist.core.methods import deploy_collectd as deploy_collectd_method
user = user_from_email(email)
deploy_collectd_method(user, backend_id, machine_id, extra_vars)
def create_machine_async(email, backend_id, key_id, machine_name, location_id,
image_id, size_id, script, image_extra, disk,
image_name, size_name, location_name, ips, monitoring,
networks, docker_env, docker_command,
script_id=None, script_params=None,
quantity=1, persist=False, job_id=None,
docker_port_bindings={}, docker_exposed_ports={},
hostname='', plugins=None):
from multiprocessing.dummy import Pool as ThreadPool
from mist.io.methods import create_machine
from mist.io.exceptions import MachineCreationError
log.warn('MULTICREATE ASYNC %d' % quantity)
if multi_user:
from mist.core.helpers import log_event
else:
log_event = lambda *args, **kwargs: None
job_id = job_id or uuid.uuid4().hex
log_event(email, 'job', 'async_machine_creation_started', job_id=job_id,
backend_id=backend_id, script=script, script_id=script_id,
script_params=script_params, monitoring=monitoring,
persist=persist, quantity=quantity)
THREAD_COUNT = 5
pool = ThreadPool(THREAD_COUNT)
names = []
for i in range(1, quantity+1):
names.append('%s-%d' % (machine_name,i))
user = user_from_email(email)
specs = []
for name in names:
specs.append((
(user, backend_id, key_id, name, location_id, image_id,
size_id, script, image_extra, disk, image_name, size_name,
location_name, ips, monitoring, networks, docker_env,
docker_command, 22, script_id, script_params, job_id),
{'hostname': hostname, 'plugins': plugins}
))
def create_machine_wrapper(args_kwargs):
args, kwargs = args_kwargs
error = False
try:
node = create_machine(*args, **kwargs)
except MachineCreationError as exc:
error = str(exc)
except Exception as exc:
error = repr(exc)
finally:
name = args[3]
log_event(email, 'job', 'machine_creation_finished', job_id=job_id,
backend_id=backend_id, machine_name=name, error=error)
pool.map(create_machine_wrapper, specs)
pool.close()
pool.join()
def azure_post_create_steps(self, email, backend_id, machine_id, monitoring, command,
key_id, username, password, public_key):
from mist.io.methods import ssh_command, connect_provider, enable_monitoring
from mist.io.methods import notify_user, notify_admin
user = user_from_email(email)
try:
# find the node we're looking for and get its hostname
conn = connect_provider(user.backends[backend_id])
nodes = conn.list_nodes()
node = None
for n in nodes:
if n.id == machine_id:
node = n
break
if node and len(node.public_ips):
# filter out IPv6 addresses
ips = filter(lambda ip: ':' not in ip, node.public_ips)
host = ips[0]
else:
raise self.retry(exc=Exception(), countdown=120, max_retries=5)
try:
#login with user, password. Deploy the public key, enable sudo access for
#username, disable password authentication and reload ssh.
#After this is done, call post_deploy_steps if deploy script or monitoring
#is provided
ssh=paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host, username=username, password=password)
ssh.exec_command('mkdir -p ~/.ssh && echo "%s" >> ~/.ssh/authorized_keys && chmod -R 700 ~/.ssh/' % public_key)
chan = ssh.invoke_shell()
chan = ssh.get_transport().open_session()
chan.get_pty()
chan.exec_command('sudo su -c \'echo "%s ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers\' ' % username)
chan.send('%s\n' % password)
chan = ssh.invoke_shell()
chan = ssh.get_transport().open_session()
chan.get_pty()
chan.exec_command('sudo su -c \'echo "%s ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/waagent\' ' % username)
chan.send('%s\n' % password)
cmd = 'sudo su -c \'sed -i "s|[#]*PasswordAuthentication yes|PasswordAuthentication no|g" /etc/ssh/sshd_config && /etc/init.d/ssh reload; service ssh reload\' '
ssh.exec_command(cmd)
ssh.close()
if command or monitoring:
post_deploy_steps.delay(email, backend_id, machine_id,
monitoring, command, key_id)
except Exception as exc:
raise self.retry(exc=exc, countdown=60, max_retries=10)
except Exception as exc:
if str(exc).startswith('Retry'):
raise
def execute(self, email, backend_id):
log.warn("Running list images for user %s backend %s" % (email, backend_id))
from mist.io import methods
user = user_from_email(email)
images = methods.list_images(user, backend_id)
log.warn("Returning list images for user %s backend %s" % (email, backend_id))
return {"backend_id": backend_id, "images": images}
def execute(self, email, backend_id):
log.warn("Running list networks for user %s backend %s" % (email, backend_id))
from mist.io import methods
user = user_from_email(email)
networks = methods.list_networks(user, backend_id)
log.warn("Returning list networks for user %s backend %s" % (email, backend_id))
return {"backend_id": backend_id, "networks": networks}
def execute(self, email, backend_id, machine_id, host):
user = user_from_email(email)
from mist.io.methods import probe_ssh_only
res = probe_ssh_only(user, backend_id, machine_id, host)
return {'backend_id': backend_id,
'machine_id': machine_id,
'host': host,
'result': res}
def hpcloud_post_create_steps(self, email, backend_id, machine_id, monitoring,
command, key_id, username, password, public_key,
script_id='', script_params='', job_id=None,
hostname='', plugins=None,
post_script_id='', post_script_params=''):
from mist.io.methods import connect_provider
user = user_from_email(email)
try:
conn = connect_provider(user.backends[backend_id])
nodes = conn.list_nodes()
node = None
for n in nodes:
if n.id == machine_id:
node = n
break
if node and node.state == 0 and len(node.public_ips):
# filter out IPv6 addresses
ips = filter(lambda ip: ':' not in ip, node.public_ips)
host = ips[0]
post_deploy_steps.delay(
email, backend_id, machine_id, monitoring, command, key_id,
script_id=script_id, script_params=script_params,
job_id=job_id, hostname=hostname, plugins=plugins,
post_script_id=post_script_id,
post_script_params=post_script_params,
)
else:
try:
available_networks = conn.ex_list_networks()
external_networks = [net for net in available_networks if net.router_external]
if external_networks:
ext_net_id = external_networks[0].id
else:
ext_net_id = ""
ports = conn.ex_list_ports()
port = [port for port in ports if port.get("device_id", "") == node.id][0]
ip = conn.ex_create_floating_ip(ext_net_id, port['id'])
post_deploy_steps.delay(
email, backend_id, machine_id, monitoring, command, key_id,
script_id=script_id, script_params=script_params,
job_id=job_id, hostname=hostname, plugins=plugins,
post_script_id=post_script_id,
post_script_params=post_script_params,
)
except:
raise self.retry(exc=Exception(), max_retries=20)
except Exception as exc:
if str(exc).startswith('Retry'):
raise
def update_machine_count(email, backend_id, machine_count):
if not multi_user:
return
user = user_from_email(email)
with user.lock_n_load():
user.backends[backend_id].machine_count = machine_count
user.total_machine_count = sum([backend.machine_count for backend in user.backends.values()])
user.save()
def rackspace_first_gen_post_create_steps(
self, email, cloud_id, machine_id, monitoring, command, key_id,
password, public_key, username='******', script_id='', script_params='',
job_id=None, hostname='', plugins=None, post_script_id='',
post_script_params='', cronjob={}
):
from mist.io.methods import connect_provider
user = user_from_email(email)
try:
# find the node we're looking for and get its hostname
conn = connect_provider(user.clouds[cloud_id])
nodes = conn.list_nodes()
node = None
for n in nodes:
if n.id == machine_id:
node = n
break
if node and node.state == 0 and len(node.public_ips):
# filter out IPv6 addresses
ips = filter(lambda ip: ':' not in ip, node.public_ips)
host = ips[0]
else:
raise self.retry(exc=Exception(), max_retries=20)
try:
# login with user, password and deploy the ssh public key.
# Disable password authentication and reload ssh.
# After this is done, call post_deploy_steps
# if deploy script or monitoring
# is provided
ssh=paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host, username=username, password=password, timeout=None, allow_agent=False, look_for_keys=False)
ssh.exec_command('mkdir -p ~/.ssh && echo "%s" >> ~/.ssh/authorized_keys && chmod -R 700 ~/.ssh/' % public_key)
cmd = 'sudo su -c \'sed -i "s|[#]*PasswordAuthentication yes|PasswordAuthentication no|g" /etc/ssh/sshd_config && /etc/init.d/ssh reload; service ssh reload\' '
ssh.exec_command(cmd)
ssh.close()
post_deploy_steps.delay(
email, cloud_id, machine_id, monitoring, command, key_id,
script_id=script_id, script_params=script_params,
job_id=job_id, hostname=hostname, plugins=plugins,
post_script_id=post_script_id,
post_script_params=post_script_params, cronjob=cronjob
)
except Exception as exc:
raise self.retry(exc=exc, countdown=10, max_retries=15)
except Exception as exc:
if str(exc).startswith('Retry'):
raise
def ssh_command(email, backend_id, machine_id, host, command, key_id=None, username=None, password=None, port=22):
user = user_from_email(email)
shell = Shell(host)
key_id, ssh_user = shell.autoconfigure(user, backend_id, machine_id, key_id, username, password, port)
retval, output = shell.command(command)
shell.disconnect()
if retval:
from mist.io.methods import notify_user
notify_user(user, "Async command failed for machine %s (%s)" % (machine_id, host), output)
def run_deploy_script(self, email, backend_id, machine_id, command,
key_id=None, username=None, password=None, port=22):
from mist.io.methods import ssh_command, connect_provider
from mist.io.methods import notify_user, notify_admin
user = user_from_email(email)
try:
# find the node we're looking for and get its hostname
conn = connect_provider(user.backends[backend_id])
nodes = conn.list_nodes()
node = None
for n in nodes:
if n.id == machine_id:
node = n
break
if node and len(node.public_ips):
# filter out IPv6 addresses
ips = filter(lambda ip: ':' not in ip, node.public_ips)
host = ips[0]
else:
raise self.retry(exc=Exception(), countdown=60, max_retries=5)
try:
from mist.io.shell import Shell
shell = Shell(host)
key_id, ssh_user = shell.autoconfigure(user, backend_id, node.id,
key_id, username, password, port)
start_time = time()
retval, output = shell.command(command)
execution_time = time() - start_time
shell.disconnect()
msg = """
Command: %s
Return value: %s
Duration: %s seconds
Output:
%s""" % (command, retval, execution_time, output)
if retval:
notify_user(user, "Deployment script failed for machine %s (%s)" % (node.name, node.id), msg)
amqp_log("Deployment script failed for user %s machine %s (%s): %s" % (user, node.name, node.id, msg))
else:
notify_user(user, "Deployment script succeeded for machine %s (%s)" % (node.name, node.id), msg)
amqp_log("Deployment script succeeded for user %s machine %s (%s): %s" % (user, node.name, node.id, msg))
except ServiceUnavailableError as exc:
raise self.retry(exc=exc, countdown=60, max_retries=5)
except Exception as exc:
if str(exc).startswith('Retry'):
return
amqp_log("Deployment script failed for machine %s in backend %s by user %s after 5 retries: %s" % (node.id, backend_id, email, repr(exc)))
notify_user(user, "Deployment script failed for machine %s after 5 retries" % node.id)
notify_admin("Deployment script failed for machine %s in backend %s by user %s after 5 retries" % (node.id, backend_id, email), repr(exc))
def ssh_command(email, backend_id, machine_id, host, command,
key_id=None, username=None, password=None, port=22):
user = user_from_email(email)
shell = Shell(host)
key_id, ssh_user = shell.autoconfigure(user, backend_id, machine_id,
key_id, username, password, port)
retval, output = shell.command(command)
shell.disconnect()
if retval:
from mist.io.methods import notify_user
notify_user(user, "Async command failed for machine %s (%s)" %
(machine_id, host), output)
def execute(self, email, backend_id):
log.warn('Running list machines for user %s backend %s' % (email, backend_id))
from mist.io import methods
user = user_from_email(email)
machines = methods.list_machines(user, backend_id)
if multi_user:
for machine in machines:
kwargs = {}
kwargs['backend_id'] = backend_id
kwargs['machine_id'] = machine.get('id')
from mist.core.methods import list_tags
mistio_tags = list_tags(user, resource_type='machine', **kwargs)
# optimized for js
for tag in mistio_tags:
for key, value in tag.items():
tag_dict = {'key': key, 'value': value}
if tag_dict not in machine['tags']:
machine['tags'].append(tag_dict)
# FIXME: optimize!
log.warn('Returning list machines for user %s backend %s' % (email, backend_id))
return {'backend_id': backend_id, 'machines': machines}
def error_rerun_handler(self, exc, errors, email, cloud_id):
from mist.io.methods import notify_user
if len(errors) < 6:
return self.result_fresh # Retry when the result is no longer fresh
user = user_from_email(email)
if len(errors) == 6: # If does not respond for a minute
notify_user(user, 'Cloud %s does not respond'
% user.clouds[cloud_id].title,
email_notify=False, cloud_id=cloud_id)
# Keep retrying every 30 secs for 10 minutes, then every 60 secs for
# 20 minutes and finally every 20 minutes
times = [30]*20 + [60]*20
index = len(errors) - 6
if index < len(times):
return times[index]
else:
return 20*60
def error_rerun_handler(self, exc, errors, email, cloud_id):
from mist.io.methods import notify_user
if len(errors) < 6:
return self.result_fresh # Retry when the result is no longer fresh
user = user_from_email(email)
if len(errors) == 6: # If does not respond for a minute
notify_user(user,
'Cloud %s does not respond' %
user.clouds[cloud_id].title,
email_notify=False,
cloud_id=cloud_id)
# Keep retrying every 30 secs for 10 minutes, then every 60 secs for
# 20 minutes and finally every 20 minutes
times = [30] * 20 + [60] * 20
index = len(errors) - 6
if index < len(times):
return times[index]
else:
return 20 * 60
def execute(self, email, backend_id):
log.warn("Running list machines for user %s backend %s" % (email, backend_id))
from mist.io import methods
user = user_from_email(email)
machines = methods.list_machines(user, backend_id)
if multi_user:
for machine in machines:
kwargs = {}
kwargs["backend_id"] = backend_id
kwargs["machine_id"] = machine.get("id")
from mist.core.methods import list_tags
mistio_tags = list_tags(user, resource_type="machine", **kwargs)
# optimized for js
for tag in mistio_tags:
for key, value in tag.items():
tag_dict = {"key": key, "value": value}
if tag_dict not in machine["tags"]:
machine["tags"].append(tag_dict)
# FIXME: optimize!
log.warn("Returning list machines for user %s backend %s" % (email, backend_id))
return {"backend_id": backend_id, "machines": machines}
def create_machine_async(email,
cloud_id,
key_id,
machine_name,
location_id,
image_id,
size_id,
script,
image_extra,
disk,
image_name,
size_name,
location_name,
ips,
monitoring,
networks,
docker_env,
docker_command,
script_id='',
script_params='',
post_script_id='',
post_script_params='',
quantity=1,
persist=False,
job_id=None,
docker_port_bindings={},
docker_exposed_ports={},
azure_port_bindings='',
hostname='',
plugins=None,
disk_size=None,
disk_path=None,
cloud_init='',
associate_floating_ip=False,
associate_floating_ip_subnet=None,
project_id=None):
from multiprocessing.dummy import Pool as ThreadPool
from mist.io.methods import create_machine
from mist.io.exceptions import MachineCreationError
log.warn('MULTICREATE ASYNC %d' % quantity)
if multi_user:
from mist.core.helpers import log_event
else:
log_event = lambda *args, **kwargs: None
job_id = job_id or uuid.uuid4().hex
log_event(email,
'job',
'async_machine_creation_started',
job_id=job_id,
cloud_id=cloud_id,
script=script,
script_id=script_id,
script_params=script_params,
monitoring=monitoring,
persist=persist,
quantity=quantity)
THREAD_COUNT = 5
pool = ThreadPool(THREAD_COUNT)
names = []
for i in range(1, quantity + 1):
names.append('%s-%d' % (machine_name, i))
user = user_from_email(email)
specs = []
for name in names:
specs.append(
((user, cloud_id, key_id, name, location_id, image_id, size_id,
script, image_extra, disk, image_name, size_name, location_name,
ips, monitoring, networks, docker_env, docker_command, 22,
script_id, script_params, job_id), {
'hostname': hostname,
'plugins': plugins,
'post_script_id': post_script_id,
'post_script_params': post_script_params,
'azure_port_bindings': azure_port_bindings,
'associate_floating_ip': associate_floating_ip,
'cloud_init': cloud_init,
'disk_size': disk_size,
'disk_path': disk_path,
'project_id': project_id
}))
def create_machine_wrapper(args_kwargs):
args, kwargs = args_kwargs
error = False
try:
node = create_machine(*args, **kwargs)
except MachineCreationError as exc:
error = str(exc)
except Exception as exc:
error = repr(exc)
finally:
name = args[3]
log_event(email,
'job',
'machine_creation_finished',
job_id=job_id,
cloud_id=cloud_id,
machine_name=name,
error=error)
pool.map(create_machine_wrapper, specs)
pool.close()
pool.join()
def execute(self, email, cloud_id):
from mist.io import methods
user = user_from_email(email)
sizes = methods.list_sizes(user, cloud_id)
return {'cloud_id': cloud_id, 'sizes': sizes}
def execute(self, email, backend_id):
from mist.io import methods
user = user_from_email(email)
networks = methods.list_networks(user, backend_id)
return {'backend_id': backend_id, 'networks': networks}
def post_deploy_steps(self, email, backend_id, machine_id, monitoring, command,
key_id=None, username=None, password=None, port=22,
script_id='', script_params='', job_id=None,
hostname='', plugins=None,
post_script_id='', post_script_params=''):
from mist.io.methods import connect_provider, probe_ssh_only
from mist.io.methods import notify_user, notify_admin
from mist.io.methods import create_dns_a_record
if multi_user:
from mist.core.methods import enable_monitoring
from mist.core.tasks import run_script
from mist.core.helpers import log_event
else:
from mist.io.methods import enable_monitoring
log_event = lambda *args, **kwargs: None
job_id = job_id or uuid.uuid4().hex
user = user_from_email(email)
tmp_log = lambda msg, *args: log.error('Post deploy: %s' % msg, *args)
tmp_log('Entering post deploy steps for %s %s %s',
user.email, backend_id, machine_id)
try:
# find the node we're looking for and get its hostname
node = None
try:
conn = connect_provider(user.backends[backend_id])
nodes = conn.list_nodes() # TODO: use cache
for n in nodes:
if n.id == machine_id:
node = n
break
tmp_log('run list_machines')
except:
raise self.retry(exc=Exception(), countdown=10, max_retries=10)
if node and len(node.public_ips):
# filter out IPv6 addresses
ips = filter(lambda ip: ':' not in ip, node.public_ips)
host = ips[0]
else:
tmp_log('ip not found, retrying')
raise self.retry(exc=Exception(), countdown=60, max_retries=20)
try:
from mist.io.shell import Shell
shell = Shell(host)
# connect with ssh even if no command, to create association
# to be able to enable monitoring
tmp_log('attempting to connect to shell')
key_id, ssh_user = shell.autoconfigure(
user, backend_id, node.id, key_id, username, password, port
)
tmp_log('connected to shell')
result = probe_ssh_only(user, backend_id, machine_id, host=None,
key_id=key_id, ssh_user=ssh_user, shell=shell)
log_dict = {
'email': email,
'event_type': 'job',
'backend_id': backend_id,
'machine_id': machine_id,
'job_id': job_id,
'host': host,
'key_id': key_id,
'ssh_user': ssh_user,
}
log_event(action='probe', result=result, **log_dict)
backend = user.backends[backend_id]
msg = "Backend:\n Name: %s\n Id: %s\n" % (backend.title,
backend_id)
msg += "Machine:\n Name: %s\n Id: %s\n" % (node.name,
node.id)
if hostname:
try:
record = create_dns_a_record(user, hostname, host)
hostname = '.'.join((record.name, record.zone.domain))
log_event(action='create_dns_a_record', hostname=hostname,
**log_dict)
except Exception as exc:
log_event(action='create_dns_a_record', error=str(exc),
**log_dict)
error = False
if script_id and multi_user:
tmp_log('will run script_id %s', script_id)
ret = run_script.run(
user.email, script_id, backend_id, machine_id,
params=script_params, host=host, job_id=job_id
)
error = ret['error']
tmp_log('executed script_id %s', script_id)
elif command:
tmp_log('will run command %s', command)
log_event(action='deployment_script_started', command=command, **log_dict)
start_time = time()
retval, output = shell.command(command)
tmp_log('executed command %s', command)
execution_time = time() - start_time
output = output.decode('utf-8','ignore')
title = "Deployment script %s" % ('failed' if retval
else 'succeeded')
error = retval > 0
notify_user(user, title,
backend_id=backend_id,
machine_id=machine_id,
machine_name=node.name,
command=command,
output=output,
duration=execution_time,
retval=retval,
error=retval > 0)
log_event(action='deployment_script_finished',
error=retval > 0,
return_value=retval,
command=command,
stdout=output,
**log_dict)
shell.disconnect()
if monitoring:
try:
enable_monitoring(user, backend_id, node.id,
name=node.name, dns_name=node.extra.get('dns_name',''),
public_ips=ips, no_ssh=False, dry=False, job_id=job_id,
plugins=plugins, deploy_async=False,
)
except Exception as e:
print repr(e)
error = True
notify_user(user, "Enable monitoring failed for machine %s" % machine_id, repr(e))
notify_admin('Enable monitoring on creation failed for user %s machine %s: %r' % (email, machine_id, e))
log_event(action='enable_monitoring_failed', error=repr(e),
**log_dict)
if post_script_id and multi_user:
tmp_log('will run post_script_id %s', post_script_id)
ret = run_script.run(
user.email, post_script_id, backend_id, machine_id,
params=post_script_params, host=host, job_id=job_id,
action_prefix='post_',
)
error = ret['error']
tmp_log('executed post_script_id %s', script_id)
log_event(action='post_deploy_finished', error=error, **log_dict)
except (ServiceUnavailableError, SSHException) as exc:
tmp_log(repr(exc))
raise self.retry(exc=exc, countdown=60, max_retries=15)
except Exception as exc:
tmp_log(repr(exc))
if str(exc).startswith('Retry'):
raise
notify_user(user, "Deployment script failed for machine %s" % machine_id)
notify_admin("Deployment script failed for machine %s in backend %s by user %s" % (machine_id, backend_id, email), repr(exc))
log_event(
email=email,
event_type='job',
action='post_deploy_finished',
backend_id=backend_id,
machine_id=machine_id,
enable_monitoring=bool(monitoring),
command=command,
error="Couldn't connect to run post deploy steps.",
job_id=job_id
)
def deploy_collectd(email, backend_id, machine_id, extra_vars):
import mist.io.methods
user = user_from_email(email)
mist.io.methods.deploy_collectd(user, backend_id, machine_id, extra_vars)
def execute(self, email, backend_id):
from mist.io import methods
user = user_from_email(email)
locations = methods.list_locations(user, backend_id)
return {'backend_id': backend_id, 'locations': locations}
def undeploy_collectd(email, backend_id, machine_id):
user = user_from_email(email)
import mist.io.methods
mist.io.methods.undeploy_collectd(user, backend_id, machine_id)
def openstack_post_create_steps(self,
email,
cloud_id,
machine_id,
monitoring,
command,
key_id,
username,
password,
public_key,
script_id='',
script_params='',
job_id=None,
hostname='',
plugins=None,
post_script_id='',
post_script_params='',
networks=[]):
from mist.io.methods import connect_provider
user = user_from_email(email)
try:
conn = connect_provider(user.clouds[cloud_id])
nodes = conn.list_nodes()
node = None
for n in nodes:
if n.id == machine_id:
node = n
break
if node and node.state == 0 and len(node.public_ips):
# filter out IPv6 addresses
ips = filter(lambda ip: ':' not in ip, node.public_ips)
host = ips[0]
post_deploy_steps.delay(
email,
cloud_id,
machine_id,
monitoring,
command,
key_id,
script_id=script_id,
script_params=script_params,
job_id=job_id,
hostname=hostname,
plugins=plugins,
post_script_id=post_script_id,
post_script_params=post_script_params,
)
else:
try:
created_floating_ips = []
for network in networks['public']:
created_floating_ips += [
floating_ip for floating_ip in network['floating_ips']
]
# From the already created floating ips try to find one that is not associated to a node
unassociated_floating_ip = None
for ip in created_floating_ips:
if not ip['node_id']:
unassociated_floating_ip = ip
break
# Find the ports which are associated to the machine (e.g. the ports of the private ips)
# and use one to associate a floating ip
ports = conn.ex_list_ports()
machine_port_id = None
for port in ports:
if port.get('device_id') == node.id:
machine_port_id = port.get('id')
break
if unassociated_floating_ip:
log.info("Associating floating ip with machine: %s" %
node.id)
ip = conn.ex_associate_floating_ip_to_node(
unassociated_floating_ip['id'], machine_port_id)
else:
# Find the external network
log.info(
"Create and associating floating ip with machine: %s" %
node.id)
ext_net_id = networks['public'][0]['id']
ip = conn.ex_create_floating_ip(ext_net_id,
machine_port_id)
post_deploy_steps.delay(
email,
cloud_id,
machine_id,
monitoring,
command,
key_id,
script_id=script_id,
script_params=script_params,
job_id=job_id,
hostname=hostname,
plugins=plugins,
post_script_id=post_script_id,
post_script_params=post_script_params,
)
except:
raise self.retry(exc=Exception(), max_retries=20)
except Exception as exc:
if str(exc).startswith('Retry'):
raise
def azure_post_create_steps(self,
email,
cloud_id,
machine_id,
monitoring,
command,
key_id,
username,
password,
public_key,
script_id='',
script_params='',
job_id=None,
hostname='',
plugins=None,
post_script_id='',
post_script_params=''):
from mist.io.methods import connect_provider
user = user_from_email(email)
try:
# find the node we're looking for and get its hostname
conn = connect_provider(user.clouds[cloud_id])
nodes = conn.list_nodes()
node = None
for n in nodes:
if n.id == machine_id:
node = n
break
if node and node.state == 0 and len(node.public_ips):
# filter out IPv6 addresses
ips = filter(lambda ip: ':' not in ip, node.public_ips)
host = ips[0]
else:
raise self.retry(exc=Exception(), max_retries=20)
try:
# login with user, password. Deploy the public key, enable sudo access for
# username, disable password authentication and reload ssh.
# After this is done, call post_deploy_steps if deploy script or monitoring
# is provided
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host,
username=username,
password=password,
timeout=None,
allow_agent=False,
look_for_keys=False)
ssh.exec_command(
'mkdir -p ~/.ssh && echo "%s" >> ~/.ssh/authorized_keys && chmod -R 700 ~/.ssh/'
% public_key)
chan = ssh.get_transport().open_session()
chan.get_pty()
chan.exec_command(
'sudo su -c \'echo "%s ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers\' '
% username)
chan.send('%s\n' % password)
check_sudo_command = 'sudo su -c \'whoami\''
chan = ssh.get_transport().open_session()
chan.get_pty()
chan.exec_command(check_sudo_command)
output = chan.recv(1024)
if not output.startswith('root'):
raise
cmd = 'sudo su -c \'sed -i "s|[#]*PasswordAuthentication yes|PasswordAuthentication no|g" /etc/ssh/sshd_config && /etc/init.d/ssh reload; service ssh reload\' '
ssh.exec_command(cmd)
ssh.close()
post_deploy_steps.delay(
email,
cloud_id,
machine_id,
monitoring,
command,
key_id,
script_id=script_id,
script_params=script_params,
job_id=job_id,
hostname=hostname,
plugins=plugins,
post_script_id=post_script_id,
post_script_params=post_script_params,
)
except Exception as exc:
raise self.retry(exc=exc, countdown=10, max_retries=15)
except Exception as exc:
if str(exc).startswith('Retry'):
raise
def post_deploy_steps(self,
email,
cloud_id,
machine_id,
monitoring,
command,
key_id=None,
username=None,
password=None,
port=22,
script_id='',
script_params='',
job_id=None,
hostname='',
plugins=None,
post_script_id='',
post_script_params=''):
from mist.io.methods import connect_provider, probe_ssh_only
from mist.io.methods import notify_user, notify_admin
from mist.io.methods import create_dns_a_record
if multi_user:
from mist.core.methods import enable_monitoring
from mist.core.tasks import run_script
from mist.core.helpers import log_event
else:
from mist.io.methods import enable_monitoring
log_event = lambda *args, **kwargs: None
job_id = job_id or uuid.uuid4().hex
user = user_from_email(email)
tmp_log = lambda msg, *args: log.error('Post deploy: %s' % msg, *args)
tmp_log('Entering post deploy steps for %s %s %s', user.email, cloud_id,
machine_id)
try:
# find the node we're looking for and get its hostname
node = None
try:
conn = connect_provider(user.clouds[cloud_id])
nodes = conn.list_nodes() # TODO: use cache
for n in nodes:
if n.id == machine_id:
node = n
break
tmp_log('run list_machines')
except:
raise self.retry(exc=Exception(), countdown=10, max_retries=10)
if node and len(node.public_ips):
# filter out IPv6 addresses
ips = filter(lambda ip: ':' not in ip, node.public_ips)
host = ips[0]
else:
tmp_log('ip not found, retrying')
raise self.retry(exc=Exception(), countdown=60, max_retries=20)
try:
from mist.io.shell import Shell
shell = Shell(host)
# connect with ssh even if no command, to create association
# to be able to enable monitoring
tmp_log('attempting to connect to shell')
key_id, ssh_user = shell.autoconfigure(user, cloud_id, node.id,
key_id, username, password,
port)
tmp_log('connected to shell')
result = probe_ssh_only(user,
cloud_id,
machine_id,
host=None,
key_id=key_id,
ssh_user=ssh_user,
shell=shell)
log_dict = {
'email': email,
'event_type': 'job',
'cloud_id': cloud_id,
'machine_id': machine_id,
'job_id': job_id,
'host': host,
'key_id': key_id,
'ssh_user': ssh_user,
}
log_event(action='probe', result=result, **log_dict)
cloud = user.clouds[cloud_id]
msg = "Cloud:\n Name: %s\n Id: %s\n" % (cloud.title, cloud_id)
msg += "Machine:\n Name: %s\n Id: %s\n" % (node.name, node.id)
if hostname:
try:
record = create_dns_a_record(user, hostname, host)
hostname = '.'.join((record.name, record.zone.domain))
log_event(action='create_dns_a_record',
hostname=hostname,
**log_dict)
except Exception as exc:
log_event(action='create_dns_a_record',
error=str(exc),
**log_dict)
error = False
if script_id and multi_user:
tmp_log('will run script_id %s', script_id)
ret = run_script.run(user.email,
script_id,
cloud_id,
machine_id,
params=script_params,
host=host,
job_id=job_id)
error = ret['error']
tmp_log('executed script_id %s', script_id)
elif command:
tmp_log('will run command %s', command)
log_event(action='deployment_script_started',
command=command,
**log_dict)
start_time = time()
retval, output = shell.command(command)
tmp_log('executed command %s', command)
execution_time = time() - start_time
output = output.decode('utf-8', 'ignore')
title = "Deployment script %s" % ('failed'
if retval else 'succeeded')
error = retval > 0
notify_user(user,
title,
cloud_id=cloud_id,
machine_id=machine_id,
machine_name=node.name,
command=command,
output=output,
duration=execution_time,
retval=retval,
error=retval > 0)
log_event(action='deployment_script_finished',
error=retval > 0,
return_value=retval,
command=command,
stdout=output,
**log_dict)
shell.disconnect()
if monitoring:
try:
enable_monitoring(
user,
cloud_id,
node.id,
name=node.name,
dns_name=node.extra.get('dns_name', ''),
public_ips=ips,
no_ssh=False,
dry=False,
job_id=job_id,
plugins=plugins,
deploy_async=False,
)
except Exception as e:
print repr(e)
error = True
notify_user(
user,
"Enable monitoring failed for machine %s" % machine_id,
repr(e))
notify_admin(
'Enable monitoring on creation failed for user %s machine %s: %r'
% (email, machine_id, e))
log_event(action='enable_monitoring_failed',
error=repr(e),
**log_dict)
if post_script_id and multi_user:
tmp_log('will run post_script_id %s', post_script_id)
ret = run_script.run(
user.email,
post_script_id,
cloud_id,
machine_id,
params=post_script_params,
host=host,
job_id=job_id,
action_prefix='post_',
)
error = ret['error']
tmp_log('executed post_script_id %s', script_id)
log_event(action='post_deploy_finished', error=error, **log_dict)
except (ServiceUnavailableError, SSHException) as exc:
tmp_log(repr(exc))
raise self.retry(exc=exc, countdown=60, max_retries=15)
except Exception as exc:
tmp_log(repr(exc))
if str(exc).startswith('Retry'):
raise
notify_user(user,
"Deployment script failed for machine %s" % machine_id)
notify_admin(
"Deployment script failed for machine %s in cloud %s by user %s" %
(machine_id, cloud_id, email), repr(exc))
log_event(email=email,
event_type='job',
action='post_deploy_finished',
cloud_id=cloud_id,
machine_id=machine_id,
enable_monitoring=bool(monitoring),
command=command,
error="Couldn't connect to run post deploy steps.",
job_id=job_id)
def deploy_collectd(email, backend_id, machine_id, extra_vars):
import mist.io.methods
user = user_from_email(email)
mist.io.methods.deploy_collectd(user, backend_id, machine_id, extra_vars)
def undeploy_collectd(email, backend_id, machine_id):
user = user_from_email(email)
import mist.io.methods
mist.io.methods.undeploy_collectd(user, backend_id, machine_id)
def execute(self, email, backend_id):
from mist.io import methods
user = user_from_email(email)
locations = methods.list_locations(user, backend_id)
return {'backend_id': backend_id, 'locations': locations}
def post_deploy_steps(self,
email,
backend_id,
machine_id,
monitoring,
command,
key_id=None,
username=None,
password=None,
port=22):
from mist.io.methods import ssh_command, connect_provider, enable_monitoring
from mist.io.methods import notify_user, notify_admin
if multi_user:
from mist.core.methods import enable_monitoring
from mist.core.helpers import log_event
else:
from mist.io.methods import enable_monitoring
log_event = lambda *args, **kwargs: None
user = user_from_email(email)
try:
# find the node we're looking for and get its hostname
conn = connect_provider(user.backends[backend_id])
nodes = conn.list_nodes()
node = None
for n in nodes:
if n.id == machine_id:
node = n
break
if node and len(node.public_ips):
# filter out IPv6 addresses
ips = filter(lambda ip: ':' not in ip, node.public_ips)
host = ips[0]
else:
raise self.retry(exc=Exception(), countdown=120, max_retries=5)
try:
from mist.io.shell import Shell
shell = Shell(host)
# connect with ssh even if no command, to create association
# to be able to enable monitoring
key_id, ssh_user = shell.autoconfigure(user, backend_id, node.id,
key_id, username, password,
port)
backend = user.backends[backend_id]
msg = "Backend:\n Name: %s\n Id: %s\n" % (backend.title,
backend_id)
msg += "Machine:\n Name: %s\n Id: %s\n" % (node.name, node.id)
if command:
log_dict = {
'email': email,
'event_type': 'job',
'backend_id': backend_id,
'machine_id': machine_id,
'job_id': uuid.uuid4().hex,
'command': command,
'host': host,
'key_id': key_id,
'ssh_user': ssh_user,
}
log_event(action='deployment_script_started', **log_dict)
start_time = time()
retval, output = shell.command(command)
execution_time = time() - start_time
output = output.decode('utf-8', 'ignore')
title = "Deployment script %s" % ('failed'
if retval else 'succeeded')
notify_user(user,
title,
backend_id=backend_id,
machine_id=machine_id,
machine_name=node.name,
command=command,
output=output,
duration=execution_time,
retval=retval,
error=retval > 0)
log_event(action='deployment_script_finished',
error=retval > 0,
return_value=retval,
stdout=output,
**log_dict)
shell.disconnect()
if monitoring:
try:
enable_monitoring(
user,
backend_id,
node.id,
name=node.name,
dns_name=node.extra.get('dns_name', ''),
public_ips=ips,
no_ssh=False,
dry=False,
)
except Exception as e:
print repr(e)
notify_user(
user, "Enable monitoring failed for machine %s (%s)" %
(node.name, node.id), repr(e))
notify_admin(
'Enable monitoring on creation failed for user %s machine %s: %r'
% (email, node.name, e))
except (ServiceUnavailableError, SSHException) as exc:
raise self.retry(exc=exc, countdown=60, max_retries=5)
except Exception as exc:
if str(exc).startswith('Retry'):
raise
notify_user(
user, "Deployment script failed for machine %s after 5 retries" %
node.id)
notify_admin(
"Deployment script failed for machine %s in backend %s by user %s after 5 retries"
% (node.id, backend_id, email), repr(exc))
log_event(
email=email,
event_type='job',
action='deployment_script_failed',
backend_id=backend_id,
machine_id=machine_id,
enable_monitoring=bool(monitoring),
command=command,
error="Couldn't connect to run post deploy steps (5 attempts).",
)
def execute(self, email, backend_id):
from mist.io import methods
user = user_from_email(email)
images = methods.list_images(user, backend_id)
return {'backend_id': backend_id, 'images': images}
def hpcloud_post_create_steps(self,
email,
cloud_id,
machine_id,
monitoring,
command,
key_id,
username,
password,
public_key,
script_id='',
script_params='',
job_id=None,
hostname='',
plugins=None,
post_script_id='',
post_script_params=''):
from mist.io.methods import connect_provider
user = user_from_email(email)
try:
conn = connect_provider(user.clouds[cloud_id])
nodes = conn.list_nodes()
node = None
for n in nodes:
if n.id == machine_id:
node = n
break
if node and node.state == 0 and len(node.public_ips):
# filter out IPv6 addresses
ips = filter(lambda ip: ':' not in ip, node.public_ips)
host = ips[0]
post_deploy_steps.delay(
email,
cloud_id,
machine_id,
monitoring,
command,
key_id,
script_id=script_id,
script_params=script_params,
job_id=job_id,
hostname=hostname,
plugins=plugins,
post_script_id=post_script_id,
post_script_params=post_script_params,
)
else:
try:
available_networks = conn.ex_list_networks()
external_networks = [
net for net in available_networks if net.router_external
]
if external_networks:
ext_net_id = external_networks[0].id
else:
ext_net_id = ""
ports = conn.ex_list_ports()
port = [
port for port in ports
if port.get("device_id", "") == node.id
][0]
ip = conn.ex_create_floating_ip(ext_net_id, port['id'])
post_deploy_steps.delay(
email,
cloud_id,
machine_id,
monitoring,
command,
key_id,
script_id=script_id,
script_params=script_params,
job_id=job_id,
hostname=hostname,
plugins=plugins,
post_script_id=post_script_id,
post_script_params=post_script_params,
)
except:
raise self.retry(exc=Exception(), max_retries=20)
except Exception as exc:
if str(exc).startswith('Retry'):
raise
def openstack_post_create_steps(self, email, backend_id, machine_id, monitoring,
command, key_id, username, password, public_key,
script_id='', script_params='', job_id=None,
hostname='', plugins=None,
post_script_id='', post_script_params='', networks=[]):
from mist.io.methods import connect_provider
user = user_from_email(email)
try:
conn = connect_provider(user.backends[backend_id])
nodes = conn.list_nodes()
node = None
for n in nodes:
if n.id == machine_id:
node = n
break
if node and node.state == 0 and len(node.public_ips):
# filter out IPv6 addresses
ips = filter(lambda ip: ':' not in ip, node.public_ips)
host = ips[0]
post_deploy_steps.delay(
email, backend_id, machine_id, monitoring, command, key_id,
script_id=script_id, script_params=script_params,
job_id=job_id, hostname=hostname, plugins=plugins,
post_script_id=post_script_id,
post_script_params=post_script_params,
)
else:
try:
created_floating_ips = []
for network in networks['public']:
created_floating_ips += [floating_ip for floating_ip in network['floating_ips']]
# From the already created floating ips try to find one that is not associated to a node
unassociated_floating_ip = None
for ip in created_floating_ips:
if not ip['node_id']:
unassociated_floating_ip = ip
break
# Find the ports which are associated to the machine (e.g. the ports of the private ips)
# and use one to associate a floating ip
ports = conn.ex_list_ports()
machine_port_id = None
for port in ports:
if port.get('device_id') == node.id:
machine_port_id = port.get('id')
break
if unassociated_floating_ip:
log.info("Associating floating ip with machine: %s" % node.id)
ip = conn.ex_associate_floating_ip_to_node(unassociated_floating_ip['id'], machine_port_id)
else:
# Find the external network
log.info("Create and associating floating ip with machine: %s" % node.id)
ext_net_id = networks['public'][0]['id']
ip = conn.ex_create_floating_ip(ext_net_id, machine_port_id)
post_deploy_steps.delay(
email, backend_id, machine_id, monitoring, command, key_id,
script_id=script_id, script_params=script_params,
job_id=job_id, hostname=hostname, plugins=plugins,
post_script_id=post_script_id,
post_script_params=post_script_params,
)
except:
raise self.retry(exc=Exception(), max_retries=20)
except Exception as exc:
if str(exc).startswith('Retry'):
raise
def azure_post_create_steps(self, email, backend_id, machine_id, monitoring,
command, key_id, username, password, public_key,
script_id='', script_params='', job_id=None,
hostname='', plugins=None,
post_script_id='', post_script_params=''):
from mist.io.methods import connect_provider
user = user_from_email(email)
try:
# find the node we're looking for and get its hostname
conn = connect_provider(user.backends[backend_id])
nodes = conn.list_nodes()
node = None
for n in nodes:
if n.id == machine_id:
node = n
break
if node and node.state == 0 and len(node.public_ips):
# filter out IPv6 addresses
ips = filter(lambda ip: ':' not in ip, node.public_ips)
host = ips[0]
else:
raise self.retry(exc=Exception(), max_retries=20)
try:
# login with user, password. Deploy the public key, enable sudo access for
# username, disable password authentication and reload ssh.
# After this is done, call post_deploy_steps if deploy script or monitoring
# is provided
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host, username=username, password=password, timeout=None, allow_agent=False, look_for_keys=False)
ssh.exec_command('mkdir -p ~/.ssh && echo "%s" >> ~/.ssh/authorized_keys && chmod -R 700 ~/.ssh/' % public_key)
chan = ssh.get_transport().open_session()
chan.get_pty()
chan.exec_command('sudo su -c \'echo "%s ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers\' ' % username)
chan.send('%s\n' % password)
check_sudo_command = 'sudo su -c \'whoami\''
chan = ssh.get_transport().open_session()
chan.get_pty()
chan.exec_command(check_sudo_command)
output = chan.recv(1024)
if not output.startswith('root'):
raise
cmd = 'sudo su -c \'sed -i "s|[#]*PasswordAuthentication yes|PasswordAuthentication no|g" /etc/ssh/sshd_config && /etc/init.d/ssh reload; service ssh reload\' '
ssh.exec_command(cmd)
ssh.close()
post_deploy_steps.delay(
email, backend_id, machine_id, monitoring, command, key_id,
script_id=script_id, script_params=script_params,
job_id=job_id, hostname=hostname, plugins=plugins,
post_script_id=post_script_id,
post_script_params=post_script_params,
)
except Exception as exc:
raise self.retry(exc=exc, countdown=10, max_retries=15)
except Exception as exc:
if str(exc).startswith('Retry'):
raise
def rackspace_first_gen_post_create_steps(self,
email,
backend_id,
machine_id,
monitoring,
command,
key_id,
password,
public_key,
username='******'):
from mist.io.methods import ssh_command, connect_provider, enable_monitoring
from mist.io.methods import notify_user, notify_admin
user = user_from_email(email)
try:
# find the node we're looking for and get its hostname
conn = connect_provider(user.backends[backend_id])
nodes = conn.list_nodes()
node = None
for n in nodes:
if n.id == machine_id:
node = n
break
if node and node.state == 0 and len(node.public_ips):
# filter out IPv6 addresses
ips = filter(lambda ip: ':' not in ip, node.public_ips)
host = ips[0]
else:
raise self.retry(exc=Exception(), max_retries=20)
try:
# login with user, password and deploy the ssh public key. Disable password authentication and reload ssh.
# After this is done, call post_deploy_steps if deploy script or monitoring
# is provided
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host,
username=username,
password=password,
timeout=None,
allow_agent=False,
look_for_keys=False)
ssh.exec_command(
'mkdir -p ~/.ssh && echo "%s" >> ~/.ssh/authorized_keys && chmod -R 700 ~/.ssh/'
% public_key)
cmd = 'sudo su -c \'sed -i "s|[#]*PasswordAuthentication yes|PasswordAuthentication no|g" /etc/ssh/sshd_config && /etc/init.d/ssh reload; service ssh reload\' '
ssh.exec_command(cmd)
ssh.close()
if command or monitoring:
post_deploy_steps.delay(email, backend_id, machine_id,
monitoring, command, key_id)
except Exception as exc:
raise self.retry(exc=exc, countdown=10, max_retries=15)
except Exception as exc:
if str(exc).startswith('Retry'):
raise
def run_deploy_script(self,
email,
backend_id,
machine_id,
command,
key_id=None,
username=None,
password=None,
port=22):
from mist.io.methods import ssh_command, connect_provider
from mist.io.methods import notify_user, notify_admin
user = user_from_email(email)
try:
# find the node we're looking for and get its hostname
conn = connect_provider(user.backends[backend_id])
nodes = conn.list_nodes()
node = None
for n in nodes:
if n.id == machine_id:
node = n
break
if node and len(node.public_ips):
# filter out IPv6 addresses
ips = filter(lambda ip: ':' not in ip, node.public_ips)
host = ips[0]
else:
raise self.retry(exc=Exception(), countdown=60, max_retries=5)
try:
from mist.io.shell import Shell
shell = Shell(host)
key_id, ssh_user = shell.autoconfigure(user, backend_id, node.id,
key_id, username, password,
port)
start_time = time()
retval, output = shell.command(command)
execution_time = time() - start_time
shell.disconnect()
msg = """
Command: %s
Return value: %s
Duration: %s seconds
Output:
%s""" % (command, retval, execution_time, output)
if retval:
notify_user(
user, "Deployment script failed for machine %s (%s)" %
(node.name, node.id), msg)
amqp_log(
"Deployment script failed for user %s machine %s (%s): %s"
% (user, node.name, node.id, msg))
else:
notify_user(
user, "Deployment script succeeded for machine %s (%s)" %
(node.name, node.id), msg)
amqp_log(
"Deployment script succeeded for user %s machine %s (%s): %s"
% (user, node.name, node.id, msg))
except ServiceUnavailableError as exc:
raise self.retry(exc=exc, countdown=60, max_retries=5)
except Exception as exc:
if str(exc).startswith('Retry'):
return
amqp_log(
"Deployment script failed for machine %s in backend %s by user %s after 5 retries: %s"
% (node.id, backend_id, email, repr(exc)))
notify_user(
user, "Deployment script failed for machine %s after 5 retries" %
node.id)
notify_admin(
"Deployment script failed for machine %s in backend %s by user %s after 5 retries"
% (node.id, backend_id, email), repr(exc))
def post_deploy_steps(self, email, backend_id, machine_id, monitoring, command,
key_id=None, username=None, password=None, port=22):
from mist.io.methods import ssh_command, connect_provider, enable_monitoring
from mist.io.methods import notify_user, notify_admin
if multi_user:
from mist.core.methods import enable_monitoring
else:
from mist.io.methods import enable_monitoring
user = user_from_email(email)
try:
# find the node we're looking for and get its hostname
conn = connect_provider(user.backends[backend_id])
nodes = conn.list_nodes()
node = None
for n in nodes:
if n.id == machine_id:
node = n
break
if node and len(node.public_ips):
# filter out IPv6 addresses
ips = filter(lambda ip: ':' not in ip, node.public_ips)
host = ips[0]
else:
raise self.retry(exc=Exception(), countdown=120, max_retries=5)
try:
from mist.io.shell import Shell
shell = Shell(host)
# connect with ssh even if no command, to create association
# to be able to enable monitoring
key_id, ssh_user = shell.autoconfigure(
user, backend_id, node.id, key_id, username, password, port
)
if command:
start_time = time()
retval, output = shell.command(command)
execution_time = time() - start_time
output = output.decode('utf-8','ignore')
msg = ("Command: %s\n"
"Return value: %s\n"
"Duration: %d seconds\n"
"Output:%s\n") % (command, retval,
execution_time, output)
msg = msg.encode('utf-8', 'ignore')
msg_title = "Deployment script %s for machine %s (%s)" % (
'failed' if retval else 'succeeded',
node.name, node.id
)
notify_user(user, msg_title, msg)
shell.disconnect()
if monitoring:
try:
enable_monitoring(user, backend_id, node.id,
name=node.name, dns_name=node.extra.get('dns_name',''),
public_ips=ips, no_ssh=False, dry=False,
)
except Exception as e:
print repr(e)
notify_user(user, "Enable monitoring failed for machine %s (%s)" % (node.name, node.id), repr(e))
notify_admin('Enable monitoring on creation failed for user %s machine %s: %r' % (email, node.name, e))
except (ServiceUnavailableError, SSHException) as exc:
raise self.retry(exc=exc, countdown=60, max_retries=5)
except Exception as exc:
if str(exc).startswith('Retry'):
raise
amqp_log("Deployment script failed for machine %s in backend %s by user %s after 5 retries: %s" % (node.id, backend_id, email, repr(exc)))
notify_user(user, "Deployment script failed for machine %s after 5 retries" % node.id)
notify_admin("Deployment script failed for machine %s in backend %s by user %s after 5 retries" % (node.id, backend_id, email), repr(exc))
def execute(self, email, backend_id):
from mist.io import methods
user = user_from_email(email)
sizes = methods.list_sizes(user, backend_id)
return {'backend_id': backend_id, 'sizes': sizes}