Exemple #1
0
 def execute(self, email, cloud_id):
     log.warn('Running list images for user %s cloud %s' % (email, cloud_id))
     from mist.io import methods
     user = user_from_email(email)
     images = methods.list_images(user, cloud_id)
     log.warn('Returning list images for user %s cloud %s' % (email, cloud_id))
     return {'cloud_id': cloud_id, 'images': images}
Exemple #2
0
    def error_rerun_handler(self, exc, errors, email, backend_id):
        from mist.io.methods import notify_user

        if len(errors) < 6:
            return self.result_fresh  # Retry when the result is no longer fresh

        user = user_from_email(email)

        if len(errors) == 6: # If does not respond for a minute
            notify_user(user, 'Backend %s does not respond' % user.backends[backend_id].title,
                        email_notify=False, backend_id=backend_id)

        # Keep retrying for 30 minutes
        times = [60, 60, 120, 300, 600, 600]
        index = len(errors) - 6
        if index < len(times):
            return times[index]
        else: # If backend still unresponsive disable it & notify user
            with user.lock_n_load():
                user.backends[backend_id].enabled = False
                user.save()
            notify_user(user, "Backend %s disabled after not responding for 30mins" % user.backends[backend_id].title,
                        email_notify=True, backend_id=backend_id)
            log_event(user.email, 'incident', action='disable_backend',
                      backend_id=backend_id, error="Backend unresponsive")
Exemple #3
0
 def execute(self, email, cloud_id):
     log.warn('Running list projects for user %s cloud %s' % (email, cloud_id))
     from mist.io import methods
     user = user_from_email(email)
     projects = methods.list_projects(user, cloud_id)
     log.warn('Returning list projects for user %s cloud %s' % (email, cloud_id))
     return {'cloud_id': cloud_id, 'projects': projects}
Exemple #4
0
 def execute(self, email, backend_id):
     log.warn('Running list networks for user %s backend %s' % (email, backend_id))
     from mist.io import methods
     user = user_from_email(email)
     networks = methods.list_networks(user, backend_id)
     log.warn('Returning list networks for user %s backend %s' % (email, backend_id))
     return {'backend_id': backend_id, 'networks': networks}
Exemple #5
0
 def error_rerun_handler(self, exc, errors, email, backend_id):
     if len(errors) < 6:
         return self.result_fresh  # Retry when the result is no longer fresh
     user = user_from_email(email)
     with user.lock_n_load():
         user.backends[backend_id].enabled = False
         user.save()
Exemple #6
0
 def execute(self, email, backend_id):
     log.warn('Running list machines for user %s backend %s' % (email, backend_id))
     from mist.io import methods
     user = user_from_email(email)
     machines = methods.list_machines(user, backend_id)
     log.warn('Returning list machines for user %s backend %s' % (email, backend_id))
     return {'backend_id': backend_id, 'machines': machines}
Exemple #7
0
def deploy_collectd(email, backend_id, machine_id, extra_vars):
    if not multi_user:
        from mist.io.methods import deploy_collectd as deploy_collectd_method
    else:
        from mist.core.methods import deploy_collectd as deploy_collectd_method
    user = user_from_email(email)
    deploy_collectd_method(user, backend_id, machine_id, extra_vars)
Exemple #8
0
def create_machine_async(email, backend_id, key_id, machine_name, location_id,
                         image_id, size_id, script, image_extra, disk,
                          image_name, size_name, location_name, ips, monitoring,
                          networks, docker_env, docker_command,
                          script_id=None, script_params=None,
                          quantity=1, persist=False, job_id=None,
                          docker_port_bindings={}, docker_exposed_ports={},
                          hostname='', plugins=None):
    from multiprocessing.dummy import Pool as ThreadPool
    from mist.io.methods import create_machine
    from mist.io.exceptions import MachineCreationError
    log.warn('MULTICREATE ASYNC %d' % quantity)

    if multi_user:
        from mist.core.helpers import log_event
    else:
        log_event = lambda *args, **kwargs: None
    job_id = job_id or uuid.uuid4().hex

    log_event(email, 'job', 'async_machine_creation_started', job_id=job_id,
              backend_id=backend_id, script=script, script_id=script_id,
              script_params=script_params, monitoring=monitoring,
              persist=persist, quantity=quantity)

    THREAD_COUNT = 5
    pool = ThreadPool(THREAD_COUNT)

    names = []
    for i in range(1, quantity+1):
        names.append('%s-%d' % (machine_name,i))

    user = user_from_email(email)
    specs = []
    for name in names:
        specs.append((
            (user, backend_id, key_id, name, location_id, image_id,
             size_id, script, image_extra, disk, image_name, size_name,
             location_name, ips, monitoring, networks, docker_env,
             docker_command, 22, script_id, script_params, job_id),
            {'hostname': hostname, 'plugins': plugins}
        ))

    def create_machine_wrapper(args_kwargs):
        args, kwargs = args_kwargs
        error = False
        try:
            node = create_machine(*args, **kwargs)
        except MachineCreationError as exc:
            error = str(exc)
        except Exception as exc:
            error = repr(exc)
        finally:
            name = args[3]
            log_event(email, 'job', 'machine_creation_finished', job_id=job_id,
                      backend_id=backend_id, machine_name=name, error=error)

    pool.map(create_machine_wrapper, specs)
    pool.close()
    pool.join()
Exemple #9
0
def azure_post_create_steps(self, email, backend_id, machine_id, monitoring, command,
                      key_id, username, password, public_key):
    from mist.io.methods import ssh_command, connect_provider, enable_monitoring
    from mist.io.methods import notify_user, notify_admin
    user = user_from_email(email)

    try:
        # find the node we're looking for and get its hostname
        conn = connect_provider(user.backends[backend_id])
        nodes = conn.list_nodes()
        node = None
        for n in nodes:
            if n.id == machine_id:
                node = n
                break

        if node and len(node.public_ips):
            # filter out IPv6 addresses
            ips = filter(lambda ip: ':' not in ip, node.public_ips)
            host = ips[0]
        else:
            raise self.retry(exc=Exception(), countdown=120, max_retries=5)

        try:
            #login with user, password. Deploy the public key, enable sudo access for
            #username, disable password authentication and reload ssh.
            #After this is done, call post_deploy_steps if deploy script or monitoring
            #is provided
            ssh=paramiko.SSHClient()
            ssh.load_system_host_keys()
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            ssh.connect(host, username=username, password=password)
            ssh.exec_command('mkdir -p ~/.ssh && echo "%s" >> ~/.ssh/authorized_keys && chmod -R 700 ~/.ssh/' % public_key)

            chan = ssh.invoke_shell()
            chan = ssh.get_transport().open_session()
            chan.get_pty()
            chan.exec_command('sudo su -c \'echo "%s ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers\' ' % username)
            chan.send('%s\n' % password)

            chan = ssh.invoke_shell()
            chan = ssh.get_transport().open_session()
            chan.get_pty()
            chan.exec_command('sudo su -c \'echo "%s ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/waagent\' ' % username)
            chan.send('%s\n' % password)

            cmd = 'sudo su -c \'sed -i "s|[#]*PasswordAuthentication yes|PasswordAuthentication no|g" /etc/ssh/sshd_config &&  /etc/init.d/ssh reload; service ssh reload\' '
            ssh.exec_command(cmd)
            ssh.close()

            if command or monitoring:
                post_deploy_steps.delay(email, backend_id, machine_id,
                                          monitoring, command, key_id)

        except Exception as exc:
            raise self.retry(exc=exc, countdown=60, max_retries=10)
    except Exception as exc:
        if str(exc).startswith('Retry'):
            raise
Exemple #10
0
    def execute(self, email, backend_id):
        log.warn("Running list images for user %s backend %s" % (email, backend_id))
        from mist.io import methods

        user = user_from_email(email)
        images = methods.list_images(user, backend_id)
        log.warn("Returning list images for user %s backend %s" % (email, backend_id))
        return {"backend_id": backend_id, "images": images}
Exemple #11
0
    def execute(self, email, backend_id):
        log.warn("Running list networks for user %s backend %s" % (email, backend_id))
        from mist.io import methods

        user = user_from_email(email)
        networks = methods.list_networks(user, backend_id)
        log.warn("Returning list networks for user %s backend %s" % (email, backend_id))
        return {"backend_id": backend_id, "networks": networks}
Exemple #12
0
 def execute(self, email, backend_id, machine_id, host):
     user = user_from_email(email)
     from mist.io.methods import probe_ssh_only
     res = probe_ssh_only(user, backend_id, machine_id, host)
     return {'backend_id': backend_id,
             'machine_id': machine_id,
             'host': host,
             'result': res}
Exemple #13
0
def hpcloud_post_create_steps(self, email, backend_id, machine_id, monitoring,
                              command, key_id, username, password, public_key,
                              script_id='', script_params='', job_id=None,
                              hostname='', plugins=None,
                              post_script_id='', post_script_params=''):
    from mist.io.methods import connect_provider
    user = user_from_email(email)

    try:
        conn = connect_provider(user.backends[backend_id])
        nodes = conn.list_nodes()
        node = None

        for n in nodes:
            if n.id == machine_id:
                node = n
                break

        if node and node.state == 0 and len(node.public_ips):
            # filter out IPv6 addresses
            ips = filter(lambda ip: ':' not in ip, node.public_ips)
            host = ips[0]

            post_deploy_steps.delay(
                email, backend_id, machine_id, monitoring, command, key_id,
                script_id=script_id, script_params=script_params,
                job_id=job_id, hostname=hostname, plugins=plugins,
                post_script_id=post_script_id,
                post_script_params=post_script_params,
            )

        else:
            try:
                available_networks = conn.ex_list_networks()
                external_networks = [net for net in available_networks if net.router_external]
                if external_networks:
                    ext_net_id = external_networks[0].id
                else:
                    ext_net_id = ""

                ports = conn.ex_list_ports()

                port = [port for port in ports if port.get("device_id", "") == node.id][0]

                ip = conn.ex_create_floating_ip(ext_net_id, port['id'])
                post_deploy_steps.delay(
                    email, backend_id, machine_id, monitoring, command, key_id,
                    script_id=script_id, script_params=script_params,
                    job_id=job_id, hostname=hostname, plugins=plugins,
                    post_script_id=post_script_id,
                    post_script_params=post_script_params,
                )

            except:
                raise self.retry(exc=Exception(), max_retries=20)
    except Exception as exc:
        if str(exc).startswith('Retry'):
            raise
Exemple #14
0
def update_machine_count(email, backend_id, machine_count):
    if not multi_user:
        return

    user = user_from_email(email)
    with user.lock_n_load():
        user.backends[backend_id].machine_count = machine_count
        user.total_machine_count = sum([backend.machine_count for backend in user.backends.values()])
        user.save()
Exemple #15
0
def rackspace_first_gen_post_create_steps(
    self, email, cloud_id, machine_id, monitoring, command, key_id,
    password, public_key, username='******', script_id='', script_params='',
    job_id=None, hostname='', plugins=None, post_script_id='',
    post_script_params='', cronjob={}
):
    from mist.io.methods import connect_provider
    user = user_from_email(email)

    try:
        # find the node we're looking for and get its hostname
        conn = connect_provider(user.clouds[cloud_id])
        nodes = conn.list_nodes()
        node = None
        for n in nodes:
            if n.id == machine_id:
                node = n
                break

        if node and node.state == 0 and len(node.public_ips):
            # filter out IPv6 addresses
            ips = filter(lambda ip: ':' not in ip, node.public_ips)
            host = ips[0]
        else:
            raise self.retry(exc=Exception(), max_retries=20)

        try:
            # login with user, password and deploy the ssh public key.
            # Disable password authentication and reload ssh.
            # After this is done, call post_deploy_steps
            # if deploy script or monitoring
            # is provided
            ssh=paramiko.SSHClient()
            ssh.load_system_host_keys()
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            ssh.connect(host, username=username, password=password, timeout=None, allow_agent=False, look_for_keys=False)

            ssh.exec_command('mkdir -p ~/.ssh && echo "%s" >> ~/.ssh/authorized_keys && chmod -R 700 ~/.ssh/' % public_key)

            cmd = 'sudo su -c \'sed -i "s|[#]*PasswordAuthentication yes|PasswordAuthentication no|g" /etc/ssh/sshd_config &&  /etc/init.d/ssh reload; service ssh reload\' '
            ssh.exec_command(cmd)

            ssh.close()

            post_deploy_steps.delay(
                email, cloud_id, machine_id, monitoring, command, key_id,
                script_id=script_id, script_params=script_params,
                job_id=job_id, hostname=hostname, plugins=plugins,
                post_script_id=post_script_id,
                post_script_params=post_script_params, cronjob=cronjob
            )

        except Exception as exc:
            raise self.retry(exc=exc, countdown=10, max_retries=15)
    except Exception as exc:
        if str(exc).startswith('Retry'):
            raise
Exemple #16
0
def ssh_command(email, backend_id, machine_id, host, command, key_id=None, username=None, password=None, port=22):
    user = user_from_email(email)
    shell = Shell(host)
    key_id, ssh_user = shell.autoconfigure(user, backend_id, machine_id, key_id, username, password, port)
    retval, output = shell.command(command)
    shell.disconnect()
    if retval:
        from mist.io.methods import notify_user

        notify_user(user, "Async command failed for machine %s (%s)" % (machine_id, host), output)
Exemple #17
0
def run_deploy_script(self, email, backend_id, machine_id, command,
                      key_id=None, username=None, password=None, port=22):
    from mist.io.methods import ssh_command, connect_provider
    from mist.io.methods import notify_user, notify_admin

    user = user_from_email(email)

    try:
        # find the node we're looking for and get its hostname
        conn = connect_provider(user.backends[backend_id])
        nodes = conn.list_nodes()
        node = None
        for n in nodes:
            if n.id == machine_id:
                node = n
                break

        if node and len(node.public_ips):
            # filter out IPv6 addresses
            ips = filter(lambda ip: ':' not in ip, node.public_ips)
            host = ips[0]
        else:
            raise self.retry(exc=Exception(), countdown=60, max_retries=5)

        try:
            from mist.io.shell import Shell
            shell = Shell(host)
            key_id, ssh_user = shell.autoconfigure(user, backend_id, node.id,
                                                   key_id, username, password, port)
            start_time = time()
            retval, output = shell.command(command)
            execution_time = time() - start_time
            shell.disconnect()
            msg = """
Command: %s
Return value: %s
Duration: %s seconds
Output:
%s""" % (command, retval, execution_time, output)

            if retval:
                notify_user(user, "Deployment script failed for machine %s (%s)" % (node.name, node.id), msg)
                amqp_log("Deployment script failed for user %s machine %s (%s): %s" % (user, node.name, node.id, msg))
            else:
                notify_user(user, "Deployment script succeeded for machine %s (%s)" % (node.name, node.id), msg)
                amqp_log("Deployment script succeeded for user %s machine %s (%s): %s" % (user, node.name, node.id, msg))

        except ServiceUnavailableError as exc:
            raise self.retry(exc=exc, countdown=60, max_retries=5)
    except Exception as exc:
        if str(exc).startswith('Retry'):
            return
        amqp_log("Deployment script failed for machine %s in backend %s by user %s after 5 retries: %s" % (node.id, backend_id, email, repr(exc)))
        notify_user(user, "Deployment script failed for machine %s after 5 retries" % node.id)
        notify_admin("Deployment script failed for machine %s in backend %s by user %s after 5 retries" % (node.id, backend_id, email), repr(exc))
Exemple #18
0
def ssh_command(email, backend_id, machine_id, host, command,
                      key_id=None, username=None, password=None, port=22):
    user = user_from_email(email)
    shell = Shell(host)
    key_id, ssh_user = shell.autoconfigure(user, backend_id, machine_id,
                                           key_id, username, password, port)
    retval, output = shell.command(command)
    shell.disconnect()
    if retval:
        from mist.io.methods import notify_user
        notify_user(user, "Async command failed for machine %s (%s)" %
                    (machine_id, host), output)
Exemple #19
0
 def execute(self, email, backend_id):
     log.warn('Running list machines for user %s backend %s' % (email, backend_id))
     from mist.io import methods
     user = user_from_email(email)
     machines = methods.list_machines(user, backend_id)
     if multi_user:
         for machine in machines:
             kwargs = {}
             kwargs['backend_id'] = backend_id
             kwargs['machine_id'] = machine.get('id')
             from mist.core.methods import list_tags
             mistio_tags = list_tags(user, resource_type='machine', **kwargs)
             # optimized for js
             for tag in mistio_tags:
                 for key, value in tag.items():
                     tag_dict = {'key': key, 'value': value}
                     if tag_dict not in machine['tags']:
                         machine['tags'].append(tag_dict)
             # FIXME: optimize!
     log.warn('Returning list machines for user %s backend %s' % (email, backend_id))
     return {'backend_id': backend_id, 'machines': machines}
Exemple #20
0
    def error_rerun_handler(self, exc, errors, email, cloud_id):
        from mist.io.methods import notify_user

        if len(errors) < 6:
            return self.result_fresh  # Retry when the result is no longer fresh

        user = user_from_email(email)

        if len(errors) == 6:  # If does not respond for a minute
            notify_user(user, 'Cloud %s does not respond'
                        % user.clouds[cloud_id].title,
                        email_notify=False, cloud_id=cloud_id)

        # Keep retrying every 30 secs for 10 minutes, then every 60 secs for
        # 20 minutes and finally every 20 minutes
        times = [30]*20 + [60]*20
        index = len(errors) - 6
        if index < len(times):
            return times[index]
        else:
            return 20*60
Exemple #21
0
    def error_rerun_handler(self, exc, errors, email, cloud_id):
        from mist.io.methods import notify_user

        if len(errors) < 6:
            return self.result_fresh  # Retry when the result is no longer fresh

        user = user_from_email(email)

        if len(errors) == 6:  # If does not respond for a minute
            notify_user(user,
                        'Cloud %s does not respond' %
                        user.clouds[cloud_id].title,
                        email_notify=False,
                        cloud_id=cloud_id)

        # Keep retrying every 30 secs for 10 minutes, then every 60 secs for
        # 20 minutes and finally every 20 minutes
        times = [30] * 20 + [60] * 20
        index = len(errors) - 6
        if index < len(times):
            return times[index]
        else:
            return 20 * 60
Exemple #22
0
    def execute(self, email, backend_id):
        log.warn("Running list machines for user %s backend %s" % (email, backend_id))
        from mist.io import methods

        user = user_from_email(email)
        machines = methods.list_machines(user, backend_id)
        if multi_user:
            for machine in machines:
                kwargs = {}
                kwargs["backend_id"] = backend_id
                kwargs["machine_id"] = machine.get("id")
                from mist.core.methods import list_tags

                mistio_tags = list_tags(user, resource_type="machine", **kwargs)
                # optimized for js
                for tag in mistio_tags:
                    for key, value in tag.items():
                        tag_dict = {"key": key, "value": value}
                        if tag_dict not in machine["tags"]:
                            machine["tags"].append(tag_dict)
                # FIXME: optimize!
        log.warn("Returning list machines for user %s backend %s" % (email, backend_id))
        return {"backend_id": backend_id, "machines": machines}
Exemple #23
0
def create_machine_async(email,
                         cloud_id,
                         key_id,
                         machine_name,
                         location_id,
                         image_id,
                         size_id,
                         script,
                         image_extra,
                         disk,
                         image_name,
                         size_name,
                         location_name,
                         ips,
                         monitoring,
                         networks,
                         docker_env,
                         docker_command,
                         script_id='',
                         script_params='',
                         post_script_id='',
                         post_script_params='',
                         quantity=1,
                         persist=False,
                         job_id=None,
                         docker_port_bindings={},
                         docker_exposed_ports={},
                         azure_port_bindings='',
                         hostname='',
                         plugins=None,
                         disk_size=None,
                         disk_path=None,
                         cloud_init='',
                         associate_floating_ip=False,
                         associate_floating_ip_subnet=None,
                         project_id=None):
    from multiprocessing.dummy import Pool as ThreadPool
    from mist.io.methods import create_machine
    from mist.io.exceptions import MachineCreationError
    log.warn('MULTICREATE ASYNC %d' % quantity)

    if multi_user:
        from mist.core.helpers import log_event
    else:
        log_event = lambda *args, **kwargs: None
    job_id = job_id or uuid.uuid4().hex

    log_event(email,
              'job',
              'async_machine_creation_started',
              job_id=job_id,
              cloud_id=cloud_id,
              script=script,
              script_id=script_id,
              script_params=script_params,
              monitoring=monitoring,
              persist=persist,
              quantity=quantity)

    THREAD_COUNT = 5
    pool = ThreadPool(THREAD_COUNT)

    names = []
    for i in range(1, quantity + 1):
        names.append('%s-%d' % (machine_name, i))

    user = user_from_email(email)
    specs = []
    for name in names:
        specs.append(
            ((user, cloud_id, key_id, name, location_id, image_id, size_id,
              script, image_extra, disk, image_name, size_name, location_name,
              ips, monitoring, networks, docker_env, docker_command, 22,
              script_id, script_params, job_id), {
                  'hostname': hostname,
                  'plugins': plugins,
                  'post_script_id': post_script_id,
                  'post_script_params': post_script_params,
                  'azure_port_bindings': azure_port_bindings,
                  'associate_floating_ip': associate_floating_ip,
                  'cloud_init': cloud_init,
                  'disk_size': disk_size,
                  'disk_path': disk_path,
                  'project_id': project_id
              }))

    def create_machine_wrapper(args_kwargs):
        args, kwargs = args_kwargs
        error = False
        try:
            node = create_machine(*args, **kwargs)
        except MachineCreationError as exc:
            error = str(exc)
        except Exception as exc:
            error = repr(exc)
        finally:
            name = args[3]
            log_event(email,
                      'job',
                      'machine_creation_finished',
                      job_id=job_id,
                      cloud_id=cloud_id,
                      machine_name=name,
                      error=error)

    pool.map(create_machine_wrapper, specs)
    pool.close()
    pool.join()
Exemple #24
0
 def execute(self, email, cloud_id):
     from mist.io import methods
     user = user_from_email(email)
     sizes = methods.list_sizes(user, cloud_id)
     return {'cloud_id': cloud_id, 'sizes': sizes}
Exemple #25
0
 def execute(self, email, backend_id):
     from mist.io import methods
     user = user_from_email(email)
     networks = methods.list_networks(user, backend_id)
     return {'backend_id': backend_id, 'networks': networks}
Exemple #26
0
def post_deploy_steps(self, email, backend_id, machine_id, monitoring, command,
                      key_id=None, username=None, password=None, port=22,
                      script_id='', script_params='', job_id=None,
                      hostname='', plugins=None,
                      post_script_id='', post_script_params=''):
    from mist.io.methods import connect_provider, probe_ssh_only
    from mist.io.methods import notify_user, notify_admin
    from mist.io.methods import create_dns_a_record
    if multi_user:
        from mist.core.methods import enable_monitoring
        from mist.core.tasks import run_script
        from mist.core.helpers import log_event
    else:
        from mist.io.methods import enable_monitoring
        log_event = lambda *args, **kwargs: None

    job_id = job_id or uuid.uuid4().hex

    user = user_from_email(email)
    tmp_log = lambda msg, *args: log.error('Post deploy: %s' % msg, *args)
    tmp_log('Entering post deploy steps for %s %s %s',
            user.email, backend_id, machine_id)
    try:
        # find the node we're looking for and get its hostname
        node = None
        try:
            conn = connect_provider(user.backends[backend_id])
            nodes = conn.list_nodes() # TODO: use cache
            for n in nodes:
                if n.id == machine_id:
                    node = n
                    break
            tmp_log('run list_machines')
        except:
            raise self.retry(exc=Exception(), countdown=10, max_retries=10)

        if node and len(node.public_ips):
            # filter out IPv6 addresses
            ips = filter(lambda ip: ':' not in ip, node.public_ips)
            host = ips[0]
        else:
            tmp_log('ip not found, retrying')
            raise self.retry(exc=Exception(), countdown=60, max_retries=20)

        try:
            from mist.io.shell import Shell
            shell = Shell(host)
            # connect with ssh even if no command, to create association
            # to be able to enable monitoring
            tmp_log('attempting to connect to shell')
            key_id, ssh_user = shell.autoconfigure(
                user, backend_id, node.id, key_id, username, password, port
            )
            tmp_log('connected to shell')
            result = probe_ssh_only(user, backend_id, machine_id, host=None,
                           key_id=key_id, ssh_user=ssh_user, shell=shell)
            log_dict = {
                    'email': email,
                    'event_type': 'job',
                    'backend_id': backend_id,
                    'machine_id': machine_id,
                    'job_id': job_id,
                    'host': host,
                    'key_id': key_id,
                    'ssh_user': ssh_user,
                }

            log_event(action='probe', result=result, **log_dict)
            backend = user.backends[backend_id]
            msg = "Backend:\n  Name: %s\n  Id: %s\n" % (backend.title,
                                                        backend_id)
            msg += "Machine:\n  Name: %s\n  Id: %s\n" % (node.name,
                                                             node.id)

            if hostname:
                try:
                    record = create_dns_a_record(user, hostname, host)
                    hostname = '.'.join((record.name, record.zone.domain))
                    log_event(action='create_dns_a_record', hostname=hostname,
                              **log_dict)
                except Exception as exc:
                    log_event(action='create_dns_a_record', error=str(exc),
                              **log_dict)

            error = False
            if script_id and multi_user:
                tmp_log('will run script_id %s', script_id)
                ret = run_script.run(
                    user.email, script_id, backend_id, machine_id,
                    params=script_params, host=host, job_id=job_id
                )
                error = ret['error']
                tmp_log('executed script_id %s', script_id)
            elif command:
                tmp_log('will run command %s', command)
                log_event(action='deployment_script_started', command=command, **log_dict)
                start_time = time()
                retval, output = shell.command(command)
                tmp_log('executed command %s', command)
                execution_time = time() - start_time
                output = output.decode('utf-8','ignore')
                title = "Deployment script %s" % ('failed' if retval
                                                  else 'succeeded')
                error = retval > 0
                notify_user(user, title,
                            backend_id=backend_id,
                            machine_id=machine_id,
                            machine_name=node.name,
                            command=command,
                            output=output,
                            duration=execution_time,
                            retval=retval,
                            error=retval > 0)
                log_event(action='deployment_script_finished',
                          error=retval > 0,
                          return_value=retval,
                          command=command,
                          stdout=output,
                          **log_dict)

            shell.disconnect()

            if monitoring:
                try:
                    enable_monitoring(user, backend_id, node.id,
                        name=node.name, dns_name=node.extra.get('dns_name',''),
                        public_ips=ips, no_ssh=False, dry=False, job_id=job_id,
                        plugins=plugins, deploy_async=False,
                    )
                except Exception as e:
                    print repr(e)
                    error = True
                    notify_user(user, "Enable monitoring failed for machine %s" % machine_id, repr(e))
                    notify_admin('Enable monitoring on creation failed for user %s machine %s: %r' % (email, machine_id, e))
                    log_event(action='enable_monitoring_failed', error=repr(e),
                              **log_dict)

            if post_script_id and multi_user:
                tmp_log('will run post_script_id %s', post_script_id)
                ret = run_script.run(
                    user.email, post_script_id, backend_id, machine_id,
                    params=post_script_params, host=host, job_id=job_id,
                    action_prefix='post_',
                )
                error = ret['error']
                tmp_log('executed post_script_id %s', script_id)

            log_event(action='post_deploy_finished', error=error, **log_dict)

        except (ServiceUnavailableError, SSHException) as exc:
            tmp_log(repr(exc))
            raise self.retry(exc=exc, countdown=60, max_retries=15)
    except Exception as exc:
        tmp_log(repr(exc))
        if str(exc).startswith('Retry'):
            raise
        notify_user(user, "Deployment script failed for machine %s" % machine_id)
        notify_admin("Deployment script failed for machine %s in backend %s by user %s" % (machine_id, backend_id, email), repr(exc))
        log_event(
            email=email,
            event_type='job',
            action='post_deploy_finished',
            backend_id=backend_id,
            machine_id=machine_id,
            enable_monitoring=bool(monitoring),
            command=command,
            error="Couldn't connect to run post deploy steps.",
            job_id=job_id
        )
Exemple #27
0
def deploy_collectd(email, backend_id, machine_id, extra_vars):
    import mist.io.methods
    user = user_from_email(email)
    mist.io.methods.deploy_collectd(user, backend_id, machine_id, extra_vars)
Exemple #28
0
 def execute(self, email, backend_id):
     from mist.io import methods
     user = user_from_email(email)
     locations = methods.list_locations(user, backend_id)
     return {'backend_id': backend_id, 'locations': locations}
Exemple #29
0
def undeploy_collectd(email, backend_id, machine_id):
    user = user_from_email(email)
    import mist.io.methods
    mist.io.methods.undeploy_collectd(user, backend_id, machine_id)
Exemple #30
0
def openstack_post_create_steps(self,
                                email,
                                cloud_id,
                                machine_id,
                                monitoring,
                                command,
                                key_id,
                                username,
                                password,
                                public_key,
                                script_id='',
                                script_params='',
                                job_id=None,
                                hostname='',
                                plugins=None,
                                post_script_id='',
                                post_script_params='',
                                networks=[]):

    from mist.io.methods import connect_provider
    user = user_from_email(email)

    try:
        conn = connect_provider(user.clouds[cloud_id])
        nodes = conn.list_nodes()
        node = None

        for n in nodes:
            if n.id == machine_id:
                node = n
                break

        if node and node.state == 0 and len(node.public_ips):
            # filter out IPv6 addresses
            ips = filter(lambda ip: ':' not in ip, node.public_ips)
            host = ips[0]

            post_deploy_steps.delay(
                email,
                cloud_id,
                machine_id,
                monitoring,
                command,
                key_id,
                script_id=script_id,
                script_params=script_params,
                job_id=job_id,
                hostname=hostname,
                plugins=plugins,
                post_script_id=post_script_id,
                post_script_params=post_script_params,
            )

        else:
            try:
                created_floating_ips = []
                for network in networks['public']:
                    created_floating_ips += [
                        floating_ip for floating_ip in network['floating_ips']
                    ]

                # From the already created floating ips try to find one that is not associated to a node
                unassociated_floating_ip = None
                for ip in created_floating_ips:
                    if not ip['node_id']:
                        unassociated_floating_ip = ip
                        break

                # Find the ports which are associated to the machine (e.g. the ports of the private ips)
                # and use one to associate a floating ip
                ports = conn.ex_list_ports()
                machine_port_id = None
                for port in ports:
                    if port.get('device_id') == node.id:
                        machine_port_id = port.get('id')
                        break

                if unassociated_floating_ip:
                    log.info("Associating floating ip with machine: %s" %
                             node.id)
                    ip = conn.ex_associate_floating_ip_to_node(
                        unassociated_floating_ip['id'], machine_port_id)
                else:
                    # Find the external network
                    log.info(
                        "Create and associating floating ip with machine: %s" %
                        node.id)
                    ext_net_id = networks['public'][0]['id']
                    ip = conn.ex_create_floating_ip(ext_net_id,
                                                    machine_port_id)

                post_deploy_steps.delay(
                    email,
                    cloud_id,
                    machine_id,
                    monitoring,
                    command,
                    key_id,
                    script_id=script_id,
                    script_params=script_params,
                    job_id=job_id,
                    hostname=hostname,
                    plugins=plugins,
                    post_script_id=post_script_id,
                    post_script_params=post_script_params,
                )

            except:
                raise self.retry(exc=Exception(), max_retries=20)
    except Exception as exc:
        if str(exc).startswith('Retry'):
            raise
Exemple #31
0
def azure_post_create_steps(self,
                            email,
                            cloud_id,
                            machine_id,
                            monitoring,
                            command,
                            key_id,
                            username,
                            password,
                            public_key,
                            script_id='',
                            script_params='',
                            job_id=None,
                            hostname='',
                            plugins=None,
                            post_script_id='',
                            post_script_params=''):
    from mist.io.methods import connect_provider
    user = user_from_email(email)

    try:
        # find the node we're looking for and get its hostname
        conn = connect_provider(user.clouds[cloud_id])
        nodes = conn.list_nodes()
        node = None
        for n in nodes:
            if n.id == machine_id:
                node = n
                break

        if node and node.state == 0 and len(node.public_ips):
            # filter out IPv6 addresses
            ips = filter(lambda ip: ':' not in ip, node.public_ips)
            host = ips[0]
        else:
            raise self.retry(exc=Exception(), max_retries=20)

        try:
            # login with user, password. Deploy the public key, enable sudo access for
            # username, disable password authentication and reload ssh.
            # After this is done, call post_deploy_steps if deploy script or monitoring
            # is provided
            ssh = paramiko.SSHClient()
            ssh.load_system_host_keys()
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            ssh.connect(host,
                        username=username,
                        password=password,
                        timeout=None,
                        allow_agent=False,
                        look_for_keys=False)

            ssh.exec_command(
                'mkdir -p ~/.ssh && echo "%s" >> ~/.ssh/authorized_keys && chmod -R 700 ~/.ssh/'
                % public_key)

            chan = ssh.get_transport().open_session()
            chan.get_pty()
            chan.exec_command(
                'sudo su -c \'echo "%s ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers\' '
                % username)
            chan.send('%s\n' % password)

            check_sudo_command = 'sudo su -c \'whoami\''

            chan = ssh.get_transport().open_session()
            chan.get_pty()
            chan.exec_command(check_sudo_command)
            output = chan.recv(1024)

            if not output.startswith('root'):
                raise
            cmd = 'sudo su -c \'sed -i "s|[#]*PasswordAuthentication yes|PasswordAuthentication no|g" /etc/ssh/sshd_config &&  /etc/init.d/ssh reload; service ssh reload\' '
            ssh.exec_command(cmd)

            ssh.close()

            post_deploy_steps.delay(
                email,
                cloud_id,
                machine_id,
                monitoring,
                command,
                key_id,
                script_id=script_id,
                script_params=script_params,
                job_id=job_id,
                hostname=hostname,
                plugins=plugins,
                post_script_id=post_script_id,
                post_script_params=post_script_params,
            )

        except Exception as exc:
            raise self.retry(exc=exc, countdown=10, max_retries=15)
    except Exception as exc:
        if str(exc).startswith('Retry'):
            raise
Exemple #32
0
def post_deploy_steps(self,
                      email,
                      cloud_id,
                      machine_id,
                      monitoring,
                      command,
                      key_id=None,
                      username=None,
                      password=None,
                      port=22,
                      script_id='',
                      script_params='',
                      job_id=None,
                      hostname='',
                      plugins=None,
                      post_script_id='',
                      post_script_params=''):
    from mist.io.methods import connect_provider, probe_ssh_only
    from mist.io.methods import notify_user, notify_admin
    from mist.io.methods import create_dns_a_record
    if multi_user:
        from mist.core.methods import enable_monitoring
        from mist.core.tasks import run_script
        from mist.core.helpers import log_event
    else:
        from mist.io.methods import enable_monitoring
        log_event = lambda *args, **kwargs: None

    job_id = job_id or uuid.uuid4().hex

    user = user_from_email(email)
    tmp_log = lambda msg, *args: log.error('Post deploy: %s' % msg, *args)
    tmp_log('Entering post deploy steps for %s %s %s', user.email, cloud_id,
            machine_id)
    try:
        # find the node we're looking for and get its hostname
        node = None
        try:
            conn = connect_provider(user.clouds[cloud_id])
            nodes = conn.list_nodes()  # TODO: use cache
            for n in nodes:
                if n.id == machine_id:
                    node = n
                    break
            tmp_log('run list_machines')
        except:
            raise self.retry(exc=Exception(), countdown=10, max_retries=10)

        if node and len(node.public_ips):
            # filter out IPv6 addresses
            ips = filter(lambda ip: ':' not in ip, node.public_ips)
            host = ips[0]
        else:
            tmp_log('ip not found, retrying')
            raise self.retry(exc=Exception(), countdown=60, max_retries=20)

        try:
            from mist.io.shell import Shell
            shell = Shell(host)
            # connect with ssh even if no command, to create association
            # to be able to enable monitoring
            tmp_log('attempting to connect to shell')
            key_id, ssh_user = shell.autoconfigure(user, cloud_id, node.id,
                                                   key_id, username, password,
                                                   port)
            tmp_log('connected to shell')
            result = probe_ssh_only(user,
                                    cloud_id,
                                    machine_id,
                                    host=None,
                                    key_id=key_id,
                                    ssh_user=ssh_user,
                                    shell=shell)
            log_dict = {
                'email': email,
                'event_type': 'job',
                'cloud_id': cloud_id,
                'machine_id': machine_id,
                'job_id': job_id,
                'host': host,
                'key_id': key_id,
                'ssh_user': ssh_user,
            }

            log_event(action='probe', result=result, **log_dict)
            cloud = user.clouds[cloud_id]
            msg = "Cloud:\n  Name: %s\n  Id: %s\n" % (cloud.title, cloud_id)
            msg += "Machine:\n  Name: %s\n  Id: %s\n" % (node.name, node.id)

            if hostname:
                try:
                    record = create_dns_a_record(user, hostname, host)
                    hostname = '.'.join((record.name, record.zone.domain))
                    log_event(action='create_dns_a_record',
                              hostname=hostname,
                              **log_dict)
                except Exception as exc:
                    log_event(action='create_dns_a_record',
                              error=str(exc),
                              **log_dict)

            error = False
            if script_id and multi_user:
                tmp_log('will run script_id %s', script_id)
                ret = run_script.run(user.email,
                                     script_id,
                                     cloud_id,
                                     machine_id,
                                     params=script_params,
                                     host=host,
                                     job_id=job_id)
                error = ret['error']
                tmp_log('executed script_id %s', script_id)
            elif command:
                tmp_log('will run command %s', command)
                log_event(action='deployment_script_started',
                          command=command,
                          **log_dict)
                start_time = time()
                retval, output = shell.command(command)
                tmp_log('executed command %s', command)
                execution_time = time() - start_time
                output = output.decode('utf-8', 'ignore')
                title = "Deployment script %s" % ('failed'
                                                  if retval else 'succeeded')
                error = retval > 0
                notify_user(user,
                            title,
                            cloud_id=cloud_id,
                            machine_id=machine_id,
                            machine_name=node.name,
                            command=command,
                            output=output,
                            duration=execution_time,
                            retval=retval,
                            error=retval > 0)
                log_event(action='deployment_script_finished',
                          error=retval > 0,
                          return_value=retval,
                          command=command,
                          stdout=output,
                          **log_dict)

            shell.disconnect()

            if monitoring:
                try:
                    enable_monitoring(
                        user,
                        cloud_id,
                        node.id,
                        name=node.name,
                        dns_name=node.extra.get('dns_name', ''),
                        public_ips=ips,
                        no_ssh=False,
                        dry=False,
                        job_id=job_id,
                        plugins=plugins,
                        deploy_async=False,
                    )
                except Exception as e:
                    print repr(e)
                    error = True
                    notify_user(
                        user,
                        "Enable monitoring failed for machine %s" % machine_id,
                        repr(e))
                    notify_admin(
                        'Enable monitoring on creation failed for user %s machine %s: %r'
                        % (email, machine_id, e))
                    log_event(action='enable_monitoring_failed',
                              error=repr(e),
                              **log_dict)

            if post_script_id and multi_user:
                tmp_log('will run post_script_id %s', post_script_id)
                ret = run_script.run(
                    user.email,
                    post_script_id,
                    cloud_id,
                    machine_id,
                    params=post_script_params,
                    host=host,
                    job_id=job_id,
                    action_prefix='post_',
                )
                error = ret['error']
                tmp_log('executed post_script_id %s', script_id)

            log_event(action='post_deploy_finished', error=error, **log_dict)

        except (ServiceUnavailableError, SSHException) as exc:
            tmp_log(repr(exc))
            raise self.retry(exc=exc, countdown=60, max_retries=15)
    except Exception as exc:
        tmp_log(repr(exc))
        if str(exc).startswith('Retry'):
            raise
        notify_user(user,
                    "Deployment script failed for machine %s" % machine_id)
        notify_admin(
            "Deployment script failed for machine %s in cloud %s by user %s" %
            (machine_id, cloud_id, email), repr(exc))
        log_event(email=email,
                  event_type='job',
                  action='post_deploy_finished',
                  cloud_id=cloud_id,
                  machine_id=machine_id,
                  enable_monitoring=bool(monitoring),
                  command=command,
                  error="Couldn't connect to run post deploy steps.",
                  job_id=job_id)
Exemple #33
0
def deploy_collectd(email, backend_id, machine_id, extra_vars):
    import mist.io.methods
    user = user_from_email(email)
    mist.io.methods.deploy_collectd(user, backend_id, machine_id, extra_vars)
Exemple #34
0
def undeploy_collectd(email, backend_id, machine_id):
    user = user_from_email(email)
    import mist.io.methods
    mist.io.methods.undeploy_collectd(user, backend_id, machine_id)
Exemple #35
0
 def execute(self, email, backend_id):
     from mist.io import methods
     user = user_from_email(email)
     locations = methods.list_locations(user, backend_id)
     return {'backend_id': backend_id, 'locations': locations}
Exemple #36
0
def post_deploy_steps(self,
                      email,
                      backend_id,
                      machine_id,
                      monitoring,
                      command,
                      key_id=None,
                      username=None,
                      password=None,
                      port=22):
    from mist.io.methods import ssh_command, connect_provider, enable_monitoring
    from mist.io.methods import notify_user, notify_admin
    if multi_user:
        from mist.core.methods import enable_monitoring
        from mist.core.helpers import log_event
    else:
        from mist.io.methods import enable_monitoring
        log_event = lambda *args, **kwargs: None

    user = user_from_email(email)
    try:

        # find the node we're looking for and get its hostname
        conn = connect_provider(user.backends[backend_id])
        nodes = conn.list_nodes()
        node = None
        for n in nodes:
            if n.id == machine_id:
                node = n
                break

        if node and len(node.public_ips):
            # filter out IPv6 addresses
            ips = filter(lambda ip: ':' not in ip, node.public_ips)
            host = ips[0]
        else:
            raise self.retry(exc=Exception(), countdown=120, max_retries=5)

        try:
            from mist.io.shell import Shell
            shell = Shell(host)
            # connect with ssh even if no command, to create association
            # to be able to enable monitoring
            key_id, ssh_user = shell.autoconfigure(user, backend_id, node.id,
                                                   key_id, username, password,
                                                   port)

            backend = user.backends[backend_id]
            msg = "Backend:\n  Name: %s\n  Id: %s\n" % (backend.title,
                                                        backend_id)
            msg += "Machine:\n  Name: %s\n  Id: %s\n" % (node.name, node.id)
            if command:
                log_dict = {
                    'email': email,
                    'event_type': 'job',
                    'backend_id': backend_id,
                    'machine_id': machine_id,
                    'job_id': uuid.uuid4().hex,
                    'command': command,
                    'host': host,
                    'key_id': key_id,
                    'ssh_user': ssh_user,
                }
                log_event(action='deployment_script_started', **log_dict)
                start_time = time()
                retval, output = shell.command(command)
                execution_time = time() - start_time
                output = output.decode('utf-8', 'ignore')
                title = "Deployment script %s" % ('failed'
                                                  if retval else 'succeeded')
                notify_user(user,
                            title,
                            backend_id=backend_id,
                            machine_id=machine_id,
                            machine_name=node.name,
                            command=command,
                            output=output,
                            duration=execution_time,
                            retval=retval,
                            error=retval > 0)
                log_event(action='deployment_script_finished',
                          error=retval > 0,
                          return_value=retval,
                          stdout=output,
                          **log_dict)

            shell.disconnect()

            if monitoring:
                try:
                    enable_monitoring(
                        user,
                        backend_id,
                        node.id,
                        name=node.name,
                        dns_name=node.extra.get('dns_name', ''),
                        public_ips=ips,
                        no_ssh=False,
                        dry=False,
                    )
                except Exception as e:
                    print repr(e)
                    notify_user(
                        user, "Enable monitoring failed for machine %s (%s)" %
                        (node.name, node.id), repr(e))
                    notify_admin(
                        'Enable monitoring on creation failed for user %s machine %s: %r'
                        % (email, node.name, e))

        except (ServiceUnavailableError, SSHException) as exc:
            raise self.retry(exc=exc, countdown=60, max_retries=5)
    except Exception as exc:
        if str(exc).startswith('Retry'):
            raise
        notify_user(
            user, "Deployment script failed for machine %s after 5 retries" %
            node.id)
        notify_admin(
            "Deployment script failed for machine %s in backend %s by user %s after 5 retries"
            % (node.id, backend_id, email), repr(exc))
        log_event(
            email=email,
            event_type='job',
            action='deployment_script_failed',
            backend_id=backend_id,
            machine_id=machine_id,
            enable_monitoring=bool(monitoring),
            command=command,
            error="Couldn't connect to run post deploy steps (5 attempts).",
        )
Exemple #37
0
 def execute(self, email, backend_id):
     from mist.io import methods
     user = user_from_email(email)
     images = methods.list_images(user, backend_id)
     return {'backend_id': backend_id, 'images': images}
Exemple #38
0
def hpcloud_post_create_steps(self,
                              email,
                              cloud_id,
                              machine_id,
                              monitoring,
                              command,
                              key_id,
                              username,
                              password,
                              public_key,
                              script_id='',
                              script_params='',
                              job_id=None,
                              hostname='',
                              plugins=None,
                              post_script_id='',
                              post_script_params=''):
    from mist.io.methods import connect_provider
    user = user_from_email(email)

    try:
        conn = connect_provider(user.clouds[cloud_id])
        nodes = conn.list_nodes()
        node = None

        for n in nodes:
            if n.id == machine_id:
                node = n
                break

        if node and node.state == 0 and len(node.public_ips):
            # filter out IPv6 addresses
            ips = filter(lambda ip: ':' not in ip, node.public_ips)
            host = ips[0]

            post_deploy_steps.delay(
                email,
                cloud_id,
                machine_id,
                monitoring,
                command,
                key_id,
                script_id=script_id,
                script_params=script_params,
                job_id=job_id,
                hostname=hostname,
                plugins=plugins,
                post_script_id=post_script_id,
                post_script_params=post_script_params,
            )

        else:
            try:
                available_networks = conn.ex_list_networks()
                external_networks = [
                    net for net in available_networks if net.router_external
                ]
                if external_networks:
                    ext_net_id = external_networks[0].id
                else:
                    ext_net_id = ""

                ports = conn.ex_list_ports()

                port = [
                    port for port in ports
                    if port.get("device_id", "") == node.id
                ][0]

                ip = conn.ex_create_floating_ip(ext_net_id, port['id'])
                post_deploy_steps.delay(
                    email,
                    cloud_id,
                    machine_id,
                    monitoring,
                    command,
                    key_id,
                    script_id=script_id,
                    script_params=script_params,
                    job_id=job_id,
                    hostname=hostname,
                    plugins=plugins,
                    post_script_id=post_script_id,
                    post_script_params=post_script_params,
                )

            except:
                raise self.retry(exc=Exception(), max_retries=20)
    except Exception as exc:
        if str(exc).startswith('Retry'):
            raise
Exemple #39
0
def openstack_post_create_steps(self, email, backend_id, machine_id, monitoring,
                              command, key_id, username, password, public_key,
                              script_id='', script_params='', job_id=None,
                              hostname='', plugins=None,
                              post_script_id='', post_script_params='', networks=[]):

    from mist.io.methods import connect_provider
    user = user_from_email(email)

    try:
        conn = connect_provider(user.backends[backend_id])
        nodes = conn.list_nodes()
        node = None

        for n in nodes:
            if n.id == machine_id:
                node = n
                break

        if node and node.state == 0 and len(node.public_ips):
            # filter out IPv6 addresses
            ips = filter(lambda ip: ':' not in ip, node.public_ips)
            host = ips[0]

            post_deploy_steps.delay(
                email, backend_id, machine_id, monitoring, command, key_id,
                script_id=script_id, script_params=script_params,
                job_id=job_id, hostname=hostname, plugins=plugins,
                post_script_id=post_script_id,
                post_script_params=post_script_params,
            )

        else:
            try:
                created_floating_ips = []
                for network in networks['public']:
                    created_floating_ips += [floating_ip for floating_ip in network['floating_ips']]

                # From the already created floating ips try to find one that is not associated to a node
                unassociated_floating_ip = None
                for ip in created_floating_ips:
                    if not ip['node_id']:
                        unassociated_floating_ip = ip
                        break

                # Find the ports which are associated to the machine (e.g. the ports of the private ips)
                # and use one to associate a floating ip
                ports = conn.ex_list_ports()
                machine_port_id = None
                for port in ports:
                    if port.get('device_id') == node.id:
                        machine_port_id = port.get('id')
                        break

                if unassociated_floating_ip:
                    log.info("Associating floating ip with machine: %s" % node.id)
                    ip = conn.ex_associate_floating_ip_to_node(unassociated_floating_ip['id'], machine_port_id)
                else:
                    # Find the external network
                    log.info("Create and associating floating ip with machine: %s" % node.id)
                    ext_net_id = networks['public'][0]['id']
                    ip = conn.ex_create_floating_ip(ext_net_id, machine_port_id)

                post_deploy_steps.delay(
                    email, backend_id, machine_id, monitoring, command, key_id,
                    script_id=script_id, script_params=script_params,
                    job_id=job_id, hostname=hostname, plugins=plugins,
                    post_script_id=post_script_id,
                    post_script_params=post_script_params,
                )

            except:
                raise self.retry(exc=Exception(), max_retries=20)
    except Exception as exc:
        if str(exc).startswith('Retry'):
            raise
Exemple #40
0
def azure_post_create_steps(self, email, backend_id, machine_id, monitoring,
                            command, key_id, username, password, public_key,
                            script_id='', script_params='', job_id=None,
                            hostname='', plugins=None,
                            post_script_id='', post_script_params=''):
    from mist.io.methods import connect_provider
    user = user_from_email(email)

    try:
        # find the node we're looking for and get its hostname
        conn = connect_provider(user.backends[backend_id])
        nodes = conn.list_nodes()
        node = None
        for n in nodes:
            if n.id == machine_id:
                node = n
                break

        if node and node.state == 0 and len(node.public_ips):
            # filter out IPv6 addresses
            ips = filter(lambda ip: ':' not in ip, node.public_ips)
            host = ips[0]
        else:
            raise self.retry(exc=Exception(), max_retries=20)

        try:
            # login with user, password. Deploy the public key, enable sudo access for
            # username, disable password authentication and reload ssh.
            # After this is done, call post_deploy_steps if deploy script or monitoring
            # is provided
            ssh = paramiko.SSHClient()
            ssh.load_system_host_keys()
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            ssh.connect(host, username=username, password=password, timeout=None, allow_agent=False, look_for_keys=False)

            ssh.exec_command('mkdir -p ~/.ssh && echo "%s" >> ~/.ssh/authorized_keys && chmod -R 700 ~/.ssh/' % public_key)

            chan = ssh.get_transport().open_session()
            chan.get_pty()
            chan.exec_command('sudo su -c \'echo "%s ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers\' ' % username)
            chan.send('%s\n' % password)

            check_sudo_command = 'sudo su -c \'whoami\''

            chan = ssh.get_transport().open_session()
            chan.get_pty()
            chan.exec_command(check_sudo_command)
            output = chan.recv(1024)

            if not output.startswith('root'):
                raise
            cmd = 'sudo su -c \'sed -i "s|[#]*PasswordAuthentication yes|PasswordAuthentication no|g" /etc/ssh/sshd_config &&  /etc/init.d/ssh reload; service ssh reload\' '
            ssh.exec_command(cmd)

            ssh.close()

            post_deploy_steps.delay(
                email, backend_id, machine_id, monitoring, command, key_id,
                script_id=script_id, script_params=script_params,
                job_id=job_id, hostname=hostname, plugins=plugins,
                post_script_id=post_script_id,
                post_script_params=post_script_params,
            )

        except Exception as exc:
            raise self.retry(exc=exc, countdown=10, max_retries=15)
    except Exception as exc:
        if str(exc).startswith('Retry'):
            raise
Exemple #41
0
def rackspace_first_gen_post_create_steps(self,
                                          email,
                                          backend_id,
                                          machine_id,
                                          monitoring,
                                          command,
                                          key_id,
                                          password,
                                          public_key,
                                          username='******'):
    from mist.io.methods import ssh_command, connect_provider, enable_monitoring
    from mist.io.methods import notify_user, notify_admin
    user = user_from_email(email)

    try:
        # find the node we're looking for and get its hostname
        conn = connect_provider(user.backends[backend_id])
        nodes = conn.list_nodes()
        node = None
        for n in nodes:
            if n.id == machine_id:
                node = n
                break

        if node and node.state == 0 and len(node.public_ips):
            # filter out IPv6 addresses
            ips = filter(lambda ip: ':' not in ip, node.public_ips)
            host = ips[0]
        else:
            raise self.retry(exc=Exception(), max_retries=20)

        try:
            # login with user, password and deploy the ssh public key. Disable password authentication and reload ssh.
            # After this is done, call post_deploy_steps if deploy script or monitoring
            # is provided
            ssh = paramiko.SSHClient()
            ssh.load_system_host_keys()
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            ssh.connect(host,
                        username=username,
                        password=password,
                        timeout=None,
                        allow_agent=False,
                        look_for_keys=False)

            ssh.exec_command(
                'mkdir -p ~/.ssh && echo "%s" >> ~/.ssh/authorized_keys && chmod -R 700 ~/.ssh/'
                % public_key)

            cmd = 'sudo su -c \'sed -i "s|[#]*PasswordAuthentication yes|PasswordAuthentication no|g" /etc/ssh/sshd_config &&  /etc/init.d/ssh reload; service ssh reload\' '
            ssh.exec_command(cmd)

            ssh.close()

            if command or monitoring:
                post_deploy_steps.delay(email, backend_id, machine_id,
                                        monitoring, command, key_id)

        except Exception as exc:
            raise self.retry(exc=exc, countdown=10, max_retries=15)
    except Exception as exc:
        if str(exc).startswith('Retry'):
            raise
Exemple #42
0
def run_deploy_script(self,
                      email,
                      backend_id,
                      machine_id,
                      command,
                      key_id=None,
                      username=None,
                      password=None,
                      port=22):
    from mist.io.methods import ssh_command, connect_provider
    from mist.io.methods import notify_user, notify_admin

    user = user_from_email(email)

    try:
        # find the node we're looking for and get its hostname
        conn = connect_provider(user.backends[backend_id])
        nodes = conn.list_nodes()
        node = None
        for n in nodes:
            if n.id == machine_id:
                node = n
                break

        if node and len(node.public_ips):
            # filter out IPv6 addresses
            ips = filter(lambda ip: ':' not in ip, node.public_ips)
            host = ips[0]
        else:
            raise self.retry(exc=Exception(), countdown=60, max_retries=5)

        try:
            from mist.io.shell import Shell
            shell = Shell(host)
            key_id, ssh_user = shell.autoconfigure(user, backend_id, node.id,
                                                   key_id, username, password,
                                                   port)
            start_time = time()
            retval, output = shell.command(command)
            execution_time = time() - start_time
            shell.disconnect()
            msg = """
Command: %s
Return value: %s
Duration: %s seconds
Output:
%s""" % (command, retval, execution_time, output)

            if retval:
                notify_user(
                    user, "Deployment script failed for machine %s (%s)" %
                    (node.name, node.id), msg)
                amqp_log(
                    "Deployment script failed for user %s machine %s (%s): %s"
                    % (user, node.name, node.id, msg))
            else:
                notify_user(
                    user, "Deployment script succeeded for machine %s (%s)" %
                    (node.name, node.id), msg)
                amqp_log(
                    "Deployment script succeeded for user %s machine %s (%s): %s"
                    % (user, node.name, node.id, msg))

        except ServiceUnavailableError as exc:
            raise self.retry(exc=exc, countdown=60, max_retries=5)
    except Exception as exc:
        if str(exc).startswith('Retry'):
            return
        amqp_log(
            "Deployment script failed for machine %s in backend %s by user %s after 5 retries: %s"
            % (node.id, backend_id, email, repr(exc)))
        notify_user(
            user, "Deployment script failed for machine %s after 5 retries" %
            node.id)
        notify_admin(
            "Deployment script failed for machine %s in backend %s by user %s after 5 retries"
            % (node.id, backend_id, email), repr(exc))
Exemple #43
0
def post_deploy_steps(self, email, backend_id, machine_id, monitoring, command,
                      key_id=None, username=None, password=None, port=22):
    from mist.io.methods import ssh_command, connect_provider, enable_monitoring
    from mist.io.methods import notify_user, notify_admin
    if multi_user:
        from mist.core.methods import enable_monitoring
    else:
        from mist.io.methods import enable_monitoring

    user = user_from_email(email)
    try:

        # find the node we're looking for and get its hostname
        conn = connect_provider(user.backends[backend_id])
        nodes = conn.list_nodes()
        node = None
        for n in nodes:
            if n.id == machine_id:
                node = n
                break

        if node and len(node.public_ips):
            # filter out IPv6 addresses
            ips = filter(lambda ip: ':' not in ip, node.public_ips)
            host = ips[0]
        else:
            raise self.retry(exc=Exception(), countdown=120, max_retries=5)

        try:
            from mist.io.shell import Shell
            shell = Shell(host)
            # connect with ssh even if no command, to create association
            # to be able to enable monitoring
            key_id, ssh_user = shell.autoconfigure(
                user, backend_id, node.id, key_id, username, password, port
            )

            if command:
                start_time = time()
                retval, output = shell.command(command)
                execution_time = time() - start_time
                output = output.decode('utf-8','ignore')
                msg = ("Command: %s\n"
                       "Return value: %s\n"
                       "Duration: %d seconds\n"
                       "Output:%s\n") % (command, retval,
                                         execution_time, output)
                msg = msg.encode('utf-8', 'ignore')
                msg_title = "Deployment script %s for machine %s (%s)" % (
                    'failed' if retval else 'succeeded',
                    node.name, node.id
                )
                notify_user(user, msg_title, msg)

            shell.disconnect()

            if monitoring:
                try:
                    enable_monitoring(user, backend_id, node.id,
                        name=node.name, dns_name=node.extra.get('dns_name',''),
                        public_ips=ips, no_ssh=False, dry=False,
                    )
                except Exception as e:
                    print repr(e)
                    notify_user(user, "Enable monitoring failed for machine %s (%s)" % (node.name, node.id), repr(e))
                    notify_admin('Enable monitoring on creation failed for user %s machine %s: %r' % (email, node.name, e))

        except (ServiceUnavailableError, SSHException) as exc:
            raise self.retry(exc=exc, countdown=60, max_retries=5)
    except Exception as exc:
        if str(exc).startswith('Retry'):
            raise
        amqp_log("Deployment script failed for machine %s in backend %s by user %s after 5 retries: %s" % (node.id, backend_id, email, repr(exc)))
        notify_user(user, "Deployment script failed for machine %s after 5 retries" % node.id)
        notify_admin("Deployment script failed for machine %s in backend %s by user %s after 5 retries" % (node.id, backend_id, email), repr(exc))
Exemple #44
0
 def execute(self, email, backend_id):
     from mist.io import methods
     user = user_from_email(email)
     sizes = methods.list_sizes(user, backend_id)
     return {'backend_id': backend_id, 'sizes': sizes}