コード例 #1
0
ファイル: stack.py プロジェクト: experimentAccount0/client
 def sync_metadata(self):
     path = os.path.join(self.path, '.cloudmesh_metadata')
     Console.debug_msg('Saving {} to {}'.format(self.__class__.__name__,
                                                path))
     y = yaml.dump(self.__dict__, default_flow_style=False)
     with open(path, 'w') as fd:
         fd.write(y)
コード例 #2
0
ファイル: vm.py プロジェクト: cloudmesh/client
    def refresh(cls, **kwargs):
        # print("Inside refresh")

        refreshed = cls.cm.refresh("vm", kwargs["cloud"])

        # update counter
        vms = cls.cm.find(kind='vm')
        me  = Default.user
        for vm in vms:
            name = vm['name']
            if not name.startswith(me):
                continue

            number = name.split('-')[-1]

            try:
                # +1 as the stored counter is the next available counter
                new_counter = int(number) + 1
            except ValueError:
                # name is not formatted correctly, possibly due to not
                # being started using cloudmesh
                continue

            old_counter = Default.get_counter(Names.VM_COUNTER)
            counter = max(new_counter, old_counter)
            Default.set_counter(Names.VM_COUNTER, counter)

        Console.debug_msg('Set counter ' + Names.VM_COUNTER + ' to ' +
                          str(Default.get_counter(Names.VM_COUNTER)))

        return refreshed
コード例 #3
0
ファイル: vm.py プロジェクト: jemitchell/client
    def refresh(cls, **kwargs):
        # print("Inside refresh")

        refreshed = cls.cm.refresh("vm", kwargs["cloud"])

        # update counter
        vms = cls.cm.find(kind='vm')
        me = Default.user
        for vm in vms:
            name = vm['name']
            if not name.startswith(me):
                continue

            number = name.split('-')[-1]

            try:
                # +1 as the stored counter is the next available counter
                new_counter = int(number) + 1
            except ValueError:
                # name is not formatted correctly, possibly due to not
                # being started using cloudmesh
                continue

            old_counter = Default.get_counter(Names.VM_COUNTER)
            counter = max(new_counter, old_counter)
            Default.set_counter(Names.VM_COUNTER, counter)

        Console.debug_msg('Set counter ' + Names.VM_COUNTER + ' to ' +
                          str(Default.get_counter(Names.VM_COUNTER)))

        return refreshed
コード例 #4
0
ファイル: stack.py プロジェクト: experimentAccount0/client
 def load(cls, path):
     filename = os.path.join(path, '.cloudmesh_metadata')
     Console.debug_msg('Loading {} to {}'.format(cls.__name__, filename))
     with open(filename) as fd:
         d = yaml.load(fd)
     stack = cls(dest=path, **d)
     stack._env = d['_env']
     return stack
コード例 #5
0
ファイル: Shell.py プロジェクト: cloudmesh/client
    def __init__(self, cmd, cwd=None, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=None):

        Console.debug_msg('Running cmd: {}'.format(' '.join(map(quote, cmd))))

        proc = subprocess.Popen(cmd, stderr=stderr, stdout=stdout, cwd=cwd, env=env)
        stdout, stderr = proc.communicate()

        self.returncode = proc.returncode
        self.stderr = stderr
        self.stdout = stdout

        if self.returncode != 0:
            raise SubprocessError(cmd, self.returncode, self.stderr, self.stdout)
コード例 #6
0
ファイル: stack.py プロジェクト: jemitchell/client
    def init(self, force=False, update=False):
        """Initialize by cloning (or updating if requested) a local copy of
        the Big Data Stack repository.

        :param bool force: setup previously setup project
        :type force: :class:`bool`
        :param bool update: update the local repository from the origin
        :type update: :class:`bool`

        """

        if not os.path.isdir(os.path.join(self.path, '.git')):
            Console.debug_msg('Cloning branch {} of {} to {}'.format(
                self.branch, self.repo, self.path))
            Subprocess([
                'git', 'clone', '--recursive', '--branch', self.branch,
                self.repo, self.path
            ])

        elif update:
            Console.debug_msg('Updating to branch {} for {}'.format(
                self.branch, self.path))
            Subprocess([
                'git', 'fetch', '--recurse-submodules', 'origin', self.branch
            ],
                       cwd=self.path)
            Subprocess(['git', 'checkout', self.branch], cwd=self.path)
            Subprocess(['git', 'merge', 'origin/{}'.format(self.branch)],
                       cwd=self.path)

        venvname = 'venv'
        venvdir = os.path.join(self.path, venvname)

        if force and os.path.isfile(os.path.join(venvdir, 'bin', 'activate')):
            Console.debug_msg('Removing {}'.format(venvdir))
            shutil.rmtree(venvdir)

        if not os.path.isdir(venvdir):
            Console.debug_msg('Creating virtualenv {}'.format(venvdir))
            Subprocess(['virtualenv', venvdir])

        self._env = get_virtualenv_environment(venvdir)
        cmd = ['pip', 'install', '-r', 'requirements.txt'
               ] + (['-U'] if force else [])
        Console.debug_msg('Installing requirements to {}'.format(venvdir))
        Subprocess(cmd, cwd=self.path, env=self._env)
コード例 #7
0
    def __init__(self,
                 inventory,
                 roles=None,
                 path=None,
                 username=None,
                 become=False,
                 become_user=None,
                 forks=None,
                 modifyKnownHosts=True,
                 extra_vars=None,
                 subprocess_kwargs=None):
        self._inventory = inventory
        self._roles = roles or []
        self._path = path
        self._username = username
        self._become = become
        self._become_user = become_user
        self._forks = forks
        self._modifyKnownHosts = modifyKnownHosts
        self._extra_vars = extra_vars
        self._subprocess_kwargs = subprocess_kwargs or dict()

        ################################################################ rest of setup

        self._inventory_file = tempfile.NamedTemporaryFile()
        self._playbook_file = tempfile.NamedTemporaryFile()

        # write inventory
        self._inventory_file.write(self._inventory.ini())
        self._inventory_file.flush()

        Console.debug_msg('Inventory:\n' + self._inventory.ini())

        # write playbook
        playbook = [{
            'name': role.name,
            'hosts': role.hosts,
            'become': role.become,
            'roles': [dict(role=role.path, **role.variables)]
        } for role in roles]
        yml = yaml.dump(playbook)
        self._playbook_file.write(yml)
        self._playbook_file.flush()

        Console.debug_msg('Playbook:\n' + yml)
コード例 #8
0
ファイル: playbook.py プロジェクト: cloudmesh/client
    def __init__(self, inventory, roles=None, path=None, username=None, become=False,
                 become_user=None, forks=None, modifyKnownHosts=True,
                 extra_vars=None, subprocess_kwargs=None):
        self._inventory = inventory
        self._roles = roles or []
        self._path = path
        self._username = username
        self._become = become
        self._become_user = become_user
        self._forks = forks
        self._modifyKnownHosts = modifyKnownHosts
        self._extra_vars = extra_vars
        self._subprocess_kwargs = subprocess_kwargs or dict()

        ################################################################ rest of setup

        self._inventory_file = tempfile.NamedTemporaryFile()
        self._playbook_file  = tempfile.NamedTemporaryFile()

        # write inventory
        self._inventory_file.write(self._inventory.ini())
        self._inventory_file.flush()

        Console.debug_msg('Inventory:\n' + self._inventory.ini())

        # write playbook
        playbook = [{
            'name': role.name,
            'hosts': role.hosts,
            'become': role.become,
            'roles': [dict(role=role.path,
                           **role.variables)]}
                    for role in roles]
        yml = yaml.dump(playbook)
        self._playbook_file.write(yml)
        self._playbook_file.flush()

        Console.debug_msg('Playbook:\n' + yml)
コード例 #9
0
    def __init__(self,
                 cmd,
                 cwd=None,
                 stderr=subprocess.PIPE,
                 stdout=subprocess.PIPE,
                 env=None):

        Console.debug_msg('Running cmd: {}'.format(' '.join(map(quote, cmd))))

        proc = subprocess.Popen(cmd,
                                stderr=stderr,
                                stdout=stdout,
                                cwd=cwd,
                                env=env)
        stdout, stderr = proc.communicate()

        self.returncode = proc.returncode
        self.stderr = stderr
        self.stdout = stdout

        if self.returncode != 0:
            raise SubprocessError(cmd, self.returncode, self.stderr,
                                  self.stdout)
コード例 #10
0
ファイル: stack.py プロジェクト: jemitchell/client
 def use_bds(self):
     """Use the BigDataStack backend"""
     Console.debug_msg('Factory use_bds')
     self.is_bds = True
     return self
コード例 #11
0
ファイル: CloudmeshDatabase.py プロジェクト: cloudmesh/client
 def set_defaults(self, **kwargs):   # TODO: what is this method used for?
     Console.debug_msg('Call to CloudmeshVMMixin')
     pass
コード例 #12
0
 def playbook(self, cluster=None, path=None):
     Console.debug_msg('NotImplementedError')
コード例 #13
0
 def __init__(self, context):
     Console.debug_msg("init command deploy")
コード例 #14
0
ファイル: stack.py プロジェクト: jemitchell/client
    def deploy(self,
               ips=None,
               name=None,
               user=None,
               playbooks=None,
               defines=None,
               ping_max=10,
               ping_sleep=10):
        """Deploy the big-data-stack to a previously stood up cluster located
        at `ips` with login user `user`.

        :param ips: the ip addresses of the cluster to deploy to
        :type ips: :class:`list` of :class:`str` IP addresses
        :param name: the name of the cluster
        :type name: :class:`str`
        :param user: the login username of the cluster
        :type user: :class:`str`
        :param playbooks: the list of playbooks to deploy. These are paths relative to the root directory of the BDS repository.
        :type playbooks: :class:`list` of :class:`str`
        :param defines: the overridden variables defined for each playbook
        :type defines: :class:`dict` from playbook name to :class:`dict` of variable name to value
        :param ping_max: the maximum number of time to attempt to ping the cluster during the verification step.
        :type ping_max: :class:`int`
        :param ping_sleep: the number of seconds to wait between each attempt to ping
        :type ping_sleep: :class:`int`
        """
        assert ips is not None
        assert ping_max > 0, ping_max
        assert ping_sleep > 0, ping_sleep

        name = name or os.getenv('USER') + '-' + os.path.basename(self.path)
        user = user or 'defaultuser'
        playbooks = playbooks or list()
        defines = defines or defaultdict(list)

        Console.debug_msg('Calling mk-inventory in {}'.format(self.path))
        cmd = ['python', 'mk-inventory', '-n', name] + ips
        inventory = Subprocess(cmd, cwd=self.path, env=self._env)
        Console.debug_msg('Writing inventory file')
        Console.debug_msg('\n    ' +
                          ('\n' + 4 * ' ').join(inventory.stdout.split('\n')))
        with open(os.path.join(self.path, 'inventory.txt'), 'w') as fd:
            fd.write(inventory.stdout)

        Console.info('Waiting for cluster to be accessible')
        ping_ok = False
        for i in xrange(ping_max):
            Console.debug_msg('Attempt {} / {}'.format(i + 1, ping_max))
            try:
                Subprocess(['ansible', 'all', '-m', 'ping', '-u', user],
                           cwd=self.path,
                           env=self._env,
                           stdout=None,
                           stderr=None)
                ping_ok = True
                Console.debug_msg('Success!')
                break
            except SubprocessError:
                Console.debug_msg(
                    'Failure, sleeping for {} seconds'.format(ping_sleep))
                time.sleep(ping_sleep)

        if not ping_ok:
            msg = 'Ping Failure'
            reason = 'Unable to connect to all nodes'
            Console.error(' '.join([msg, reason]))
            raise SanityCheckError(message=msg, reason=reason)

        basic_command = ['ansible-playbook', '-u', user]
        Console.debug_msg('Running playbooks {}'.format(playbooks))
        for play in playbooks:
            cmd = basic_command + [play]
            define = ['{}={}'.format(k, v) for k, v in defines[play]]
            if define:
                cmd.extend(['-e', ','.join(define)])
            Console.info('Running playbook {} with overrides {}'.format(
                play, define))
            Subprocess(cmd,
                       cwd=self.path,
                       env=self._env,
                       stdout=None,
                       stderr=None)
コード例 #15
0
 def set_defaults(self, **kwargs):  # TODO: what is this method used for?
     Console.debug_msg('Call to CloudmeshVMMixin')
     pass
コード例 #16
0
ファイル: stack.py プロジェクト: jemitchell/client
 def activate(self, make_active=True):
     """Set whether or not the created project should be activated"""
     Console.debug_msg('Factory activate: {}'.format(make_active))
     self.make_active = make_active
     return self
コード例 #17
0
ファイル: stack.py プロジェクト: jemitchell/client
 def set_repo(self, repo):
     """Set the repository to get the stack from"""
     Console.debug_msg('Factory set_repo: {}'.format(repo))
     self.repo = repo
     return self
コード例 #18
0
ファイル: stack.py プロジェクト: jemitchell/client
 def set_force(self, force=False):
     """Set the `force` parameter"""
     Console.debug_msg('Factory set_force: {}'.format(force))
     self.force = force
     return self
コード例 #19
0
ファイル: stack.py プロジェクト: jemitchell/client
 def set_update(self, update=False):
     """Set the `update` parameter"""
     Console.debug_msg('Factory set_update: {}'.format(update))
     self.update = update
     return self
コード例 #20
0
ファイル: stack.py プロジェクト: jemitchell/client
 def set_project_name(self, name):
     """Set the project name"""
     Console.debug_msg('Factory set_project_name: {}'.format(name))
     self.project_name = name
     return self
コード例 #21
0
ファイル: stack.py プロジェクト: jemitchell/client
 def set_overrides(self, overrides=None):
     """Set the overrides for deployment"""
     Console.debug_msg('Factory set_overrides: {}'.format(overrides))
     overrides = overrides or defaultdict(lambda: defaultdict(list))
     self.overrides = overrides
     return self
コード例 #22
0
ファイル: stack.py プロジェクト: jemitchell/client
 def set_branch(self, branch='master'):
     """Set the branch of the repository to use"""
     Console.debug_msg('Factory set_branch: {}'.format(branch))
     self.branch = branch
     return self
コード例 #23
0
ファイル: stack.py プロジェクト: experimentAccount0/client
    def deploy(self,
               ips=None,
               name=None,
               user=None,
               playbooks=None,
               defines=None):
        """Deploy the big-data-stack to a previously stood up cluster located
        at `ips` with login user `user`.

        :param ips: the ip addresses of the cluster to deploy to
        :type ips: :class:`list` of :class:`str` IP addresses
        :param name: the name of the cluster
        :type name: :class:`str`
        :param user: the login username of the cluster
        :type user: :class:`str`
        :param playbooks: the list of playbooks to deploy. These are paths relative to the root directory of the BDS repository.
        :type playbooks: :class:`list` of :class:`str`
        :param defines: the overridden variables defined for each playbook
        :type defines: :class:`dict` from playbook name to :class:`dict` of variable name to value
        :param ping_max: the maximum number of time to attempt to ping the cluster during the verification step.
        :type ping_max: :class:`int`
        :param ping_sleep: the number of seconds to wait between each attempt to ping
        :type ping_sleep: :class:`int`
        """
        assert ips is not None

        name = name or os.getenv('USER') + '-' + os.path.basename(self.path)
        user = user or 'defaultuser'
        playbooks = playbooks or list()
        defines = defines or defaultdict(list)

        Console.debug_msg('Calling mk-inventory in {}'.format(self.path))
        cmd = ['python', 'mk-inventory', '-n', name] + ips
        inventory = Subprocess(cmd, cwd=self.path, env=self._env)
        Console.debug_msg('Writing inventory file')
        Console.debug_msg('\n    ' +
                          ('\n' + 4 * ' ').join(inventory.stdout.split('\n')))
        with open(os.path.join(self.path, 'inventory.txt'), 'w') as fd:
            fd.write(inventory.stdout)

        Console.info('Waiting for cluster to be accessible')

        def ping():
            try:
                Subprocess(['ansible', 'all', '-m', 'ping', '-u', user],
                           cwd=self.path,
                           env=self._env,
                           stdout=None,
                           stderr=None)
                return True
            except SubprocessError:
                return False

        exponential_backoff(ping)

        basic_command = ['ansible-playbook', '-u', user]
        Console.debug_msg('Running playbooks {}'.format(playbooks))
        for play in playbooks:
            donefile = os.path.join(self.path, play) + '.done'
            if os.path.exists(donefile):
                Console.ok('Skipping completed play %s' % play)
                continue

            cmd = basic_command + [play]
            define = ['{}={}'.format(k, v) for k, v in defines[play]]
            if define:
                cmd.extend(['-e', ','.join(define)])
            Console.info('Running playbook {} with overrides {}'.format(
                play, define))
            Subprocess(cmd,
                       cwd=self.path,
                       env=self._env,
                       stdout=None,
                       stderr=None)
            with open(donefile, 'w') as fd:
                fd.write('')
コード例 #24
0
ファイル: stack.py プロジェクト: jemitchell/client
 def set_user_name(self, username):
     """Set the cluster login user name"""
     Console.debug_msg('Factory set_user_name: {}'.format(username))
     self.username = username
     return self
コード例 #25
0
ファイル: stack.py プロジェクト: jemitchell/client
 def set_playbooks(self, playbooks=None):
     """Set the playbooks to deploy"""
     Console.debug_msg('Factory set_playbooks: {}'.format(playbooks))
     playbooks = playbooks or list()
     self.playbooks = playbooks
     return self
コード例 #26
0
ファイル: stack.py プロジェクト: jemitchell/client
 def set_ips(self, ips):
     """Set the cluster IP addresses"""
     Console.debug_msg('Factory set_ips: {}'.format(ips))
     assert len(ips) >= 0
     self.ips = ips
     return self