def sync_metadata(self):
path = os.path.join(self.path, '.cloudmesh_metadata')
Console.debug_msg('Saving {} to {}'.format(self.__class__.__name__,
path))
y = yaml.dump(self.__dict__, default_flow_style=False)
with open(path, 'w') as fd:
fd.write(y)
def refresh(cls, **kwargs):
# print("Inside refresh")
refreshed = cls.cm.refresh("vm", kwargs["cloud"])
# update counter
vms = cls.cm.find(kind='vm')
me = Default.user
for vm in vms:
name = vm['name']
if not name.startswith(me):
continue
number = name.split('-')[-1]
try:
# +1 as the stored counter is the next available counter
new_counter = int(number) + 1
except ValueError:
# name is not formatted correctly, possibly due to not
# being started using cloudmesh
continue
old_counter = Default.get_counter(Names.VM_COUNTER)
counter = max(new_counter, old_counter)
Default.set_counter(Names.VM_COUNTER, counter)
Console.debug_msg('Set counter ' + Names.VM_COUNTER + ' to ' +
str(Default.get_counter(Names.VM_COUNTER)))
return refreshed
def refresh(cls, **kwargs):
# print("Inside refresh")
refreshed = cls.cm.refresh("vm", kwargs["cloud"])
# update counter
vms = cls.cm.find(kind='vm')
me = Default.user
for vm in vms:
name = vm['name']
if not name.startswith(me):
continue
number = name.split('-')[-1]
try:
# +1 as the stored counter is the next available counter
new_counter = int(number) + 1
except ValueError:
# name is not formatted correctly, possibly due to not
# being started using cloudmesh
continue
old_counter = Default.get_counter(Names.VM_COUNTER)
counter = max(new_counter, old_counter)
Default.set_counter(Names.VM_COUNTER, counter)
Console.debug_msg('Set counter ' + Names.VM_COUNTER + ' to ' +
str(Default.get_counter(Names.VM_COUNTER)))
return refreshed
def load(cls, path):
filename = os.path.join(path, '.cloudmesh_metadata')
Console.debug_msg('Loading {} to {}'.format(cls.__name__, filename))
with open(filename) as fd:
d = yaml.load(fd)
stack = cls(dest=path, **d)
stack._env = d['_env']
return stack
def __init__(self, cmd, cwd=None, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=None):
Console.debug_msg('Running cmd: {}'.format(' '.join(map(quote, cmd))))
proc = subprocess.Popen(cmd, stderr=stderr, stdout=stdout, cwd=cwd, env=env)
stdout, stderr = proc.communicate()
self.returncode = proc.returncode
self.stderr = stderr
self.stdout = stdout
if self.returncode != 0:
raise SubprocessError(cmd, self.returncode, self.stderr, self.stdout)
def init(self, force=False, update=False):
"""Initialize by cloning (or updating if requested) a local copy of
the Big Data Stack repository.
:param bool force: setup previously setup project
:type force: :class:`bool`
:param bool update: update the local repository from the origin
:type update: :class:`bool`
"""
if not os.path.isdir(os.path.join(self.path, '.git')):
Console.debug_msg('Cloning branch {} of {} to {}'.format(
self.branch, self.repo, self.path))
Subprocess([
'git', 'clone', '--recursive', '--branch', self.branch,
self.repo, self.path
])
elif update:
Console.debug_msg('Updating to branch {} for {}'.format(
self.branch, self.path))
Subprocess([
'git', 'fetch', '--recurse-submodules', 'origin', self.branch
],
cwd=self.path)
Subprocess(['git', 'checkout', self.branch], cwd=self.path)
Subprocess(['git', 'merge', 'origin/{}'.format(self.branch)],
cwd=self.path)
venvname = 'venv'
venvdir = os.path.join(self.path, venvname)
if force and os.path.isfile(os.path.join(venvdir, 'bin', 'activate')):
Console.debug_msg('Removing {}'.format(venvdir))
shutil.rmtree(venvdir)
if not os.path.isdir(venvdir):
Console.debug_msg('Creating virtualenv {}'.format(venvdir))
Subprocess(['virtualenv', venvdir])
self._env = get_virtualenv_environment(venvdir)
cmd = ['pip', 'install', '-r', 'requirements.txt'
] + (['-U'] if force else [])
Console.debug_msg('Installing requirements to {}'.format(venvdir))
Subprocess(cmd, cwd=self.path, env=self._env)
def __init__(self,
inventory,
roles=None,
path=None,
username=None,
become=False,
become_user=None,
forks=None,
modifyKnownHosts=True,
extra_vars=None,
subprocess_kwargs=None):
self._inventory = inventory
self._roles = roles or []
self._path = path
self._username = username
self._become = become
self._become_user = become_user
self._forks = forks
self._modifyKnownHosts = modifyKnownHosts
self._extra_vars = extra_vars
self._subprocess_kwargs = subprocess_kwargs or dict()
################################################################ rest of setup
self._inventory_file = tempfile.NamedTemporaryFile()
self._playbook_file = tempfile.NamedTemporaryFile()
# write inventory
self._inventory_file.write(self._inventory.ini())
self._inventory_file.flush()
Console.debug_msg('Inventory:\n' + self._inventory.ini())
# write playbook
playbook = [{
'name': role.name,
'hosts': role.hosts,
'become': role.become,
'roles': [dict(role=role.path, **role.variables)]
} for role in roles]
yml = yaml.dump(playbook)
self._playbook_file.write(yml)
self._playbook_file.flush()
Console.debug_msg('Playbook:\n' + yml)
def __init__(self, inventory, roles=None, path=None, username=None, become=False,
become_user=None, forks=None, modifyKnownHosts=True,
extra_vars=None, subprocess_kwargs=None):
self._inventory = inventory
self._roles = roles or []
self._path = path
self._username = username
self._become = become
self._become_user = become_user
self._forks = forks
self._modifyKnownHosts = modifyKnownHosts
self._extra_vars = extra_vars
self._subprocess_kwargs = subprocess_kwargs or dict()
################################################################ rest of setup
self._inventory_file = tempfile.NamedTemporaryFile()
self._playbook_file = tempfile.NamedTemporaryFile()
# write inventory
self._inventory_file.write(self._inventory.ini())
self._inventory_file.flush()
Console.debug_msg('Inventory:\n' + self._inventory.ini())
# write playbook
playbook = [{
'name': role.name,
'hosts': role.hosts,
'become': role.become,
'roles': [dict(role=role.path,
**role.variables)]}
for role in roles]
yml = yaml.dump(playbook)
self._playbook_file.write(yml)
self._playbook_file.flush()
Console.debug_msg('Playbook:\n' + yml)
def __init__(self,
cmd,
cwd=None,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
env=None):
Console.debug_msg('Running cmd: {}'.format(' '.join(map(quote, cmd))))
proc = subprocess.Popen(cmd,
stderr=stderr,
stdout=stdout,
cwd=cwd,
env=env)
stdout, stderr = proc.communicate()
self.returncode = proc.returncode
self.stderr = stderr
self.stdout = stdout
if self.returncode != 0:
raise SubprocessError(cmd, self.returncode, self.stderr,
self.stdout)
def use_bds(self):
"""Use the BigDataStack backend"""
Console.debug_msg('Factory use_bds')
self.is_bds = True
return self
def set_defaults(self, **kwargs): # TODO: what is this method used for?
Console.debug_msg('Call to CloudmeshVMMixin')
pass
def playbook(self, cluster=None, path=None):
Console.debug_msg('NotImplementedError')
def __init__(self, context):
Console.debug_msg("init command deploy")
def deploy(self,
ips=None,
name=None,
user=None,
playbooks=None,
defines=None,
ping_max=10,
ping_sleep=10):
"""Deploy the big-data-stack to a previously stood up cluster located
at `ips` with login user `user`.
:param ips: the ip addresses of the cluster to deploy to
:type ips: :class:`list` of :class:`str` IP addresses
:param name: the name of the cluster
:type name: :class:`str`
:param user: the login username of the cluster
:type user: :class:`str`
:param playbooks: the list of playbooks to deploy. These are paths relative to the root directory of the BDS repository.
:type playbooks: :class:`list` of :class:`str`
:param defines: the overridden variables defined for each playbook
:type defines: :class:`dict` from playbook name to :class:`dict` of variable name to value
:param ping_max: the maximum number of time to attempt to ping the cluster during the verification step.
:type ping_max: :class:`int`
:param ping_sleep: the number of seconds to wait between each attempt to ping
:type ping_sleep: :class:`int`
"""
assert ips is not None
assert ping_max > 0, ping_max
assert ping_sleep > 0, ping_sleep
name = name or os.getenv('USER') + '-' + os.path.basename(self.path)
user = user or 'defaultuser'
playbooks = playbooks or list()
defines = defines or defaultdict(list)
Console.debug_msg('Calling mk-inventory in {}'.format(self.path))
cmd = ['python', 'mk-inventory', '-n', name] + ips
inventory = Subprocess(cmd, cwd=self.path, env=self._env)
Console.debug_msg('Writing inventory file')
Console.debug_msg('\n ' +
('\n' + 4 * ' ').join(inventory.stdout.split('\n')))
with open(os.path.join(self.path, 'inventory.txt'), 'w') as fd:
fd.write(inventory.stdout)
Console.info('Waiting for cluster to be accessible')
ping_ok = False
for i in xrange(ping_max):
Console.debug_msg('Attempt {} / {}'.format(i + 1, ping_max))
try:
Subprocess(['ansible', 'all', '-m', 'ping', '-u', user],
cwd=self.path,
env=self._env,
stdout=None,
stderr=None)
ping_ok = True
Console.debug_msg('Success!')
break
except SubprocessError:
Console.debug_msg(
'Failure, sleeping for {} seconds'.format(ping_sleep))
time.sleep(ping_sleep)
if not ping_ok:
msg = 'Ping Failure'
reason = 'Unable to connect to all nodes'
Console.error(' '.join([msg, reason]))
raise SanityCheckError(message=msg, reason=reason)
basic_command = ['ansible-playbook', '-u', user]
Console.debug_msg('Running playbooks {}'.format(playbooks))
for play in playbooks:
cmd = basic_command + [play]
define = ['{}={}'.format(k, v) for k, v in defines[play]]
if define:
cmd.extend(['-e', ','.join(define)])
Console.info('Running playbook {} with overrides {}'.format(
play, define))
Subprocess(cmd,
cwd=self.path,
env=self._env,
stdout=None,
stderr=None)
def set_defaults(self, **kwargs): # TODO: what is this method used for?
Console.debug_msg('Call to CloudmeshVMMixin')
pass
def activate(self, make_active=True):
"""Set whether or not the created project should be activated"""
Console.debug_msg('Factory activate: {}'.format(make_active))
self.make_active = make_active
return self
def set_repo(self, repo):
"""Set the repository to get the stack from"""
Console.debug_msg('Factory set_repo: {}'.format(repo))
self.repo = repo
return self
def set_force(self, force=False):
"""Set the `force` parameter"""
Console.debug_msg('Factory set_force: {}'.format(force))
self.force = force
return self
def set_update(self, update=False):
"""Set the `update` parameter"""
Console.debug_msg('Factory set_update: {}'.format(update))
self.update = update
return self
def set_project_name(self, name):
"""Set the project name"""
Console.debug_msg('Factory set_project_name: {}'.format(name))
self.project_name = name
return self
def set_overrides(self, overrides=None):
"""Set the overrides for deployment"""
Console.debug_msg('Factory set_overrides: {}'.format(overrides))
overrides = overrides or defaultdict(lambda: defaultdict(list))
self.overrides = overrides
return self
def set_branch(self, branch='master'):
"""Set the branch of the repository to use"""
Console.debug_msg('Factory set_branch: {}'.format(branch))
self.branch = branch
return self
def deploy(self,
ips=None,
name=None,
user=None,
playbooks=None,
defines=None):
"""Deploy the big-data-stack to a previously stood up cluster located
at `ips` with login user `user`.
:param ips: the ip addresses of the cluster to deploy to
:type ips: :class:`list` of :class:`str` IP addresses
:param name: the name of the cluster
:type name: :class:`str`
:param user: the login username of the cluster
:type user: :class:`str`
:param playbooks: the list of playbooks to deploy. These are paths relative to the root directory of the BDS repository.
:type playbooks: :class:`list` of :class:`str`
:param defines: the overridden variables defined for each playbook
:type defines: :class:`dict` from playbook name to :class:`dict` of variable name to value
:param ping_max: the maximum number of time to attempt to ping the cluster during the verification step.
:type ping_max: :class:`int`
:param ping_sleep: the number of seconds to wait between each attempt to ping
:type ping_sleep: :class:`int`
"""
assert ips is not None
name = name or os.getenv('USER') + '-' + os.path.basename(self.path)
user = user or 'defaultuser'
playbooks = playbooks or list()
defines = defines or defaultdict(list)
Console.debug_msg('Calling mk-inventory in {}'.format(self.path))
cmd = ['python', 'mk-inventory', '-n', name] + ips
inventory = Subprocess(cmd, cwd=self.path, env=self._env)
Console.debug_msg('Writing inventory file')
Console.debug_msg('\n ' +
('\n' + 4 * ' ').join(inventory.stdout.split('\n')))
with open(os.path.join(self.path, 'inventory.txt'), 'w') as fd:
fd.write(inventory.stdout)
Console.info('Waiting for cluster to be accessible')
def ping():
try:
Subprocess(['ansible', 'all', '-m', 'ping', '-u', user],
cwd=self.path,
env=self._env,
stdout=None,
stderr=None)
return True
except SubprocessError:
return False
exponential_backoff(ping)
basic_command = ['ansible-playbook', '-u', user]
Console.debug_msg('Running playbooks {}'.format(playbooks))
for play in playbooks:
donefile = os.path.join(self.path, play) + '.done'
if os.path.exists(donefile):
Console.ok('Skipping completed play %s' % play)
continue
cmd = basic_command + [play]
define = ['{}={}'.format(k, v) for k, v in defines[play]]
if define:
cmd.extend(['-e', ','.join(define)])
Console.info('Running playbook {} with overrides {}'.format(
play, define))
Subprocess(cmd,
cwd=self.path,
env=self._env,
stdout=None,
stderr=None)
with open(donefile, 'w') as fd:
fd.write('')
def set_user_name(self, username):
"""Set the cluster login user name"""
Console.debug_msg('Factory set_user_name: {}'.format(username))
self.username = username
return self
def set_playbooks(self, playbooks=None):
"""Set the playbooks to deploy"""
Console.debug_msg('Factory set_playbooks: {}'.format(playbooks))
playbooks = playbooks or list()
self.playbooks = playbooks
return self
def set_ips(self, ips):
"""Set the cluster IP addresses"""
Console.debug_msg('Factory set_ips: {}'.format(ips))
assert len(ips) >= 0
self.ips = ips
return self
