def test_cluster_from_to_list():
input_dict = [{
'uid': 0,
'ip': '0.0.0.0',
'port': 11
}, {
'uid': 1,
'ip': '1.1.1.1',
'port': 22
}, {
'uid': 2,
'ip': '2.2.2.2',
'port': 33
}]
cluster = Cluster.from_list(input_dict, settings)
exported = cluster.to_list()
assert isinstance(exported, list)
assert exported == input_dict
assert len(cluster.instances) == 3
for i, instance in enumerate(cluster.instances):
assert isinstance(instance, BareInstance)
assert cluster.instances[i].uid == i
assert cluster.instances[i].ip == '{0}.{0}.{0}.{0}'.format(i)
assert cluster.instances[i].port == 11 * (i + 1)
def read_instances(self):
"""
Read `.dsb/instances.yaml`
Populates `self.cluster`
"""
logger.debug('Reading instances from: %s' % self.instances_path)
if os.path.exists(self.instances_path):
with open(self.instances_path, 'r') as f:
list_ = yaml.load(f.read())
self.cluster = Cluster.from_list(list_, settings=self.settings)
def test_cluster_from_to_list():
data = [{"id": 0, "ip": "0.0.0.0"}, {"id": 1, "ip": "1.1.1.1"}, {"id": 2, "ip": "2.2.2.2"}]
cluster = Cluster.from_list(data, settings)
exported = cluster.to_list()
exported_ans = [{"id": 0, "ip": "0.0.0.0"}, {"id": 1, "ip": "1.1.1.1"}, {"id": 2, "ip": "2.2.2.2"}]
assert isinstance(exported, list)
assert exported == exported_ans
assert len(cluster.instances) == 3
def read_instances(self):
"""
Read `.dsb/instances.yaml`
Populates `self.cluster`
"""
logger.debug('Reading instances from: %s', self.instances_path)
if os.path.exists(self.instances_path):
with open(self.instances_path, 'r') as f:
list_ = yaml.load(f.read())
self.cluster = Cluster.from_list(list_, settings=self.settings)
def test_cluster_from_to_list():
input_dict = [{'uid': 0,
'ip': '0.0.0.0',
'port': 11}, {'uid': 1,
'ip': '1.1.1.1',
'port': 22}, {'uid': 2,
'ip': '2.2.2.2',
'port': 33}]
cluster = Cluster.from_list(input_dict, settings)
exported = cluster.to_list()
assert isinstance(exported, list)
assert exported == input_dict
assert len(cluster.instances) == 3
for i, instance in enumerate(cluster.instances):
assert isinstance(instance, BareInstance)
assert cluster.instances[i].uid == i
assert cluster.instances[i].ip == '{0}.{0}.{0}.{0}'.format(i)
assert cluster.instances[i].port == 11 * (i + 1)
def create_cluster(self):
self.cluster = Cluster(settings=self.settings)
self.cluster.create()
class Project(object):
@classmethod
def from_dir(cls, path=os.getcwd()):
dir_ = path
while dir_ != '/':
if os.path.exists(os.path.join(dir_, 'dsbfile')):
logger.debug('"dsbfile" FOUND on "%s"' % dir_)
break
logger.debug('"dsbfile" not found on "%s" trying parent directory' % dir_)
dir_ = os.path.abspath(os.path.join(dir_, os.pardir))
if not os.path.exists(os.path.join(dir_, 'dsbfile')):
raise DSBException('"dsbfile" not found on ""%s" or its parents' % path)
self = cls(path=dir_)
logger.debug('Starting project from: %s' % path)
self.read_settings()
self.read_instances()
return self
def __init__(self, path=None):
self.dir = path
self.cluster = None
self.settings = None
@property
def settings_path(self):
return os.path.join(self.dir, 'dsbfile')
@property
def settings_dir(self):
"""
Directory that contains the instances.yaml and other stuff
"""
path = os.path.join(self.dir, '.dsb')
safe_create_dir(path)
return os.path.realpath(path)
def read_settings(self):
"""
Read the settings "dsbfile" file
Populates `self.settings`
"""
logger.debug('Reading settings from: %s' % self.settings_path)
if os.path.exists(self.settings_path):
self.settings = Settings.from_dsbfile(self.settings_path)
else:
pass # TODO: do something?
def read_instances(self):
"""
Read `.dsb/instances.yaml`
Populates `self.cluster`
"""
logger.debug('Reading instances from: %s' % self.instances_path)
if os.path.exists(self.instances_path):
with open(self.instances_path, 'r') as f:
list_ = yaml.load(f.read())
self.cluster = Cluster.from_list(list_, settings=self.settings)
def save_instances(self):
logger.debug('Saving instances file to: %s' % self.instances_path)
with open(self.instances_path, 'w') as f:
yaml.safe_dump(self.cluster.to_list(), f, default_flow_style=False)
@property
def instances_path(self):
return os.path.join(self.settings_dir, 'instances.yaml')
def create_cluster(self):
self.cluster = Cluster(settings=self.settings)
self.cluster.create()
def destroy(self):
self.cluster.fetch_nodes()
self.cluster.destroy()
def update(self, force=False):
self.create_roster()
self.salt_ssh_create_dirs()
self.salt_ssh_create_master_conf()
self.copy_salt_and_pillar()
# --------------------------------------------------------------------------
# Salt-SSH
# --------------------------------------------------------------------------
def salt_ssh_create_dirs(self):
logger.debug('Creating salt-ssh dirs into: %s' % self.settings_dir)
safe_create_dir(os.path.join(self.settings_dir, 'salt'))
safe_create_dir(os.path.join(self.settings_dir, 'pillar'))
safe_create_dir(os.path.join(self.settings_dir, 'etc', 'salt'))
safe_create_dir(os.path.join(self.settings_dir, 'var', 'cache', 'salt'))
safe_create_dir(os.path.join(self.settings_dir, 'var', 'log', 'salt'))
@property
def roster_path(self):
return os.path.join(self.settings_dir, 'roster.yaml')
def create_roster(self):
logger.debug('Creating roster file to: %s' % self.roster_path)
with open(self.roster_path, 'w') as f:
yaml.safe_dump(salt.generate_roster(self.cluster), f, default_flow_style=False)
@property
def salt_ssh_config_dir(self):
return os.path.join(self.settings_dir, 'etc', 'salt')
def salt_ssh_create_master_conf(self):
this_dir = os.path.dirname(os.path.realpath(__file__))
template_path = os.path.join(this_dir, 'templates', 'master.conf')
with open(template_path, 'r') as f:
master_conf_template = f.read()
values = {}
values['salt_root'] = self.salt_dir
values['pillar_root'] = self.pillar_dir
values['root_dir'] = self.settings_dir
values['cachedir'] = os.path.join(self.settings_dir, 'var', 'cache', 'salt')
master_conf = master_conf_template.format(**values)
etc_salt_dir = os.path.join(self.settings_dir, 'etc', 'salt')
salt_master_file = os.path.join(etc_salt_dir, 'master')
with open(salt_master_file, 'w') as f:
f.write(master_conf)
@property
def salt_dir(self):
return os.path.join(self.settings_dir, 'salt')
@property
def pillar_dir(self):
return os.path.join(self.settings_dir, 'pillar')
def copy_salt_and_pillar(self):
this_dir = os.path.dirname(os.path.realpath(__file__))
salt_roots_src = os.path.join(this_dir, '..', 'salt')
salt_roots_src = os.path.realpath(salt_roots_src)
pillar_roots_src = os.path.join(this_dir, '..', 'pillar')
pillar_roots_src = os.path.realpath(pillar_roots_src)
if os.path.exists(self.salt_dir):
shutil.rmtree(self.salt_dir)
shutil.copytree(salt_roots_src, self.salt_dir)
if os.path.exists(self.pillar_dir):
shutil.rmtree(self.pillar_dir)
shutil.copytree(pillar_roots_src, self.pillar_dir)
ip = self.cluster.master.ip
self.replace_all(os.path.join(self.pillar_dir, 'salt.sls'), 'salt-master', ip)
self.replace_all(os.path.join(self.pillar_dir, 'system.sls'), 'ubuntu', self.settings['USERNAME'])
@staticmethod
def replace_all(file, searchExp, replaceExp):
for line in fileinput.input(file, inplace=1):
if searchExp in line:
line = line.replace(searchExp, replaceExp)
sys.stdout.write(line)
def salt(self, module, args=None, kwargs=None, target='*', ssh=False):
if ssh:
salt.salt_ssh(self, target, module, args, kwargs)
else:
salt.salt_master(self, target, module, args, kwargs)
import pytest
from datasciencebox.core.project import Project
from datasciencebox.core.settings import Settings
from datasciencebox.core.cloud.cluster import Cluster
settings = Settings()
settings['USERNAME'] = '******'
settings['KEYPAIR'] = '~/.ssh/id_rsa'
_ = [{'id': 0, 'ip': '0.0.0.0'}, {'id': 1, 'ip': '1.1.1.1'}, {'id': 2, 'ip': '2.2.2.2'}]
cluster = Cluster.from_list(_, settings)
project = Project()
project.settings = settings
project.cluster = cluster
class Project(object):
"""
Main DataScienceBox entrypoint.
Manages all the IO to and from files.
"""
@classmethod
def from_dir(cls, path=os.getcwd(), settingsfile='dsbfile'):
dir_ = path
while dir_ != '/':
if os.path.exists(os.path.join(settingsfile, dir_)):
logger.debug('"%s" FOUND on "%s"', settingsfile, dir_)
break
logger.debug('"%s" not found on "%s" trying parent directory', settingsfile, dir_)
dir_ = os.path.abspath(os.path.join(dir_, os.pardir))
if not os.path.exists(os.path.join(dir_, 'dsbfile')):
raise DSBException('"{}" not found on ""{}" or its parents'.format(settingsfile, path))
return cls.from_file(filepath=os.path.join(dir_, settingsfile))
@classmethod
def from_file(cls, filepath):
dir_ = os.path.dirname(filepath)
settingsfile = os.path.basename(filepath)
self = cls(path=dir_, settingsfile=settingsfile)
logger.debug('Starting project from: %s', filepath)
self.read_settings()
self.read_instances()
return self
def __init__(self, path=None, settingsfile='dsbfile'):
self.dir = path
self.settingsfile = settingsfile
self.cluster = None
self.settings = None
@property
def settings_path(self):
return os.path.join(self.dir, self.settingsfile)
@property
def settings_dir(self):
"""
Directory that contains the the settings for the project
"""
path = os.path.join(self.dir, '.dsb')
utils.create_dir(path)
return os.path.realpath(path)
def read_settings(self):
"""
Read the "dsbfile" file
Populates `self.settings`
"""
logger.debug('Reading settings from: %s', self.settings_path)
self.settings = Settings.from_dsbfile(self.settings_path)
def read_instances(self):
"""
Read `.dsb/instances.yaml`
Populates `self.cluster`
"""
logger.debug('Reading instances from: %s', self.instances_path)
if os.path.exists(self.instances_path):
with open(self.instances_path, 'r') as f:
list_ = yaml.load(f.read())
self.cluster = Cluster.from_list(list_, settings=self.settings)
def save_instances(self):
logger.debug('Saving instances file to: %s', self.instances_path)
with open(self.instances_path, 'w') as f:
yaml.safe_dump(self.cluster.to_list(), f, default_flow_style=False)
@property
def instances_path(self):
return os.path.join(self.settings_dir, 'instances.yaml')
def create_cluster(self):
self.cluster = Cluster(settings=self.settings)
self.cluster.create()
def destroy(self):
self.cluster.fetch_nodes()
self.cluster.destroy()
self.delete()
def delete(self):
shutil.rmtree(self.settings_dir)
def update(self):
self.setup_salt_ssh()
# --------------------------------------------------------------------------
# Salt
# --------------------------------------------------------------------------
def salt(self, module, target='*', args=None, kwargs=None, ssh=False):
"""
Execute a salt (or salt-ssh) command
"""
if ssh:
return salt.salt_ssh(self, target, module, args, kwargs)
else:
return salt.salt_master(self, target, module, args, kwargs)
@property
def roster_path(self):
return os.path.join(self.settings_dir, 'roster.yaml')
@property
def salt_dir(self):
return os.path.join(self.settings_dir, 'salt')
@property
def pillar_dir(self):
return os.path.join(self.settings_dir, 'pillar')
@property
def salt_ssh_config_dir(self):
return os.path.join(self.settings_dir, 'etc', 'salt')
def setup_salt_ssh(self):
"""
Setup `salt-ssh`
"""
self.copy_salt_and_pillar()
self.create_roster_file()
self.salt_ssh_create_dirs()
self.salt_ssh_create_master_file()
def create_roster_file(self):
logger.debug('Creating roster file to: %s', self.roster_path)
with open(self.roster_path, 'w') as f:
dict_ = salt.generate_roster(self)
yaml.safe_dump(dict_, f, default_flow_style=False)
def salt_ssh_create_dirs(self):
"""
Creates the `salt-ssh` required directory structure
"""
logger.debug('Creating salt-ssh dirs into: %s', self.settings_dir)
utils.create_dir(os.path.join(self.settings_dir, 'salt'))
utils.create_dir(os.path.join(self.settings_dir, 'pillar'))
utils.create_dir(os.path.join(self.settings_dir, 'etc', 'salt'))
utils.create_dir(os.path.join(self.settings_dir, 'var', 'cache', 'salt'))
utils.create_dir(os.path.join(self.settings_dir, 'var', 'log', 'salt'))
def salt_ssh_create_master_file(self):
etc_salt_dir = os.path.join(self.settings_dir, 'etc', 'salt')
master_conf_file = os.path.join(etc_salt_dir, 'master')
with open(master_conf_file, 'w') as f:
dict_ = salt.generate_salt_ssh_master_conf(project=self)
yaml.safe_dump(dict_, f, default_flow_style=False)
def copy_salt_and_pillar(self):
this_dir = os.path.dirname(os.path.realpath(__file__))
salt_roots_src = os.path.join(this_dir, '..', 'salt')
salt_roots_src = os.path.realpath(salt_roots_src)
pillar_roots_src = os.path.join(this_dir, '..', 'pillar')
pillar_roots_src = os.path.realpath(pillar_roots_src)
if os.path.exists(self.salt_dir):
shutil.rmtree(self.salt_dir)
shutil.copytree(salt_roots_src, self.salt_dir)
if os.path.exists(self.pillar_dir):
shutil.rmtree(self.pillar_dir)
shutil.copytree(pillar_roots_src, self.pillar_dir)
ip = self.cluster.head.ip
utils.replace_all(os.path.join(self.pillar_dir, 'salt.sls'), "'head'", ip)
utils.replace_all(os.path.join(self.pillar_dir, 'system.sls'), "'vagrant'",
self.settings['USERNAME'])
def create_cluster(self):
self.cluster = Cluster(settings=self.settings)
self.cluster.create()
class Project(object):
"""
Main DataScienceBox entrypoint.
Manages all the IO to and from files.
"""
@classmethod
def from_dir(cls, path=os.getcwd(), settingsfile='dsbfile'):
dir_ = path
while dir_ != '/':
if os.path.exists(os.path.join(settingsfile, dir_)):
logger.debug('"%s" FOUND on "%s"', settingsfile, dir_)
break
logger.debug('"%s" not found on "%s" trying parent directory',
settingsfile, dir_)
dir_ = os.path.abspath(os.path.join(dir_, os.pardir))
if not os.path.exists(os.path.join(dir_, 'dsbfile')):
raise DSBException('"{}" not found on ""{}" or its parents'.format(
settingsfile, path))
return cls.from_file(filepath=os.path.join(dir_, settingsfile))
@classmethod
def from_file(cls, filepath):
dir_ = os.path.dirname(filepath)
settingsfile = os.path.basename(filepath)
self = cls(path=dir_, settingsfile=settingsfile)
logger.debug('Starting project from: %s', filepath)
self.read_settings()
self.read_instances()
return self
def __init__(self, path=None, settingsfile='dsbfile'):
self.dir = path
self.settingsfile = settingsfile
self.cluster = None
self.settings = None
@property
def settings_path(self):
return os.path.join(self.dir, self.settingsfile)
@property
def settings_dir(self):
"""
Directory that contains the the settings for the project
"""
path = os.path.join(self.dir, '.dsb')
utils.create_dir(path)
return os.path.realpath(path)
def read_settings(self):
"""
Read the "dsbfile" file
Populates `self.settings`
"""
logger.debug('Reading settings from: %s', self.settings_path)
self.settings = Settings.from_dsbfile(self.settings_path)
def read_instances(self):
"""
Read `.dsb/instances.yaml`
Populates `self.cluster`
"""
logger.debug('Reading instances from: %s', self.instances_path)
if os.path.exists(self.instances_path):
with open(self.instances_path, 'r') as f:
list_ = yaml.load(f.read())
self.cluster = Cluster.from_list(list_, settings=self.settings)
def save_instances(self):
logger.debug('Saving instances file to: %s', self.instances_path)
with open(self.instances_path, 'w') as f:
yaml.safe_dump(self.cluster.to_list(), f, default_flow_style=False)
@property
def instances_path(self):
return os.path.join(self.settings_dir, 'instances.yaml')
def create_cluster(self):
self.cluster = Cluster(settings=self.settings)
self.cluster.create()
def destroy(self):
self.cluster.fetch_nodes()
self.cluster.destroy()
self.delete()
def delete(self):
shutil.rmtree(self.settings_dir)
def update(self):
self.setup_salt_ssh()
# --------------------------------------------------------------------------
# Salt
# --------------------------------------------------------------------------
def salt(self, module, target='*', args=None, kwargs=None, ssh=False):
"""
Execute a salt (or salt-ssh) command
"""
if ssh:
return salt.salt_ssh(self, target, module, args, kwargs)
else:
return salt.salt_master(self, target, module, args, kwargs)
@property
def roster_path(self):
return os.path.join(self.settings_dir, 'roster.yaml')
@property
def salt_dir(self):
return os.path.join(self.settings_dir, 'salt')
@property
def pillar_dir(self):
return os.path.join(self.settings_dir, 'pillar')
@property
def salt_ssh_config_dir(self):
return os.path.join(self.settings_dir, 'etc', 'salt')
def setup_salt_ssh(self):
"""
Setup `salt-ssh`
"""
self.copy_salt_and_pillar()
self.create_roster_file()
self.salt_ssh_create_dirs()
self.salt_ssh_create_master_file()
def create_roster_file(self):
logger.debug('Creating roster file to: %s', self.roster_path)
with open(self.roster_path, 'w') as f:
dict_ = salt.generate_roster(self)
yaml.safe_dump(dict_, f, default_flow_style=False)
def salt_ssh_create_dirs(self):
"""
Creates the `salt-ssh` required directory structure
"""
logger.debug('Creating salt-ssh dirs into: %s', self.settings_dir)
utils.create_dir(os.path.join(self.settings_dir, 'salt'))
utils.create_dir(os.path.join(self.settings_dir, 'pillar'))
utils.create_dir(os.path.join(self.settings_dir, 'etc', 'salt'))
utils.create_dir(
os.path.join(self.settings_dir, 'var', 'cache', 'salt'))
utils.create_dir(os.path.join(self.settings_dir, 'var', 'log', 'salt'))
def salt_ssh_create_master_file(self):
etc_salt_dir = os.path.join(self.settings_dir, 'etc', 'salt')
master_conf_file = os.path.join(etc_salt_dir, 'master')
with open(master_conf_file, 'w') as f:
dict_ = salt.generate_salt_ssh_master_conf(project=self)
yaml.safe_dump(dict_, f, default_flow_style=False)
def copy_salt_and_pillar(self):
this_dir = os.path.dirname(os.path.realpath(__file__))
salt_roots_src = os.path.join(this_dir, '..', 'salt')
salt_roots_src = os.path.realpath(salt_roots_src)
pillar_roots_src = os.path.join(this_dir, '..', 'pillar')
pillar_roots_src = os.path.realpath(pillar_roots_src)
if os.path.exists(self.salt_dir):
shutil.rmtree(self.salt_dir)
shutil.copytree(salt_roots_src, self.salt_dir)
if os.path.exists(self.pillar_dir):
shutil.rmtree(self.pillar_dir)
shutil.copytree(pillar_roots_src, self.pillar_dir)
ip = self.cluster.head.ip
utils.replace_all(os.path.join(self.pillar_dir, 'salt.sls'), "'head'",
ip)
utils.replace_all(os.path.join(self.pillar_dir, 'system.sls'),
"'vagrant'", self.settings['USERNAME'])
from __future__ import unicode_literals
import pytest
import os
from datasciencebox.core import salt
from datasciencebox.core.project import Project
from datasciencebox.core.cloud.cluster import Cluster
from datasciencebox.core.cloud.instance import Instance
cluster = Cluster()
cluster.instances.append(
Instance(ip='0.0.0.0', username='******', keypair='/home/ubuntu/.ssh/id_rsa'))
cluster.instances.append(
Instance(ip='1.1.1.1:2222',
username='******',
keypair='/home/ubuntu/.ssh/id_rsa2'))
cluster.instances.append(
Instance(ip='2.2.2.2',
port='3333',
username='******',
keypair='/home/ubuntu/.ssh/id_rsa3'))
project = Project()
project.cluster = cluster
head_roles = ['head', 'head2', 'conda']
compute_roles = ['minion2', 'conda']
salt.HEAD_ROLES = head_roles
salt.COMPUTE_ROLES = compute_roles
