def test_cluster_from_to_list(): input_dict = [{ 'uid': 0, 'ip': '0.0.0.0', 'port': 11 }, { 'uid': 1, 'ip': '1.1.1.1', 'port': 22 }, { 'uid': 2, 'ip': '2.2.2.2', 'port': 33 }] cluster = Cluster.from_list(input_dict, settings) exported = cluster.to_list() assert isinstance(exported, list) assert exported == input_dict assert len(cluster.instances) == 3 for i, instance in enumerate(cluster.instances): assert isinstance(instance, BareInstance) assert cluster.instances[i].uid == i assert cluster.instances[i].ip == '{0}.{0}.{0}.{0}'.format(i) assert cluster.instances[i].port == 11 * (i + 1)
def read_instances(self): """ Read `.dsb/instances.yaml` Populates `self.cluster` """ logger.debug('Reading instances from: %s' % self.instances_path) if os.path.exists(self.instances_path): with open(self.instances_path, 'r') as f: list_ = yaml.load(f.read()) self.cluster = Cluster.from_list(list_, settings=self.settings)
def test_cluster_from_to_list(): data = [{"id": 0, "ip": "0.0.0.0"}, {"id": 1, "ip": "1.1.1.1"}, {"id": 2, "ip": "2.2.2.2"}] cluster = Cluster.from_list(data, settings) exported = cluster.to_list() exported_ans = [{"id": 0, "ip": "0.0.0.0"}, {"id": 1, "ip": "1.1.1.1"}, {"id": 2, "ip": "2.2.2.2"}] assert isinstance(exported, list) assert exported == exported_ans assert len(cluster.instances) == 3
def read_instances(self): """ Read `.dsb/instances.yaml` Populates `self.cluster` """ logger.debug('Reading instances from: %s', self.instances_path) if os.path.exists(self.instances_path): with open(self.instances_path, 'r') as f: list_ = yaml.load(f.read()) self.cluster = Cluster.from_list(list_, settings=self.settings)
def test_cluster_from_to_list(): input_dict = [{'uid': 0, 'ip': '0.0.0.0', 'port': 11}, {'uid': 1, 'ip': '1.1.1.1', 'port': 22}, {'uid': 2, 'ip': '2.2.2.2', 'port': 33}] cluster = Cluster.from_list(input_dict, settings) exported = cluster.to_list() assert isinstance(exported, list) assert exported == input_dict assert len(cluster.instances) == 3 for i, instance in enumerate(cluster.instances): assert isinstance(instance, BareInstance) assert cluster.instances[i].uid == i assert cluster.instances[i].ip == '{0}.{0}.{0}.{0}'.format(i) assert cluster.instances[i].port == 11 * (i + 1)
def create_cluster(self): self.cluster = Cluster(settings=self.settings) self.cluster.create()
class Project(object): @classmethod def from_dir(cls, path=os.getcwd()): dir_ = path while dir_ != '/': if os.path.exists(os.path.join(dir_, 'dsbfile')): logger.debug('"dsbfile" FOUND on "%s"' % dir_) break logger.debug('"dsbfile" not found on "%s" trying parent directory' % dir_) dir_ = os.path.abspath(os.path.join(dir_, os.pardir)) if not os.path.exists(os.path.join(dir_, 'dsbfile')): raise DSBException('"dsbfile" not found on ""%s" or its parents' % path) self = cls(path=dir_) logger.debug('Starting project from: %s' % path) self.read_settings() self.read_instances() return self def __init__(self, path=None): self.dir = path self.cluster = None self.settings = None @property def settings_path(self): return os.path.join(self.dir, 'dsbfile') @property def settings_dir(self): """ Directory that contains the instances.yaml and other stuff """ path = os.path.join(self.dir, '.dsb') safe_create_dir(path) return os.path.realpath(path) def read_settings(self): """ Read the settings "dsbfile" file Populates `self.settings` """ logger.debug('Reading settings from: %s' % self.settings_path) if os.path.exists(self.settings_path): self.settings = Settings.from_dsbfile(self.settings_path) else: pass # TODO: do something? def read_instances(self): """ Read `.dsb/instances.yaml` Populates `self.cluster` """ logger.debug('Reading instances from: %s' % self.instances_path) if os.path.exists(self.instances_path): with open(self.instances_path, 'r') as f: list_ = yaml.load(f.read()) self.cluster = Cluster.from_list(list_, settings=self.settings) def save_instances(self): logger.debug('Saving instances file to: %s' % self.instances_path) with open(self.instances_path, 'w') as f: yaml.safe_dump(self.cluster.to_list(), f, default_flow_style=False) @property def instances_path(self): return os.path.join(self.settings_dir, 'instances.yaml') def create_cluster(self): self.cluster = Cluster(settings=self.settings) self.cluster.create() def destroy(self): self.cluster.fetch_nodes() self.cluster.destroy() def update(self, force=False): self.create_roster() self.salt_ssh_create_dirs() self.salt_ssh_create_master_conf() self.copy_salt_and_pillar() # -------------------------------------------------------------------------- # Salt-SSH # -------------------------------------------------------------------------- def salt_ssh_create_dirs(self): logger.debug('Creating salt-ssh dirs into: %s' % self.settings_dir) safe_create_dir(os.path.join(self.settings_dir, 'salt')) safe_create_dir(os.path.join(self.settings_dir, 'pillar')) safe_create_dir(os.path.join(self.settings_dir, 'etc', 'salt')) safe_create_dir(os.path.join(self.settings_dir, 'var', 'cache', 'salt')) safe_create_dir(os.path.join(self.settings_dir, 'var', 'log', 'salt')) @property def roster_path(self): return os.path.join(self.settings_dir, 'roster.yaml') def create_roster(self): logger.debug('Creating roster file to: %s' % self.roster_path) with open(self.roster_path, 'w') as f: yaml.safe_dump(salt.generate_roster(self.cluster), f, default_flow_style=False) @property def salt_ssh_config_dir(self): return os.path.join(self.settings_dir, 'etc', 'salt') def salt_ssh_create_master_conf(self): this_dir = os.path.dirname(os.path.realpath(__file__)) template_path = os.path.join(this_dir, 'templates', 'master.conf') with open(template_path, 'r') as f: master_conf_template = f.read() values = {} values['salt_root'] = self.salt_dir values['pillar_root'] = self.pillar_dir values['root_dir'] = self.settings_dir values['cachedir'] = os.path.join(self.settings_dir, 'var', 'cache', 'salt') master_conf = master_conf_template.format(**values) etc_salt_dir = os.path.join(self.settings_dir, 'etc', 'salt') salt_master_file = os.path.join(etc_salt_dir, 'master') with open(salt_master_file, 'w') as f: f.write(master_conf) @property def salt_dir(self): return os.path.join(self.settings_dir, 'salt') @property def pillar_dir(self): return os.path.join(self.settings_dir, 'pillar') def copy_salt_and_pillar(self): this_dir = os.path.dirname(os.path.realpath(__file__)) salt_roots_src = os.path.join(this_dir, '..', 'salt') salt_roots_src = os.path.realpath(salt_roots_src) pillar_roots_src = os.path.join(this_dir, '..', 'pillar') pillar_roots_src = os.path.realpath(pillar_roots_src) if os.path.exists(self.salt_dir): shutil.rmtree(self.salt_dir) shutil.copytree(salt_roots_src, self.salt_dir) if os.path.exists(self.pillar_dir): shutil.rmtree(self.pillar_dir) shutil.copytree(pillar_roots_src, self.pillar_dir) ip = self.cluster.master.ip self.replace_all(os.path.join(self.pillar_dir, 'salt.sls'), 'salt-master', ip) self.replace_all(os.path.join(self.pillar_dir, 'system.sls'), 'ubuntu', self.settings['USERNAME']) @staticmethod def replace_all(file, searchExp, replaceExp): for line in fileinput.input(file, inplace=1): if searchExp in line: line = line.replace(searchExp, replaceExp) sys.stdout.write(line) def salt(self, module, args=None, kwargs=None, target='*', ssh=False): if ssh: salt.salt_ssh(self, target, module, args, kwargs) else: salt.salt_master(self, target, module, args, kwargs)
import pytest from datasciencebox.core.project import Project from datasciencebox.core.settings import Settings from datasciencebox.core.cloud.cluster import Cluster settings = Settings() settings['USERNAME'] = '******' settings['KEYPAIR'] = '~/.ssh/id_rsa' _ = [{'id': 0, 'ip': '0.0.0.0'}, {'id': 1, 'ip': '1.1.1.1'}, {'id': 2, 'ip': '2.2.2.2'}] cluster = Cluster.from_list(_, settings) project = Project() project.settings = settings project.cluster = cluster
class Project(object): """ Main DataScienceBox entrypoint. Manages all the IO to and from files. """ @classmethod def from_dir(cls, path=os.getcwd(), settingsfile='dsbfile'): dir_ = path while dir_ != '/': if os.path.exists(os.path.join(settingsfile, dir_)): logger.debug('"%s" FOUND on "%s"', settingsfile, dir_) break logger.debug('"%s" not found on "%s" trying parent directory', settingsfile, dir_) dir_ = os.path.abspath(os.path.join(dir_, os.pardir)) if not os.path.exists(os.path.join(dir_, 'dsbfile')): raise DSBException('"{}" not found on ""{}" or its parents'.format(settingsfile, path)) return cls.from_file(filepath=os.path.join(dir_, settingsfile)) @classmethod def from_file(cls, filepath): dir_ = os.path.dirname(filepath) settingsfile = os.path.basename(filepath) self = cls(path=dir_, settingsfile=settingsfile) logger.debug('Starting project from: %s', filepath) self.read_settings() self.read_instances() return self def __init__(self, path=None, settingsfile='dsbfile'): self.dir = path self.settingsfile = settingsfile self.cluster = None self.settings = None @property def settings_path(self): return os.path.join(self.dir, self.settingsfile) @property def settings_dir(self): """ Directory that contains the the settings for the project """ path = os.path.join(self.dir, '.dsb') utils.create_dir(path) return os.path.realpath(path) def read_settings(self): """ Read the "dsbfile" file Populates `self.settings` """ logger.debug('Reading settings from: %s', self.settings_path) self.settings = Settings.from_dsbfile(self.settings_path) def read_instances(self): """ Read `.dsb/instances.yaml` Populates `self.cluster` """ logger.debug('Reading instances from: %s', self.instances_path) if os.path.exists(self.instances_path): with open(self.instances_path, 'r') as f: list_ = yaml.load(f.read()) self.cluster = Cluster.from_list(list_, settings=self.settings) def save_instances(self): logger.debug('Saving instances file to: %s', self.instances_path) with open(self.instances_path, 'w') as f: yaml.safe_dump(self.cluster.to_list(), f, default_flow_style=False) @property def instances_path(self): return os.path.join(self.settings_dir, 'instances.yaml') def create_cluster(self): self.cluster = Cluster(settings=self.settings) self.cluster.create() def destroy(self): self.cluster.fetch_nodes() self.cluster.destroy() self.delete() def delete(self): shutil.rmtree(self.settings_dir) def update(self): self.setup_salt_ssh() # -------------------------------------------------------------------------- # Salt # -------------------------------------------------------------------------- def salt(self, module, target='*', args=None, kwargs=None, ssh=False): """ Execute a salt (or salt-ssh) command """ if ssh: return salt.salt_ssh(self, target, module, args, kwargs) else: return salt.salt_master(self, target, module, args, kwargs) @property def roster_path(self): return os.path.join(self.settings_dir, 'roster.yaml') @property def salt_dir(self): return os.path.join(self.settings_dir, 'salt') @property def pillar_dir(self): return os.path.join(self.settings_dir, 'pillar') @property def salt_ssh_config_dir(self): return os.path.join(self.settings_dir, 'etc', 'salt') def setup_salt_ssh(self): """ Setup `salt-ssh` """ self.copy_salt_and_pillar() self.create_roster_file() self.salt_ssh_create_dirs() self.salt_ssh_create_master_file() def create_roster_file(self): logger.debug('Creating roster file to: %s', self.roster_path) with open(self.roster_path, 'w') as f: dict_ = salt.generate_roster(self) yaml.safe_dump(dict_, f, default_flow_style=False) def salt_ssh_create_dirs(self): """ Creates the `salt-ssh` required directory structure """ logger.debug('Creating salt-ssh dirs into: %s', self.settings_dir) utils.create_dir(os.path.join(self.settings_dir, 'salt')) utils.create_dir(os.path.join(self.settings_dir, 'pillar')) utils.create_dir(os.path.join(self.settings_dir, 'etc', 'salt')) utils.create_dir(os.path.join(self.settings_dir, 'var', 'cache', 'salt')) utils.create_dir(os.path.join(self.settings_dir, 'var', 'log', 'salt')) def salt_ssh_create_master_file(self): etc_salt_dir = os.path.join(self.settings_dir, 'etc', 'salt') master_conf_file = os.path.join(etc_salt_dir, 'master') with open(master_conf_file, 'w') as f: dict_ = salt.generate_salt_ssh_master_conf(project=self) yaml.safe_dump(dict_, f, default_flow_style=False) def copy_salt_and_pillar(self): this_dir = os.path.dirname(os.path.realpath(__file__)) salt_roots_src = os.path.join(this_dir, '..', 'salt') salt_roots_src = os.path.realpath(salt_roots_src) pillar_roots_src = os.path.join(this_dir, '..', 'pillar') pillar_roots_src = os.path.realpath(pillar_roots_src) if os.path.exists(self.salt_dir): shutil.rmtree(self.salt_dir) shutil.copytree(salt_roots_src, self.salt_dir) if os.path.exists(self.pillar_dir): shutil.rmtree(self.pillar_dir) shutil.copytree(pillar_roots_src, self.pillar_dir) ip = self.cluster.head.ip utils.replace_all(os.path.join(self.pillar_dir, 'salt.sls'), "'head'", ip) utils.replace_all(os.path.join(self.pillar_dir, 'system.sls'), "'vagrant'", self.settings['USERNAME'])
class Project(object): """ Main DataScienceBox entrypoint. Manages all the IO to and from files. """ @classmethod def from_dir(cls, path=os.getcwd(), settingsfile='dsbfile'): dir_ = path while dir_ != '/': if os.path.exists(os.path.join(settingsfile, dir_)): logger.debug('"%s" FOUND on "%s"', settingsfile, dir_) break logger.debug('"%s" not found on "%s" trying parent directory', settingsfile, dir_) dir_ = os.path.abspath(os.path.join(dir_, os.pardir)) if not os.path.exists(os.path.join(dir_, 'dsbfile')): raise DSBException('"{}" not found on ""{}" or its parents'.format( settingsfile, path)) return cls.from_file(filepath=os.path.join(dir_, settingsfile)) @classmethod def from_file(cls, filepath): dir_ = os.path.dirname(filepath) settingsfile = os.path.basename(filepath) self = cls(path=dir_, settingsfile=settingsfile) logger.debug('Starting project from: %s', filepath) self.read_settings() self.read_instances() return self def __init__(self, path=None, settingsfile='dsbfile'): self.dir = path self.settingsfile = settingsfile self.cluster = None self.settings = None @property def settings_path(self): return os.path.join(self.dir, self.settingsfile) @property def settings_dir(self): """ Directory that contains the the settings for the project """ path = os.path.join(self.dir, '.dsb') utils.create_dir(path) return os.path.realpath(path) def read_settings(self): """ Read the "dsbfile" file Populates `self.settings` """ logger.debug('Reading settings from: %s', self.settings_path) self.settings = Settings.from_dsbfile(self.settings_path) def read_instances(self): """ Read `.dsb/instances.yaml` Populates `self.cluster` """ logger.debug('Reading instances from: %s', self.instances_path) if os.path.exists(self.instances_path): with open(self.instances_path, 'r') as f: list_ = yaml.load(f.read()) self.cluster = Cluster.from_list(list_, settings=self.settings) def save_instances(self): logger.debug('Saving instances file to: %s', self.instances_path) with open(self.instances_path, 'w') as f: yaml.safe_dump(self.cluster.to_list(), f, default_flow_style=False) @property def instances_path(self): return os.path.join(self.settings_dir, 'instances.yaml') def create_cluster(self): self.cluster = Cluster(settings=self.settings) self.cluster.create() def destroy(self): self.cluster.fetch_nodes() self.cluster.destroy() self.delete() def delete(self): shutil.rmtree(self.settings_dir) def update(self): self.setup_salt_ssh() # -------------------------------------------------------------------------- # Salt # -------------------------------------------------------------------------- def salt(self, module, target='*', args=None, kwargs=None, ssh=False): """ Execute a salt (or salt-ssh) command """ if ssh: return salt.salt_ssh(self, target, module, args, kwargs) else: return salt.salt_master(self, target, module, args, kwargs) @property def roster_path(self): return os.path.join(self.settings_dir, 'roster.yaml') @property def salt_dir(self): return os.path.join(self.settings_dir, 'salt') @property def pillar_dir(self): return os.path.join(self.settings_dir, 'pillar') @property def salt_ssh_config_dir(self): return os.path.join(self.settings_dir, 'etc', 'salt') def setup_salt_ssh(self): """ Setup `salt-ssh` """ self.copy_salt_and_pillar() self.create_roster_file() self.salt_ssh_create_dirs() self.salt_ssh_create_master_file() def create_roster_file(self): logger.debug('Creating roster file to: %s', self.roster_path) with open(self.roster_path, 'w') as f: dict_ = salt.generate_roster(self) yaml.safe_dump(dict_, f, default_flow_style=False) def salt_ssh_create_dirs(self): """ Creates the `salt-ssh` required directory structure """ logger.debug('Creating salt-ssh dirs into: %s', self.settings_dir) utils.create_dir(os.path.join(self.settings_dir, 'salt')) utils.create_dir(os.path.join(self.settings_dir, 'pillar')) utils.create_dir(os.path.join(self.settings_dir, 'etc', 'salt')) utils.create_dir( os.path.join(self.settings_dir, 'var', 'cache', 'salt')) utils.create_dir(os.path.join(self.settings_dir, 'var', 'log', 'salt')) def salt_ssh_create_master_file(self): etc_salt_dir = os.path.join(self.settings_dir, 'etc', 'salt') master_conf_file = os.path.join(etc_salt_dir, 'master') with open(master_conf_file, 'w') as f: dict_ = salt.generate_salt_ssh_master_conf(project=self) yaml.safe_dump(dict_, f, default_flow_style=False) def copy_salt_and_pillar(self): this_dir = os.path.dirname(os.path.realpath(__file__)) salt_roots_src = os.path.join(this_dir, '..', 'salt') salt_roots_src = os.path.realpath(salt_roots_src) pillar_roots_src = os.path.join(this_dir, '..', 'pillar') pillar_roots_src = os.path.realpath(pillar_roots_src) if os.path.exists(self.salt_dir): shutil.rmtree(self.salt_dir) shutil.copytree(salt_roots_src, self.salt_dir) if os.path.exists(self.pillar_dir): shutil.rmtree(self.pillar_dir) shutil.copytree(pillar_roots_src, self.pillar_dir) ip = self.cluster.head.ip utils.replace_all(os.path.join(self.pillar_dir, 'salt.sls'), "'head'", ip) utils.replace_all(os.path.join(self.pillar_dir, 'system.sls'), "'vagrant'", self.settings['USERNAME'])
from __future__ import unicode_literals import pytest import os from datasciencebox.core import salt from datasciencebox.core.project import Project from datasciencebox.core.cloud.cluster import Cluster from datasciencebox.core.cloud.instance import Instance cluster = Cluster() cluster.instances.append( Instance(ip='0.0.0.0', username='******', keypair='/home/ubuntu/.ssh/id_rsa')) cluster.instances.append( Instance(ip='1.1.1.1:2222', username='******', keypair='/home/ubuntu/.ssh/id_rsa2')) cluster.instances.append( Instance(ip='2.2.2.2', port='3333', username='******', keypair='/home/ubuntu/.ssh/id_rsa3')) project = Project() project.cluster = cluster head_roles = ['head', 'head2', 'conda'] compute_roles = ['minion2', 'conda'] salt.HEAD_ROLES = head_roles salt.COMPUTE_ROLES = compute_roles