def run(cluster, pull): """Deploy services to Docker Swarm cluster""" # configuration if cluster == 'app': vm_name = "app1" yaml_name = 'app.yaml' vms = ["app1", "app2", "app3"] elif cluster == 'infra': vm_name = "infra1" yaml_name = 'infra.yaml' vms = ["infra1"] else: raise click.ClickException("Unsupported cluster name: " + cluster); # get paths currentDir = os.path.dirname(os.path.realpath(__file__)); rootDir = os.chdir(os.path.join(currentDir, 'vagrant')) v = vagrant.Vagrant(root=rootDir, quiet_stdout=False) yamlPath = os.path.join(currentDir, 'cluster', yaml_name) # pull images before deployment (startup cluster faster) if pull == True: with open(yamlPath, 'r') as yaml_content: try: # get image list document = yaml.full_load(yaml_content); task_count = len(document['services'].items()) * len(vms) task = 1; # pull for each node for vm in vms: # skip images that are already pulled print('Getting docker images for ' + vm + ' node...') images = v.ssh(vm_name=vm_name, command='docker images') images = images.split("\n") if len(images) > 0: del images[0] for i, val in enumerate(images): images[i] = re.sub(r'([^\s]*).*', "\\1", images[i]) if images[i] == '': del images[i] # pull images for key, value in document['services'].items(): if value['image'] in images: print(str(task) + '/' + str(task_count) + ' [' + vm + '] Skipping ' + value['image'] + '...') else: print(str(task) + '/' + str(task_count) + ' [' + vm + '] Pulling ' + value['image'] + '...') v.ssh(vm_name=vm, command='docker pull ' + value['image']) task = task + 1; except yaml.YAMLError as exc: print(exc) print('Transfering infrastructure definition...') v._run_vagrant_command(['scp', yamlPath, vm_name + ':/tmp/' + yaml_name]) print('Deploying to Docker Swarm...') v.ssh(vm_name=vm_name, command='docker stack deploy -c /tmp/' + yaml_name + ' oversort') # wait for Fluentd to start (required for app claster to log) if cluster == 'infra': print ("Waiting for Fluentd to start...") while True: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) result = sock.connect_ex(('192.168.10.100',24224)) if result == 0: print ("Fluentd ready") break else: time.sleep(5) print ("Waiting for Fluentd to start...") sock.close() print('DONE')
def init(self, config, calldir, force_deploy=False): """python -m enos.enos up Read the resources in the configuration files. Resource claims must be grouped by sizes according to the predefined SIZES map. """ conf = load_config(config, default_provider_config=DEFAULT_PROVIDER_CONFIG) provider_conf = conf['provider'] net_pools = { 'ip1': list(IPv4Network(u'192.168.142.0/25')), 'ip2': list(IPv4Network(u'192.168.143.0/25')), 'ip3': list(IPv4Network(u'192.168.144.0/25')), } # Build a list of machines that will be used to generate the # Vagrantfile machines = [] for size, roles in conf['resources'].items(): for role, nb in roles.items(): for i in range(nb): ip1 = str(net_pools['ip1'].pop()) _, _, _, name = ip1.split('.') machines.append({ 'role': role, # NOTE(matrohon): don't base the name of the VM on its # role, build_roles will then set the final role of # each VM 'name': "enos-%s" % name, 'size': size, 'cpu': SIZES[size]['cpu'], 'mem': SIZES[size]['mem'], 'ip1': ip1, 'ip2': str(net_pools['ip2'].pop()), 'ip3': str(net_pools['ip3'].pop()), }) loader = FileSystemLoader(searchpath=TEMPLATE_DIR) env = Environment(loader=loader) template = env.get_template('Vagrantfile.j2') vagrantfile = template.render(machines=machines, provider_conf=provider_conf) vagrantfile_path = os.path.join(calldir, "Vagrantfile") with open(vagrantfile_path, 'w') as f: f.write(vagrantfile) # Build env for Vagrant with a copy of env variables (needed by # subprocess opened by vagrant v_env = dict(os.environ) v_env['VAGRANT_DEFAULT_PROVIDER'] = provider_conf['backend'] v = vagrant.Vagrant(root=calldir, quiet_stdout=False, quiet_stderr=False, env=v_env) if force_deploy: v.destroy() v.up() v.provision() # Distribute the machines according to the resource/topology # specifications r = build_roles(conf, machines, lambda m: m['size']) roles = {} for role, machines in r.items(): roles.setdefault(role, []) for machine in machines: keyfile = v.keyfile(vm_name=machine['name']) port = v.port(vm_name=machine['name']) address = v.hostname(vm_name=machine['name']) roles[role].append( Host(address, alias=machine['name'], user=provider_conf['user'], port=port, keyfile=keyfile)) logging.info(roles) network = { 'cidr': '192.168.142.0/24', 'start': str(net_pools['ip1'][3]), 'end': str(net_pools['ip1'][-1]), 'dns': '8.8.8.8', 'gateway': '192.168.142.1' } network_interface = provider_conf['interfaces'][0] external_interface = provider_conf['interfaces'][1] return (roles, network, (network_interface, external_interface))
def __init__(self, vagrantfile): self.vagrantfile = vagrantfile self.vagrant = vagrant.Vagrant(self.vagrantfile, quiet_stdout=False)
def stop(self): print("[action] > stop\n") v1 = vagrant.Vagrant('vagrant/', quiet_stdout=False) v1.halt()
def __init__( # pylint: disable=super-init-not-called self, masters: int, agents: int, public_agents: int, files_to_copy_to_installer: List[Tuple[Path, Path]], cluster_backend: Vagrant, ) -> None: """ Create a DC/OS cluster with the given ``cluster_backend``. Args: masters: The number of master nodes to create. agents: The number of agent nodes to create. public_agents: The number of public agent nodes to create. files_to_copy_to_installer: Pairs of host paths to paths on the installer node. These are files to copy from the host to the installer node before installing DC/OS. cluster_backend: Details of the specific Docker backend to use. Raises: NotImplementedError: ``files_to_copy_to_installer`` includes files to copy to the installer. """ if files_to_copy_to_installer: message = ( 'Copying files to the installer is currently not supported by ' 'the Vagrant backend.' ) raise NotImplementedError(message) cluster_id = 'dcos-e2e-{random}'.format(random=uuid.uuid4()) self._master_prefix = cluster_id + '-master-' self._agent_prefix = cluster_id + '-agent-' self._public_agent_prefix = cluster_id + '-public-agent-' # We work in a new directory. # This helps running tests in parallel without conflicts and it # reduces the chance of side-effects affecting sequential tests. workspace_dir = cluster_backend.workspace_dir path = Path(workspace_dir) / uuid.uuid4().hex / cluster_id Path(path).mkdir(exist_ok=True, parents=True) path = Path(path).resolve() vagrantfile_path = Path(__file__).parent / 'resources' / 'Vagrantfile' shutil.copy(src=str(vagrantfile_path), dst=str(path)) vm_names = [] for nodes, prefix in ( (masters, self._master_prefix), (agents, self._agent_prefix), (public_agents, self._public_agent_prefix), ): for vm_number in range(nodes): name = prefix + str(vm_number) vm_names.append(name) vagrant_env = { 'PATH': os.environ['PATH'], 'VM_NAMES': ','.join(vm_names), 'VM_DESCRIPTION': cluster_backend.virtualbox_description, } # We import Vagrant here instead of at the top of the file because, if # the Vagrant executable is not found, a warning is logged. # # We want to avoid that warning for users of other backends who do not # have the Vagrant executable. import vagrant self._vagrant_client = vagrant.Vagrant( root=str(path), env=vagrant_env, quiet_stdout=False, quiet_stderr=True, ) self._vagrant_client.up()
file_new.write(line) file_new.flush() file_new.close() file_old.close() except Exception, e: self.logger.error("An error occured while copying '%s' to '%s'." % (pathVagrantFileOld, pathVagrantFileNew), exc_info=True) # else: # if not Util.copy_file(pathVagrantFileOld,pathVagrantFileNew,self.logger): # return False self.pathVagrantfile = pathVagrantFileNew Util.clear_vagrant_files(pathTmp, self.logger) # Create VM try: vm = vagrant.Vagrant(root=pathTmp, out_cm=log_cm, err_cm=log_cm) self.vm = vm vm.up(provider=self.provider) # with settings(host_string= vm.user_hostname_port(), # key_filename = vm.keyfile(), # disable_known_hosts = True): # run("uname -a") except Exception, e: self.logger.error('Failed while creating vm %s.' % (self.name), exc_info=True) self.logger.info( 'Since creation failed, trying to destroy vm %s.' % (self.name), exc_info=True) #if not self.vm.destroy(): # self.logger.error('Can not destroy %s.' %(self.name), exc_info=True)
from time import sleep import sys import vagrant testSystems = [ "bento/ubuntu-16.04", "bento/ubuntu-15.04", "bento/debian-8.5", "bento/opensuse-leap-42.1", "bento/fedora-24", "Sabayon/spinbase-amd64" ] progressBar = tqdm(testSystems) successfulSystems = [] failedSystems = [] for testSystem in progressBar: v = vagrant.Vagrant(quiet_stdout=False) try: progressBar.set_description("create Vagrantfile for %s" % testSystem) with open('Vagrantfile', 'w') as vagrantfile: vagrantfile.write(""" Vagrant.configure("2") do |config| config.vm.box = "%s" config.ssh.insert_key = false config.vm.provider "virtualbox" do |v| v.memory = 4096 v.cpus = 2 end # This is needed because Fedora only comes with Python 3 and Ansible needs Python 2
def list_boxes(): v = vagrant.Vagrant() list = [] for box in v.box_list(): list.append(box) return list
def stop_vm(vm_names: list): for vm_name in vm_names: os.chdir(vm_name) v = vagrant.Vagrant() v.halt() os.chdir("..")
def init(self, force_deploy=False): """Reserve and deploys the vagrant boxes. Args: force_deploy (bool): True iff new machines should be started """ machines = self.provider_conf.machines networks = self.provider_conf.networks _networks = [] for network in networks: ipnet = IPNetwork(network.cidr) _networks.append({ "netpool": list(ipnet)[10:-10], "cidr": network.cidr, "roles": network.roles, "gateway": ipnet.ip }) vagrant_machines = [] vagrant_roles = {} j = 0 for machine in machines: for _ in range(machine.number): vagrant_machine = { "name": "enos-%s" % j, "cpu": machine.flavour_desc["core"], "mem": machine.flavour_desc["mem"], "ips": [n["netpool"].pop() for n in _networks], } vagrant_machines.append(vagrant_machine) # Assign the machines to the right roles for role in machine.roles: vagrant_roles.setdefault(role, []).append(vagrant_machine) j = j + 1 logger.debug(vagrant_roles) loader = FileSystemLoader(searchpath=TEMPLATE_DIR) env = Environment(loader=loader, autoescape=True) template = env.get_template('Vagrantfile.j2') vagrantfile = template.render(machines=vagrant_machines, provider_conf=self.provider_conf) vagrantfile_path = os.path.join(os.getcwd(), "Vagrantfile") with open(vagrantfile_path, 'w') as f: f.write(vagrantfile) # Build env for Vagrant with a copy of env variables (needed by # subprocess opened by vagrant v_env = dict(os.environ) v_env['VAGRANT_DEFAULT_PROVIDER'] = self.provider_conf.backend v = vagrant.Vagrant(root=os.getcwd(), quiet_stdout=False, quiet_stderr=False, env=v_env) if force_deploy: v.destroy() v.up() v.provision() roles = {} for role, machines in vagrant_roles.items(): for machine in machines: keyfile = v.keyfile(vm_name=machine['name']) port = v.port(vm_name=machine['name']) address = v.hostname(vm_name=machine['name']) roles.setdefault(role, []).append( Host(address, alias=machine['name'], user=self.provider_conf.user, port=port, keyfile=keyfile)) networks = [{ 'cidr': str(n["cidr"]), 'start': str(n["netpool"][0]), 'end': str(n["netpool"][-1]), 'dns': '8.8.8.8', 'gateway': n["gateway"], 'roles': n["roles"] } for n in _networks] logger.debug(roles) logger.debug(networks) return (roles, networks)
def vagrant_destroy(ns): v = vagrant.Vagrant(os.getcwd() + '/drivers/Vagrant/repository/' + ns.operator.name + '/' + ns.name) v.destroy() shutil.rmtree(os.getcwd() + '/drivers/Vagrant/repository/' + ns.operator.name + "/" + ns.name)
def destroy(self): """Destroy all vagrant box involved in the deployment.""" v = vagrant.Vagrant(root=os.getcwd(), quiet_stdout=False, quiet_stderr=True) v.destroy()
def __exit__(self, etype, value, traceback): if self.script_dir: with cd(self.script_dir): v = vagrant.Vagrant(quiet_stdout=False) v.halt()
def __enter__(self): if self.script_dir: with cd(self.script_dir): v = vagrant.Vagrant(quiet_stdout=False) v.up() print("...virtual cluster is up.")
def stop_vagrant_boxes(): v = vagrant.Vagrant(vagrant_dir, quiet_stdout=False, quiet_stderr=False) v.halt() logging.info(' punchbox: vagrant boxes successfully stopped')
def startVagrant(machineName, envVars): # Starts a Vagrant instance with noisy logs # Loading the enviornement vars that was created in prepareEnvironmentVars v = vagrant.Vagrant(quiet_stdout=False, quiet_stderr=False) v.env = envVars v.up(vm_name=machineName)
def run_vm(root): """Start vagrant VM""" v = vagrant.Vagrant(root=root) print(" - Starting VM ", root) v.up()
def destroyVagrant(machineName, envVars): v = vagrant.Vagrant(quiet_stdout=False, quiet_stderr=False) v.env = envVars v.destroy(vm_name=machineName)
def _get_vagrant(self): env = os.environ.copy() env['VAGRANT_CWD'] = os.environ['MOLECULE_EPHEMERAL_DIRECTORY'] v = vagrant.Vagrant(quiet_stdout=False, quiet_stderr=False, env=env) return v
def updateBox(): v = vagrant.Vagrant(quiet_stdout=False, quiet_stderr=False) v.box_update() v.box_prune()
def get_vagrant(vagrant_path, vm_id, ran_id): vm_path = os.path.join(vagrant_path, "{0}-{1}".format(vm_id, ran_id)) if not os.path.exists(vm_path): os.makedirs(vm_path) return vagrant.Vagrant(vm_path)
def get_build_vm(srvdir, provider=None): """Factory function for getting FDroidBuildVm instances. This function tries to figure out what hypervisor should be used and creates an object for controlling a build VM. :param srvdir: path to a directory which contains a Vagrantfile :param provider: optionally this parameter allows specifiying an specific vagrant provider. :returns: FDroidBuildVm instance. """ abssrvdir = abspath(srvdir) # use supplied provider if provider: if provider == 'libvirt': logging.debug('build vm provider \'libvirt\' selected') return LibvirtBuildVm(abssrvdir) elif provider == 'virtualbox': logging.debug('build vm provider \'virtualbox\' selected') return VirtualboxBuildVm(abssrvdir) else: logging.warning('build vm provider not supported: \'%s\'', provider) # try guessing provider from installed software kvm_installed = shutil.which('kvm') is not None kvm_installed |= shutil.which('qemu') is not None kvm_installed |= shutil.which('qemu-kvm') is not None vbox_installed = shutil.which('VBoxHeadless') is not None if kvm_installed and vbox_installed: logging.debug('both kvm and vbox are installed.') elif kvm_installed: logging.debug( 'libvirt is the sole installed and supported vagrant provider, selecting \'libvirt\'' ) return LibvirtBuildVm(abssrvdir) elif vbox_installed: logging.debug( 'virtualbox is the sole installed and supported vagrant provider, selecting \'virtualbox\'' ) return VirtualboxBuildVm(abssrvdir) else: logging.debug( 'could not confirm that either virtualbox or kvm/libvirt are installed' ) # try guessing provider from .../srvdir/.vagrant internals vagrant_libvirt_path = os.path.join(abssrvdir, '.vagrant', 'machines', 'default', 'libvirt') has_libvirt_machine = isdir(vagrant_libvirt_path) \ and len(os.listdir(vagrant_libvirt_path)) > 0 vagrant_virtualbox_path = os.path.join(abssrvdir, '.vagrant', 'machines', 'default', 'virtualbox') has_vbox_machine = isdir(vagrant_virtualbox_path) \ and len(os.listdir(vagrant_virtualbox_path)) > 0 if has_libvirt_machine and has_vbox_machine: logging.info( 'build vm provider lookup found virtualbox and libvirt, defaulting to \'virtualbox\'' ) return VirtualboxBuildVm(abssrvdir) elif has_libvirt_machine: logging.debug('build vm provider lookup found \'libvirt\'') return LibvirtBuildVm(abssrvdir) elif has_vbox_machine: logging.debug('build vm provider lookup found \'virtualbox\'') return VirtualboxBuildVm(abssrvdir) # try guessing provider from available buildserver boxes available_boxes = [] import vagrant boxes = vagrant.Vagrant().box_list() for box in boxes: if box.name == "buildserver": available_boxes.append(box.provider) if "libvirt" in available_boxes and "virtualbox" in available_boxes: logging.info( 'basebox lookup found virtualbox and libvirt boxes, defaulting to \'virtualbox\'' ) return VirtualboxBuildVm(abssrvdir) elif "libvirt" in available_boxes: logging.info('\'libvirt\' buildserver box available, using that') return LibvirtBuildVm(abssrvdir) elif "virtualbox" in available_boxes: logging.info('\'virtualbox\' buildserver box available, using that') return VirtualboxBuildVm(abssrvdir) else: logging.error('No available \'buildserver\' box. Cannot proceed') os._exit(1)
def resume(self): print("[action] > resume\n") v1 = vagrant.Vagrant('vagrant/', quiet_stdout=False) v1.up()
def test_vm_sandbox_mode(): ''' Test methods for enabling/disabling the sandbox mode and committing/rolling back changes. This depends on the Sahara plugin. ''' # Only test Sahara if it is installed. # This leaves the testing of Sahara to people who care. sahara_installed = _plugin_installed(vagrant.Vagrant(TD), 'sahara') if not sahara_installed: return v = vagrant.SandboxVagrant(TD) sandbox_status = v.sandbox_status() assert sandbox_status == "unknown", "Before the VM goes up the status should be 'unknown', " + "got:'{}'".format(sandbox_status) v.up() sandbox_status = v.sandbox_status() assert sandbox_status == "off", "After the VM goes up the status should be 'off', " + "got:'{}'".format(sandbox_status) v.sandbox_on() sandbox_status = v.sandbox_status() assert sandbox_status == "on", "After enabling the sandbox mode the status should be 'on', " + "got:'{}'".format(sandbox_status) v.sandbox_off() sandbox_status = v.sandbox_status() assert sandbox_status == "off", "After disabling the sandbox mode the status should be 'off', " + "got:'{}'".format(sandbox_status) v.sandbox_on() v.halt() sandbox_status = v.sandbox_status() assert sandbox_status == "on", "After halting the VM the status should be 'on', " + "got:'{}'".format(sandbox_status) v.up() sandbox_status = v.sandbox_status() assert sandbox_status == "on", "After bringing the VM up again the status should be 'on', " + "got:'{}'".format(sandbox_status) test_file_contents = _read_test_file(v) print test_file_contents eq_(test_file_contents, None, "There should be no test file") _write_test_file(v, "foo") test_file_contents = _read_test_file(v) print test_file_contents eq_(test_file_contents, "foo", "The test file should read 'foo'") v.sandbox_rollback() time.sleep(10) # https://github.com/jedi4ever/sahara/issues/16 test_file_contents = _read_test_file(v) print test_file_contents eq_(test_file_contents, None, "There should be no test file") _write_test_file(v, "foo") test_file_contents = _read_test_file(v) print test_file_contents eq_(test_file_contents, "foo", "The test file should read 'foo'") v.sandbox_commit() _write_test_file(v, "bar") test_file_contents = _read_test_file(v) print test_file_contents eq_(test_file_contents, "bar", "The test file should read 'bar'") v.sandbox_rollback() time.sleep(10) # https://github.com/jedi4ever/sahara/issues/16 test_file_contents = _read_test_file(v) print test_file_contents eq_(test_file_contents, "foo", "The test file should read 'foo'") sandbox_status = v._parse_vagrant_sandbox_status("Usage: ...") eq_(sandbox_status, "not installed", "When 'vagrant sandbox status'" + " outputs vagrant help status should be 'not installed', " + "got:'{}'".format(sandbox_status)) v.destroy() sandbox_status = v.sandbox_status() assert sandbox_status == "unknown", "After destroying the VM the status should be 'unknown', " + "got:'{}'".format(sandbox_status)
def destroy(self, calldir, env): v = vagrant.Vagrant(root=calldir, quiet_stdout=False, quiet_stderr=True) v.destroy()
def _get_vagrant(self): env = os.environ.copy() env['VAGRANT_CWD'] = os.environ['MOLECULE_EPHEMERAL_DIRECTORY'] v = vagrant.Vagrant(err_cm=self.stderr_cm, env=env) return v
#!/usr/bin/env python import vagrant import subprocess import json v = vagrant.Vagrant() running_vms = [vm for vm in v.status() if vm.state == v.RUNNING] def convert_ssh_options(ssh_config): ssh_option_mapping = { 'HostName': 'ansible_ssh_host', 'User': '******', 'Port': 'ansible_ssh_port', 'IdentityFile': 'ansible_ssh_private_key_file', } ansible_options = {} for k, v in ssh_config.iteritems(): if k in ssh_option_mapping.keys(): ansible_options[ssh_option_mapping[k]] = v return ansible_options inventory = { "vagrant": { "hosts": [vm.name for vm in running_vms], }, "virtualbox": { "hosts": [vm.name for vm in running_vms if vm.provider == 'virtualbox'],
def launch_vagrant_boxes(): v = vagrant.Vagrant(vagrant_dir, quiet_stdout=False, quiet_stderr=False) v.up() logging.info(' punchbox: vagrant boxes successfully started')
def destroy(self): self.log.info("[action] > destroy\n") v1 = vagrant.Vagrant('vagrant/', quiet_stdout=False) v1.destroy() self.log.info( "attack_range has been destroy using vagrant successfully")
def __init__( # pylint: disable=super-init-not-called self, masters: int, agents: int, public_agents: int, cluster_backend: Vagrant, ) -> None: """ Create a DC/OS cluster with the given ``cluster_backend``. Args: masters: The number of master nodes to create. agents: The number of agent nodes to create. public_agents: The number of public agent nodes to create. cluster_backend: Details of the specific Docker backend to use. """ cluster_id = 'dcos-e2e-{random}'.format(random=uuid.uuid4()) self._master_prefix = cluster_id + '-master-' self._agent_prefix = cluster_id + '-agent-' self._public_agent_prefix = cluster_id + '-public-agent-' # We work in a new directory. # This helps running tests in parallel without conflicts and it # reduces the chance of side-effects affecting sequential tests. workspace_dir = cluster_backend.workspace_dir path = Path(workspace_dir) / uuid.uuid4().hex / cluster_id Path(path).mkdir(exist_ok=True, parents=True) path = Path(path).resolve() vagrantfile_path = Path(__file__).parent / 'resources' / 'Vagrantfile' shutil.copy(src=str(vagrantfile_path), dst=str(path)) vm_names = [] for nodes, prefix in ( (masters, self._master_prefix), (agents, self._agent_prefix), (public_agents, self._public_agent_prefix), ): for vm_number in range(nodes): name = prefix + str(vm_number) vm_names.append(name) vagrant_env = { 'HOME': os.environ['HOME'], 'PATH': os.environ['PATH'], 'VM_NAMES': ','.join(vm_names), 'VM_DESCRIPTION': cluster_backend.virtualbox_description, 'VM_MEMORY': str(cluster_backend.vm_memory_mb), 'VAGRANT_BOX_VERSION': str(cluster_backend.vagrant_box_version), 'VAGRANT_BOX_URL': cluster_backend.vagrant_box_url, } # We import Vagrant here instead of at the top of the file because, if # the Vagrant executable is not found, a warning is logged. # # We want to avoid that warning for users of other backends who do not # have the Vagrant executable. import vagrant self._vagrant_client = vagrant.Vagrant( root=str(path), env=vagrant_env, quiet_stdout=False, quiet_stderr=False, ) self._vagrant_client.up()