class SingularityVM(object):
def __init__(self, path_to_build):
self.path_to_build = path_to_build
def __enter__(self):
# We want to start with a clean environment every time.
if os.path.exists(SINGULARITY_VM_INPUT_DIR):
rmtree(SINGULARITY_VM_INPUT_DIR)
if os.path.exists(SINGULARITY_VM_OUTPUT_DIR):
rmtree(SINGULARITY_VM_OUTPUT_DIR)
makedirs(SINGULARITY_VM_OUTPUT_DIR)
copytree(self.path_to_build, SINGULARITY_VM_INPUT_DIR, symlinks=True)
self.vm = Vagrant(root=SINGULARITY_VM,
quiet_stdout=False,
quiet_stderr=False)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.vm.destroy()
rmtree(SINGULARITY_VM_INPUT_DIR)
rmtree(SINGULARITY_VM_OUTPUT_DIR)
def run(self):
try:
self.vm.up()
return SINGULARITY_VM_IMAGE
except CalledProcessError:
return None
def store_logs(self, log_path, err_path):
copy(SINGULARITY_VM_STDOUT, log_path)
copy(SINGULARITY_VM_STDERR, err_path)
class VagrantWrapper(object):
def __init__(self, root_dir, env):
self.instance = Vagrant(root=root_dir)
self.env = env
self.target = GuestOS(env[:-2], env[-2:])
def up(self):
return self.instance.up(vm_name=self.env)
def destroy(self):
return self.instance.destroy(vm_name=self.env)
def reboot(self, timeout=120):
try:
with self.ssh() as ssh:
ssh.execute('sudo reboot', no_stderr_expected=False)
except:
# SSH can be broken on reboot - ignore this error
pass
# waiting for reboot to take place
# TODO remove this spleep
sleep(2)
self.up()
wait_for_ssh(self, timeout)
def __get_ssh_params(self):
return self.instance.user_hostname_port(vm_name=self.env), self.instance.keyfile(vm_name=self.env)
def ssh(self) -> SSHClient:
return ssh_connect(*self.__get_ssh_params())
def known_hosts(vagrant_root):
vagrant_root = lookup_vagrant_root(vagrant_root)
vagrant = Vagrant(vagrant_root)
return {
machine: vagrant.hostname(machine)
for machine in list_machines(vagrant_root)
}
def poweron_box(box_id):
box = Device.query.filter(Device.id == box_id).first()
box_path = str("%s/boxes/%s" % (dirname(dirname(dirname(dirname(__file__)))), box.hostname))
vagrant_instance = Vagrant(root=box_path)
vagrant_instance.up()
box.box_online = True
db.session.commit()
return redirect(url_for("index"))
def known_hosts(vagrant_root, name=None):
vagrant_root = lookup_vagrant_root(vagrant_root)
vagrant = Vagrant(vagrant_root)
machines = list_machines(vagrant_root)
name = name if name in machines else machines[0]
return parse_content(vagrant._run_vagrant_command(('ssh', name, '-c', 'cat /etc/hosts')))
def up(self):
"""Function to start a vagrant box in current dir.
Starts the VM
Args:
challenge_id (str): the challengeId retrived during create command
"""
Vagrant.up()
def known_hosts(vagrant_root, name=None):
vagrant_root = lookup_vagrant_root(vagrant_root)
vagrant = Vagrant(vagrant_root)
machines = list_machines(vagrant_root)
name = name if name in machines else machines[0]
return parse_content(
vagrant._run_vagrant_command(('ssh', name, '-c', 'cat /etc/hosts')))
def main(self):
for minion_name, minion_path in self.parent.config.get('minions', {}).items():
vagrant = Vagrant(minion_path)
vagrant_status = vagrant.status()[0].state
salt_status = self.parent.client.get_minion_status(minion_name)
puts("%s:" % minion_name)
with indent(4):
puts("vagrant status: %s" % vagrant_status)
puts("saltstack status: %s" % salt_status)
def __init__(self, *args, **kwargs):
"""Check that the vagrant_boxes attribute is not left empty, and is populated by all boxes if left blank"""
self.vagrant = Vagrant(self.vagrant_root, err_cm=stderr_cm)
if not self.vagrant_boxes:
boxes = [s.name for s in self.vagrant.status()]
if len(boxes) == 1:
self.vagrant_boxes = ["default"]
else:
self.vagrant_boxes = boxes
super().__init__(*args, **kwargs)
def __init__(self, *args, **kwargs):
"""Check that the vagrant_boxes attribute is not left empty, and is populated by all boxes if left blank"""
self.vagrant = Vagrant(self.vagrant_root)
if not self.vagrant_boxes:
boxes = self.vagrant.status().keys()
if len(boxes) == 1:
self.vagrant_boxes = ['default']
else:
self.vagrant_boxes = boxes
super(VagrantTestCase, self).__init__(*args, **kwargs)
def main(self):
for minion_name, minion_path in self.parent.config.get('minions',
{}).items():
vagrant = Vagrant(minion_path)
vagrant_status = vagrant.status()[0].state
salt_status = self.parent.client.get_minion_status(minion_name)
puts("%s:" % minion_name)
with indent(4):
puts("vagrant status: %s" % vagrant_status)
puts("saltstack status: %s" % salt_status)
def destroy_box(box_id):
box = Device.query.filter(Device.id == box_id).first()
box_path = str("%s/boxes/%s" % (dirname(dirname(dirname(dirname(__file__)))), box.hostname))
vagrant_instance = Vagrant(root=box_path)
vagrant_instance.destroy()
os.chdir(box_path)
os.chdir("../")
shutil.rmtree(box.hostname)
db.session.delete(box)
db.session.commit()
return redirect(url_for("index"))
def __enter__(self):
# We want to start with a clean environment every time.
if os.path.exists(SINGULARITY_VM_INPUT_DIR):
rmtree(SINGULARITY_VM_INPUT_DIR)
if os.path.exists(SINGULARITY_VM_OUTPUT_DIR):
rmtree(SINGULARITY_VM_OUTPUT_DIR)
makedirs(SINGULARITY_VM_OUTPUT_DIR)
copytree(self.path_to_build, SINGULARITY_VM_INPUT_DIR, symlinks=True)
self.vm = Vagrant(root=SINGULARITY_VM,
quiet_stdout=False,
quiet_stderr=False)
return self
def __init__(self, host='default', vagrantDir=None, controlVagrant=True):
global _defaultVagrantDir
self.host = host
self._controlVagrant = controlVagrant
if vagrantDir is None:
vagrantDir = _defaultVagrantDir
self._vagrant = Vagrant(root=vagrantDir)
self._startedVagrant = False
self._sshClient = None
self._sshConfigStr = None
self._sshConfig = None
self._sshHostConfig = None
class Machine(object):
""" Class to wrap Machine functionality. """
def __init__(self, vagrantfile):
""" Configure logging. """
# Use the logger object created by Client.
self.p = logging.getLogger('qb')
self.p.debug("Using Vagrant config:\n"
" Vagrantfile: %s" % vagrantfile)
# Create a Vagrant object.
self.vagrant = Vagrant(vagrantfile)
return
def create(self, name):
""" Create a qb machine. """
try:
self.p.info("Creating %s..." % name)
self.vagrant.up(vm_name=name)
self.p.info('Done!')
except Exception as e:
self.p.debug(e)
self.p.error("Failed to create machine.")
sys.exit(1)
return
def start(self, name):
""" Start a qb machine. """
self.p.debug("Starting \"%s\"." % name)
return
def stop(self, name):
""" Stop a qb machine. """
self.p.debug("Stopping \"%s\"." % name)
return
def remove(self, name):
""" Remove a qb machine. """
self.p.debug("Removing \"%s\"." % name)
return
def __init__(self, vagrantfile):
""" Configure logging. """
# Use the logger object created by Client.
self.p = logging.getLogger('qb')
self.p.debug("Using Vagrant config:\n"
" Vagrantfile: %s" % vagrantfile)
# Create a Vagrant object.
self.vagrant = Vagrant(vagrantfile)
return
class Machine(object):
""" Class to wrap Machine functionality. """
def __init__(self, vagrantfile):
""" Configure logging. """
# Use the logger object created by Client.
self.p = logging.getLogger('qb')
self.p.debug("Using Vagrant config:\n"
" Vagrantfile: %s" % vagrantfile)
# Create a Vagrant object.
self.vagrant = Vagrant(vagrantfile)
return
def create(self, name):
""" Create a qb machine. """
try:
self.p.info("Creating %s..." % name)
self.vagrant.up(vm_name=name)
self.p.info('Done!')
except Exception as e:
self.p.debug(e)
self.p.error("Failed to create machine.")
sys.exit(1)
return
def start(self, name):
""" Start a qb machine. """
self.p.debug("Starting \"%s\"." % name)
return
def stop(self, name):
""" Stop a qb machine. """
self.p.debug("Stopping \"%s\"." % name)
return
def remove(self, name):
""" Remove a qb machine. """
self.p.debug("Removing \"%s\"." % name)
return
def vagrant():
v = Vagrant(os.path.dirname(__file__))
# Start remote VM first so local VM can copy its SSH key to it afterwards
v.up(vm_name="remote")
v.up(vm_name="local")
yield v
v.halt()
def execute_vagrant_command_on_minion(self, project_name, command):
minion_path = self.parent.config['minions'][project_name]
vagrant = Vagrant(minion_path, quiet_stdout=False, quiet_stderr=False)
puts(
colored.blue("Execute vagrant %s on minion %s" %
(command, project_name)))
getattr(vagrant, command)()
puts(colored.blue("Done"))
def __init__(self, *args, **kwargs): """Check that the vagrant_boxes attribute is not left empty, and is populated by all boxes if left blank""" self.vagrant = Vagrant(self.vagrant_root) if not self.vagrant_boxes: boxes = [s.name for s in self.vagrant.status()] if len(boxes) == 1: self.vagrant_boxes = ['default'] else: self.vagrant_boxes = boxes super(VagrantTestCase, self).__init__(*args, **kwargs)
def __init__(self, host="default", vagrantDir=None, controlVagrant=True):
global _defaultVagrantDir
self.host = host
self._controlVagrant = controlVagrant
if vagrantDir is None:
vagrantDir = _defaultVagrantDir
self._vagrant = Vagrant(root=vagrantDir)
self._startedVagrant = False
self._sshClient = None
self._sshConfigStr = None
self._sshConfig = None
self._sshHostConfig = None
def __init__(self, vagrantfile):
""" Configure logging. """
# Use the logger object created by Client.
self.p = logging.getLogger('qb')
self.p.debug("Using Vagrant config:\n"
" Vagrantfile: %s" % vagrantfile)
# Create a Vagrant object.
self.vagrant = Vagrant(vagrantfile)
return
class VagrantDriver(SandboxDriver):
"""Class to deal with Vagrant Boxes.
"""
@staticmethod
def status():
"""
Return the status of this vm instance
:return: the output of vagrant status in machine readable format
"""
return Vagrant.status()
def add_box(self, box_name):
"""Function to add a vagrant box if not exists.
Args:
box_name (str): the name of box, ex ubuntu/trusty64
"""
Vagrant.box_add(box_name)
def __init__(self, (box_name, box_id)):
""" init the challenge based on box_name in the directory named box_id
Args:
box_name (str): the name of box, ex ubuntu/trusty64
box_id (str): the id of box
"""
# Add a base box if not exists
self.vagrantAddBox(box_name)
# Create a challenge directory
if not os.path.exists("./data"):
os.makedirs("./data")
tmp_curr_dir = "./data"
if not os.path.exists(tmp_curr_dir + "/boxes"):
os.makedirs(tmp_curr_dir + "/boxes")
tmp_curr_dir += "/boxes"
challenge_id_base = box_name.replace('/', '_')
i = 1
challenge_id = challenge_id_base + str(i)
while os.path.exists(tmp_curr_dir + "/" + challenge_id):
i += 1
challenge_id = challenge_id_base + str(i)
self.boxId = challenge_id
os.makedirs(tmp_curr_dir + "/" + challenge_id)
tmp_curr_dir += "/" + challenge_id
os.chdir(SandboxDriver.HACKADEMIC_TMP_CHAL_DIR + box_id)
Vagrant.init(box_name)
self.setup()
def test_shapesBandwidth(self):
'''
Tests that bandwidth shaping works.
Examines the network speed before, during, and after shaping.
Fails if the network speeds do not reflect expected results.
'''
with Vagrant.ssh('gateway', 'client', 'server') as machines:
gateway, client, server = machines
before = speedBetween(client, server, **IPERF_OPTS)
print 'Actual speed before shaping:', before
shapedSpeed = before / 1024
print 'Desired shaping speed:', shapedSpeed
shape(gateway, client, shapedSpeed)
during = speedBetween(client, server, **IPERF_OPTS)
print 'Actual speed during shaping:', during
unshape(gateway, client)
after = speedBetween(client, server, **IPERF_OPTS)
print 'Actual speed after shaping:', after
if before.slower(Megabit):
self.fail(
'Actual speed before shaping is too slow for shape'
' testing. (is it already being shaped?)')
if during.faster(shapedSpeed):
self.fail(
'Actual speed during shaping exceeded'
' shaping speed.')
if after.slower(shapedSpeed * 2):
self.fail(
'Actual speed after shaping appears'
' to still be shaped.')
if after.slower(before * 0.7):
self.fail(
'Actual speed after shaping did not return'
' to normal value after shaping.')
def stop(self, challenge_id):
"""Function to stop a vagrant box in current dir.
Stops the VM
Args:
challenge_id (str): the challengeId retrived during create command
"""
_challengeID = challenge_id
if not os.path.exists("./data/challenges/" + _challengeID):
self.out['error'] = True
self.out[
'message'] = 'unable to stop %s, as it doesn\'t exist' % _challengeID
else:
challenge_path = "./data/challenges/" + _challengeID
# get the box ID
with open(challenge_path + '/.status', 'r') as cStatus:
try:
boxId = json.loads(cStatus.readline())['boxId']
cStatus.close()
# Update basebox status
with open("./data/boxes/" + boxId + "/.status",
'r+') as bStatus:
baseboxStatus = json.loads(bStatus.readline())
baseboxStatus['active'] -= 1
bStatus.seek(0)
bStatus.write(json.dumps(baseboxStatus))
bStatus.truncate()
except Exception:
print("Couldn't destroy challenge")
Vagrant.destroy(challenge_id)
# delete the challenegeID Dir
shutil.rmtree(challenge_path)
self.out['message'] = _challengeID + ' deleted successfully'
self.out['data'] = {}
self.out['data']['challenegeId'] = _challengeID
def do_action_run(self):
if self._is_vagrant:
self.copy_vagrantfile()
self.vagrant = Vagrant(self.user_path)
if self.params.stop:
self._project_stop()
elif self.params.restart:
self._project_restart()
elif self.params.suspend:
self._project_suspend()
elif self.params.reset:
self._project_reset()
else:
self._project_start()
def test_shapesBandwidth(self):
"""
Tests that bandwidth shaping works.
Examines the network speed before, during, and after shaping.
Fails if the network speeds do not reflect expected results.
"""
with Vagrant.ssh("gateway", "client", "server") as machines:
gateway, client, server = machines
before = speedBetween(client, server, **IPERF_OPTS)
print "Actual speed before shaping:", before
shapedSpeed = before / 1024
print "Desired shaping speed:", shapedSpeed
shape(gateway, client, shapedSpeed)
during = speedBetween(client, server, **IPERF_OPTS)
print "Actual speed during shaping:", during
unshape(gateway, client)
after = speedBetween(client, server, **IPERF_OPTS)
print "Actual speed after shaping:", after
if before.slower(Megabit):
self.fail("Actual speed before shaping is too slow for shape" " testing. (is it already being shaped?)")
if during.faster(shapedSpeed):
self.fail("Actual speed during shaping exceeded" " shaping speed.")
if after.slower(shapedSpeed * 2):
self.fail("Actual speed after shaping appears" " to still be shaped.")
if after.slower(before * 0.7):
self.fail("Actual speed after shaping did not return" " to normal value after shaping.")
def do_clean(self, project_name):
minion_path = self.parent.config['minions'][project_name]
# Check if sahara is available
minion_path = self.parent.config['minions'][project_name]
try:
plugins = Vagrant(minion_path).plugin_list()
except Exception as e:
raise
sahara = bool([p for p in plugins if p.name == "sahara"])
# Check sandbox status
sandbox_status = 'off'
if sahara:
sandbox = SandboxVagrant(minion_path)
sandbox_status = sandbox.sandbox_status()
# If sandbox is activated, rollback only
if sandbox_status == 'on' and not self.force_clean:
message = "Rollback snapshot, if you will delete snapshot, use "\
"--force-clean option"
puts(colored.blue(message))
sandbox.sandbox_rollback()
puts(colored.blue("Done"))
command = "sudo /etc/init.d/salt-minion restart"
result = get_output_cmd(command, minion_path)
puts(colored.blue("Restarted salt-minion"))
puts(colored.blue("Wait some time for salt-minion to connect"))
sleep(10)
# Else destroy and up
else:
# Destroy
VagrantDestroy.parent = self.parent
VagrantDestroy.run(['destroy', project_name], exit=False)
# Up
VagrantUp.parent = self.parent
VagrantUp.run(['up', project_name], exit=False)
class SystemVM(object):
def __init__(self, host='default', vagrantDir=None, controlVagrant=True):
global _defaultVagrantDir
self.host = host
self._controlVagrant = controlVagrant
if vagrantDir is None:
vagrantDir = _defaultVagrantDir
self._vagrant = Vagrant(root=vagrantDir)
self._startedVagrant = False
self._sshClient = None
self._sshConfigStr = None
self._sshConfig = None
self._sshHostConfig = None
def maybeUp(self):
if not self._controlVagrant:
return
state = self._vagrant.status(vm_name=self.host)[0].state
if state == Vagrant.NOT_CREATED:
self._vagrant.up(vm_name=self.host)
self._startedVagrant = True
elif state in [Vagrant.POWEROFF, Vagrant.SAVED, Vagrant.ABORTED]:
raise Exception(
"SystemVM testing does not support resume(), do not use vagrant suspend/halt"
)
elif state == Vagrant.RUNNING:
self._startedVagrant = False
else:
raise Exception("Unrecognized vagrant state %s" % state)
def maybeDestroy(self):
if not self._controlVagrant or not self._startedVagrant:
return
self._vagrant.destroy(vm_name=self.host)
if self._sshClient is not None:
self._sshClient.close()
def loadSshConfig(self):
if self._sshConfig is None:
self._sshConfigStr = self._vagrant.ssh_config(vm_name=self.host)
configObj = StringIO(self._sshConfigStr)
self._sshConfig = SSHConfig()
# noinspection PyTypeChecker
self._sshConfig.parse(configObj)
self._sshHostConfig = self._sshConfig.lookup(self.host)
@property
def sshConfig(self):
if self._sshConfig is None:
self.loadSshConfig()
return self._sshConfig
@property
def sshConfigStr(self):
if self._sshConfigStr is None:
self.loadSshConfig()
return self._sshConfigStr
@property
def sshClient(self):
if self._sshClient is None:
self.loadSshConfig()
self._sshClient = SSHClient()
self._sshClient.set_missing_host_key_policy(AutoAddPolicy())
self._sshClient.connect(self.hostname,
self.sshPort,
self.sshUser,
key_filename=self.sshKey,
timeout=10)
return self._sshClient
@property
def hostname(self):
return self._sshHostConfig.get('hostname', self.host)
@property
def sshPort(self):
return int(self._sshHostConfig.get('port', 22))
@property
def sshUser(self):
return self._sshHostConfig.get('user', 'root')
@property
def sshKey(self):
return self._sshHostConfig.get('identityfile', '~/.ssh/id_rsa')
def __init__(self, root_dir, env):
self.instance = Vagrant(root=root_dir)
self.env = env
self.target = GuestOS(env[:-2], env[-2:])
def status():
"""
Return the status of this vm instance
:return: the output of vagrant status in machine readable format
"""
return Vagrant.status()
def add_box(self, box_name):
"""Function to add a vagrant box if not exists.
Args:
box_name (str): the name of box, ex ubuntu/trusty64
"""
Vagrant.box_add(box_name)
class VagrantTestCase(TestCase):
"""
TestCase class to control vagrant boxes during testing
vagrant_boxes: An iterable of vagrant boxes. If empty or None, all boxes will be used. Defaults to []
vagrant_root: The root directory that holds a Vagrantfile for configuration. Defaults to the working directory
restart_boxes: If True, the boxes will be restored to their initial states between each test, otherwise the boxes
will remain up. Defaults to False
"""
vagrant_boxes = []
vagrant_root = None
restart_boxes = False
__initial_box_statuses = {}
__cleanup_actions = {
Vagrant.NOT_CREATED: 'destroy',
Vagrant.POWEROFF: 'halt',
Vagrant.SAVED: 'suspend',
}
def __init__(self, *args, **kwargs):
"""Check that the vagrant_boxes attribute is not left empty, and is populated by all boxes if left blank"""
self.vagrant = Vagrant(self.vagrant_root)
if not self.vagrant_boxes:
boxes = self.vagrant.status().keys()
if len(boxes) == 1:
self.vagrant_boxes = ['default']
else:
self.vagrant_boxes = boxes
super(VagrantTestCase, self).__init__(*args, **kwargs)
def assertBoxStatus(self, box, status):
"""Assertion for a box status"""
box_status = self.vagrant.status()[box]
if box_status != status:
self.failureException('{} has status {}, not {}'.format(
box, box_status, status))
def assertBoxUp(self, box):
"""Assertion for a box being up"""
self.assertBoxStatus(box, Vagrant.RUNNING)
def assertBoxSuspended(self, box):
"""Assertion for a box being up"""
self.assertBoxStatus(box, Vagrant.SAVED)
def assertBoxHalted(self, box):
"""Assertion for a box being up"""
self.assertBoxStatus(box, Vagrant.POWEROFF)
def assertBoxNotCreated(self, box):
"""Assertion for a box being up"""
self.assertBoxStatus(box, Vagrant.NOT_CREATED)
def run(self, result=None):
"""Override run to have provide a hook into an alternative to tearDownClass with a reference to self"""
self.setUpOnce()
run = super(VagrantTestCase, self).run(result)
self.tearDownOnce()
return run
def setUpOnce(self):
"""Collect the box states before starting"""
for box_name in self.vagrant_boxes:
self.__initial_box_statuses[box_name] = self.vagrant.status(
)[box_name]
def tearDownOnce(self):
"""Restore all boxes to their initial states after running all tests, unless tearDown handled it already"""
if not self.restart_boxes:
self.restore_box_states()
def restore_box_states(self):
"""Restores all boxes to their original states"""
for box_name in self.vagrant_boxes:
action = self.__cleanup_actions.get(
self.__initial_box_statuses[box_name])
if action:
getattr(self.vagrant, action)(vm_name=box_name)
def setUp(self):
"""Starts all boxes before running tests"""
for box_name in self.vagrant_boxes:
self.vagrant.up(vm_name=box_name)
super(VagrantTestCase, self).setUp()
def tearDown(self):
"""Returns boxes to their initial status after each test if self.restart_boxes is True"""
if self.restart_boxes:
self.restore_box_states()
super(VagrantTestCase, self).tearDown()
class SystemVM(object):
def __init__(self, host="default", vagrantDir=None, controlVagrant=True):
global _defaultVagrantDir
self.host = host
self._controlVagrant = controlVagrant
if vagrantDir is None:
vagrantDir = _defaultVagrantDir
self._vagrant = Vagrant(root=vagrantDir)
self._startedVagrant = False
self._sshClient = None
self._sshConfigStr = None
self._sshConfig = None
self._sshHostConfig = None
def maybeUp(self):
if not self._controlVagrant:
return
state = self._vagrant.status(vm_name=self.host)[0].state
if state == Vagrant.NOT_CREATED:
self._vagrant.up(vm_name=self.host)
self._startedVagrant = True
elif state in [Vagrant.POWEROFF, Vagrant.SAVED, Vagrant.ABORTED]:
raise Exception("SystemVM testing does not support resume(), do not use vagrant suspend/halt")
elif state == Vagrant.RUNNING:
self._startedVagrant = False
else:
raise Exception("Unrecognized vagrant state %s" % state)
def maybeDestroy(self):
if not self._controlVagrant or not self._startedVagrant:
return
self._vagrant.destroy(vm_name=self.host)
if self._sshClient is not None:
self._sshClient.close()
def loadSshConfig(self):
if self._sshConfig is None:
self._sshConfigStr = self._vagrant.ssh_config(vm_name=self.host)
configObj = StringIO(self._sshConfigStr)
self._sshConfig = SSHConfig()
# noinspection PyTypeChecker
self._sshConfig.parse(configObj)
self._sshHostConfig = self._sshConfig.lookup(self.host)
@property
def sshConfig(self):
if self._sshConfig is None:
self.loadSshConfig()
return self._sshConfig
@property
def sshConfigStr(self):
if self._sshConfigStr is None:
self.loadSshConfig()
return self._sshConfigStr
@property
def sshClient(self):
if self._sshClient is None:
self.loadSshConfig()
self._sshClient = SSHClient()
self._sshClient.set_missing_host_key_policy(AutoAddPolicy())
self._sshClient.connect(self.hostname, self.sshPort, self.sshUser, key_filename=self.sshKey, timeout=10)
return self._sshClient
@property
def hostname(self):
return self._sshHostConfig.get("hostname", self.host)
@property
def sshPort(self):
return int(self._sshHostConfig.get("port", 22))
@property
def sshUser(self):
return self._sshHostConfig.get("user", "root")
@property
def sshKey(self):
return self._sshHostConfig.get("identityfile", "~/.ssh/id_rsa")
#!/usr/bin/env python
# encoding: utf-8
import time
import json
import threading
from vagrant import Vagrant
from flask import Flask
app = Flask(__name__)
vagrant = Vagrant(quiet_stdout=False, quiet_stderr=False)
checked_out = None
minimum_standby_vms = 2
checked_in = []
def get_alive_vms():
return [x for x in vagrant.status() if x.state == 'running' and
x.name not in checked_in]
def get_dead_vms():
return [x for x in vagrant.status() if x.state != 'running']
@app.route("/")
def hello():
return "Hello World!"
def test_vagrant(temp_repo_func, destroy=False):
if os.environ.get("TRAVIS", False):
pytest.skip("Vagrant doesn't work on Travis")
backend = VagrantBackend(verbosity=2,
use_registry_name="docker.io/mikicz/arca-test",
keep_vm_running=True)
arca = Arca(backend=backend, base_dir=BASE_DIR)
if destroy:
vagrant_location = backend.get_vm_location()
if vagrant_location.exists():
vagrant = Vagrant(vagrant_location)
vagrant.destroy()
shutil.rmtree(vagrant_location)
# master branch - return colorama version
temp_repo_func.file_path.write_text(RETURN_COLORAMA_VERSION_FUNCTION)
requirements_path = temp_repo_func.repo_path / backend.requirements_location
requirements_path.write_text("colorama==0.3.9")
temp_repo_func.repo.index.add(
[str(temp_repo_func.file_path),
str(requirements_path)])
temp_repo_func.repo.index.commit("Initial")
# branch branch - return unicode
temp_repo_func.repo.create_head("branch")
temp_repo_func.repo.branches.branch.checkout()
temp_repo_func.file_path.write_text(SECOND_RETURN_STR_FUNCTION)
temp_repo_func.repo.index.add([str(temp_repo_func.file_path)])
temp_repo_func.repo.index.commit("Test unicode on a separate branch")
task = Task("test_file:return_str_function")
assert arca.run(temp_repo_func.url, temp_repo_func.branch,
task).output == "0.3.9"
# halt the VM, checks that the VM can be booted when stopped with the vagrant attribute set
backend.stop_vm()
assert arca.run(temp_repo_func.url, temp_repo_func.branch,
task).output == "0.3.9"
# halt the vm and create a new instance of the backend, to check that the vagrant attribute can be set from existing
backend.stop_vm()
backend = VagrantBackend(verbosity=2,
use_registry_name="docker.io/mikicz/arca-test",
keep_vm_running=True)
arca = Arca(backend=backend, base_dir=BASE_DIR)
assert arca.run(temp_repo_func.url, temp_repo_func.branch,
task).output == "0.3.9"
# test that two branches can work next to each other
assert arca.run(temp_repo_func.url, temp_repo_func.branch,
task).output == "0.3.9"
assert arca.run(temp_repo_func.url, "branch", task).output == TEST_UNICODE
# test timeout
temp_repo_func.repo.branches.master.checkout()
temp_repo_func.file_path.write_text(WAITING_FUNCTION)
temp_repo_func.repo.index.add([str(temp_repo_func.file_path)])
temp_repo_func.repo.index.commit("Waiting function")
task_1_second = Task("test_file:return_str_function", timeout=1)
task_3_seconds = Task("test_file:return_str_function", timeout=3)
with pytest.raises(BuildTimeoutError):
assert arca.run(temp_repo_func.url, temp_repo_func.branch,
task_1_second).output == "Some string"
assert arca.run(temp_repo_func.url, temp_repo_func.branch,
task_3_seconds).output == "Some string"
backend.stop_vm()
if destroy:
backend.destroy = True
backend.stop_vm()
class VagrantTestCase(TestCase):
"""
TestCase class to control vagrant boxes during testing
vagrant_boxes: An iterable of vagrant boxes. If empty or None, all boxes will be used. Defaults to []
vagrant_root: The root directory that holds a Vagrantfile for configuration. Defaults to the working directory
restart_boxes: If True, the boxes will be restored to their initial states between each test, otherwise the boxes
will remain up. Defaults to False
"""
vagrant_boxes = []
vagrant_root = None
restart_boxes = False
__initial_box_statuses = {}
__cleanup_actions = {
Vagrant.NOT_CREATED: 'destroy',
Vagrant.POWEROFF: 'halt',
Vagrant.SAVED: 'suspend',
}
def __init__(self, *args, **kwargs):
"""Check that the vagrant_boxes attribute is not left empty, and is populated by all boxes if left blank"""
self.vagrant = Vagrant(self.vagrant_root)
if not self.vagrant_boxes:
boxes = [s.name for s in self.vagrant.status()]
if len(boxes) == 1:
self.vagrant_boxes = ['default']
else:
self.vagrant_boxes = boxes
super(VagrantTestCase, self).__init__(*args, **kwargs)
def assertBoxStatus(self, box, status):
"""Assertion for a box status"""
box_status = [s.state for s in self.vagrant.status() if s.name == box][0]
if box_status != status:
self.failureException('{} has status {}, not {}'.format(box, box_status, status))
def assertBoxUp(self, box):
"""Assertion for a box being up"""
self.assertBoxStatus(box, Vagrant.RUNNING)
def assertBoxSuspended(self, box):
"""Assertion for a box being up"""
self.assertBoxStatus(box, Vagrant.SAVED)
def assertBoxHalted(self, box):
"""Assertion for a box being up"""
self.assertBoxStatus(box, Vagrant.POWEROFF)
def assertBoxNotCreated(self, box):
"""Assertion for a box being up"""
self.assertBoxStatus(box, Vagrant.NOT_CREATED)
def run(self, result=None):
"""Override run to have provide a hook into an alternative to tearDownClass with a reference to self"""
self.setUpOnce()
run = super(VagrantTestCase, self).run(result)
self.tearDownOnce()
return run
def setUpOnce(self):
"""Collect the box states before starting"""
for box_name in self.vagrant_boxes:
box_state = [s.state for s in self.vagrant.status() if s.name == box_name][0]
self.__initial_box_statuses[box_name] = box_state
def tearDownOnce(self):
"""Restore all boxes to their initial states after running all tests, unless tearDown handled it already"""
if not self.restart_boxes:
self.restore_box_states()
def restore_box_states(self):
"""Restores all boxes to their original states"""
for box_name in self.vagrant_boxes:
action = self.__cleanup_actions.get(self.__initial_box_statuses[box_name])
if action:
getattr(self.vagrant, action)(vm_name=box_name)
def setUp(self):
"""Starts all boxes before running tests"""
for box_name in self.vagrant_boxes:
self.vagrant.up(vm_name=box_name)
super(VagrantTestCase, self).setUp()
def tearDown(self):
"""Returns boxes to their initial status after each test if self.restart_boxes is True"""
if self.restart_boxes:
self.restore_box_states()
super(VagrantTestCase, self).tearDown()
def main(self, project_name):
minion_path = self.parent.config['minions'][project_name]
self.execute_vagrant_command_on_minion(project_name, 'up')
# Get back master IP
puts(colored.blue("Check master ip"))
command = "/sbin/ifconfig eth1 | grep 'inet addr:' | cut -d: -f2 | cut -d' ' -f 1"
vm_ip = get_output_cmd(command, minion_path)
vm_ip = vm_ip.split('.')
vm_ip[-1] = '1'
master_ip = ".".join(vm_ip)
# Check declared master ip
command = "grep master /etc/salt/minion"
declared_master_ip = get_output_cmd(command, minion_path)
declared_master_ip = declared_master_ip.split()[-1]
change = False
if declared_master_ip != master_ip:
puts(
colored.yellow(
"Master ip on minion is invalid, {0} instead of {1}".
format(declared_master_ip, master_ip)))
# Replace master ip
command = "sudo sed -i 's/{0}/{1}/g' /etc/salt/minion".format(
declared_master_ip, master_ip)
result = get_output_cmd(command, minion_path)
# Reboot minion
command = "sudo /etc/init.d/salt-minion restart"
result = get_output_cmd(command, minion_path)
# Change minion
command = "sed -i 's/{0}/{1}/g' {2}".format(
declared_master_ip, master_ip, join(minion_path, 'minion'))
check_output(command, shell=True)
# Wait for minion to connect
sleep(10)
puts(
colored.blue(
"Master ip has been updated from {0} to {1}".format(
declared_master_ip, master_ip)))
change = True
else:
puts(colored.blue("Master is good"))
# Check if sahara is available
minion_path = self.parent.config['minions'][project_name]
try:
plugins = Vagrant(minion_path).plugin_list()
except Exception as e:
raise
sahara = bool([p for p in plugins if p.name == "sahara"])
if not sahara:
message = "Sandbox support is not available, please install sahara"\
" plugin with 'vagrant plugin install sahara'"
puts(colored.yellow(message))
else:
sandbox = SandboxVagrant(minion_path,
quiet_stdout=False,
quiet_stderr=False)
sandbox_status = sandbox.sandbox_status()
if sandbox_status == 'on':
puts(colored.blue("Snapshot is already enabled on VM"))
if change:
puts(
colored.blue(
"Minion config has changes, would you like to redo the snapshot? It will save the VM in its current state"
))
choice = bool_choice("Yes/No: ")
if choice:
sandbox.sandbox_commit()
else:
puts(colored.blue("Would you like to enable snapshot on VM?"))
choice = bool_choice("Yes/No: ")
if choice:
puts(colored.blue("Starting snapshot"))
sandbox.sandbox_on()
puts(colored.blue("Done"))
def known_hosts(vagrant_root):
vagrant_root = lookup_vagrant_root(vagrant_root)
vagrant = Vagrant(vagrant_root)
return {machine: vagrant.hostname(machine) for machine in list_machines(vagrant_root)}
class RbxProjectRunAction(object):
vagrantfile = 'Vagrantfile'
configfile = 'rbx.sh'
@property
def package_path(self):
if not hasattr(self, '_package_path'):
self._package_path = os.path.abspath(
os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
return self._package_path
@property
def user_path(self):
if not hasattr(self, '_user_path'):
if self.params.location:
self._user_path = os.path.abspath(self.params.location)
else:
self._user_path = os.path.abspath(os.getcwd())
return self._user_path
def write_config(self, commands, config):
with open(os.path.join(self.user_path, self.configfile), 'w') as fd:
fd.writelines('\n'.join(commands))
for forwarded in config['ports']:
fd.write('\n#RBX_PORT:{0}'.format(forwarded))
for src, dest in config['volumes']:
fd.write('\n#RBX_VOLUME:{0}:{1}'.format(src, dest))
def _is_vagrant(self):
return not self.params.host
def do_action_run(self):
if self._is_vagrant:
self.copy_vagrantfile()
self.vagrant = Vagrant(self.user_path)
if self.params.stop:
self._project_stop()
elif self.params.restart:
self._project_restart()
elif self.params.suspend:
self._project_suspend()
elif self.params.reset:
self._project_reset()
else:
self._project_start()
def _project_start(self):
if self._is_vagrant():
status = self.vagrant.status()
if status.get('default') != Vagrant.RUNNING:
commands, config = self.build()
self.write_config(commands, config)
self.vagrant.up(False, capture_output=False)
self.vagrant.provision(capture_output=False)
else:
self.out('Sorry, not implemented yet!')
def _project_stop(self):
if self._is_vagrant():
status = self.vagrant.status()
if status.get('default') == Vagrant.RUNNING:
self.vagrant.halt(capture_output=False)
else:
self.out('Option no available for physical machine')
def _project_restart(self):
if self._is_vagrant():
self._project_stop()
self._project_start()
else:
self.out('Option no available for physical machine')
def _project_suspend(self):
if self._is_vagrant():
status = self.vagrant.status()
if status.get('default') == Vagrant.RUNNING:
self.vagrant.suspend(capture_output=False)
else:
self.out('Sorry, not implemented yet!')
def _project_reset(self):
if self._is_vagrant():
status = self.vagrant.status()
if status.get('default') == Vagrant.RUNNING:
commands, config = self.build()
self.write_config(commands, config)
self.vagrant.provision(capture_output=False)
else:
self._project_start()
else:
self.out('Sorry, not implemented yet!')
def copy_vagrantfile(self):
copy(os.path.join(self.package_path, self.vagrantfile),
os.path.join(self.user_path, self.vagrantfile))
def retrieve_components(self):
self.components = {}
for component in self.project['components']:
self.components[component['name']] = self.query('get',
'/component/{0}'.format(component['name']))
def build(self):
self.project = self.query_resource('get')
self.retrieve_components()
self.docker_command = []
names = []
commands = []
ports = []
volumes = []
for component in self.project['components']:
conf = self.components[component['name']]
commands.append(('echo "[rbx] Deploying component \'{0}\'"; '.format(component['name']) +
'[ "x$(docker ps | grep {0})" != "x" ] && docker stop -t=3 {0} &> /dev/null; ' +
'[ "x$(docker ps -a | grep {0})" != "x" ] && docker rm {0} &> /dev/null; ' +
'docker run -d -name {0} {1} {2} {3} {4} {5} &> /dev/null').format(
self._docker_name(component['name']),
self._docker_links(names),
self._docker_ports(component['port']),
self._docker_volumes(component['volume']),
conf['image'],
subprocess.list2cmdline(conf['command'])))
if component['port']:
for port in component['port']:
ports.append(port['forwarded'])
if component['volume']:
for volume in component['volume']:
_vol = (volume['mounted'], volume['path'])
if _vol in volumes:
continue
volumes.append(_vol)
names.append(component['name'])
return commands, {'ports': ports, 'volumes': volumes}
def _volume_path(self, conf):
if self.params.host:
return conf['mounted']
return conf['path']
def _docker_name(self, name):
return '{0}_{1}'.format(self.project['name'], name)
def _docker_links(self, names):
return ' '.join(['-link {0}:{1}'.format(self._docker_name(n), n)
for n in names])
def _docker_ports(self, ports):
return ' '.join(['-p {0}:{1}'.format(p['forwarded'], p['number'])
for p in ports])
def _docker_volumes(self, volumes):
return ' '.join(['-v {0}:{1}:ro'.format(self._volume_path(v),
v['path'])
for v in volumes])
from __future__ import print_function
from vagrant import Vagrant
v = Vagrant()
def mock_create(config):
try:
if config:
v.init(box_name=config['box_name'], box_url=config['box_url'])
else:
v.init()
return "SUCCESS"
except Exception as e:
return "FAILED"
def mock_vagrantfile():
with open('Vagrantfile', 'r') as myfile:
vagrantfile = myfile.read()
return vagrantfile
def mock_deploy(config):
with open('./src/mockdata/create_instance') as myfile:
status_file = myfile.read()
return status_file
