def main(machine, instances, queues=['high', 'default', 'low']):
r = StrictRedis.from_url(REDIS_URL_RQ)
machine_workers = [worker
for worker in Worker.all(connection=r)
if is_local(machine, worker.name) and \
any(works_on(worker, queue) for queue in queues)]
print "%d workers running on %s" % (len(machine_workers), machine)
if len(machine_workers):
print '\n'.join(
map(
lambda m: "%s\t%s\t%s" % (m.name, m.get_state(), "stopped"
if m.stopped else "running"),
machine_workers))
machine_info = workers(machine)
rem = SshMachine(machine_info['hostname'],
ssh_opts=SSH_OPTS,
**machine_info.get('kwargs', {}))
dir = rem.path(machine_info['rqdir'])
with rem.cwd(dir):
for i in xrange(0, instances - len(machine_workers)):
rem["./worker.sh"](' '.join(queues))
print "Worker spawned"
def open_db(urlstr=None):
urlstr = urlstr or _get_db_url(urlstr)
url = urlparse(urlstr)
if url.scheme:
machine = SshMachine(url.hostname, user=url.username, port=url.port)
else:
machine = local
yield urlstr, machine, machine.path(url.path)
if url.scheme:
machine.close()
def run_more_instances(machine, count, queues=['high', 'default', 'low']):
rem = SshMachine(machine,
ssh_opts=SSH_OPTS,
keyfile=KEYFILE,
user='******')
dir = rem.path('/home/ec2-user/rq')
with rem.cwd(dir):
for i in xrange(0, count):
rem["./worker.sh"](' '.join(queues))
print "Worker spawned"
def setup(machine):
rem = SshMachine(workers(machine)['hostname'], ssh_opts=SSH_OPTS)
dir = rem.path(workers(machine)['rqdir'])
if not dir.exists():
print "CLONING REPO..."
rem["git"]("clone", "http://github.com/darioush/rq-dist", dir)
print "CLONED..."
print "MAKING VIRTUAL ENV..."
with rem.cwd(dir):
rem["virtualenv"]("env")
print "MADE VIRTUAL ENV..."
with rem.cwd(dir):
print "UPDATING CODE ..."
rem["git"]("pull", "origin", "master")
print "UPDATING VENV ..."
rem["./update-venv.sh"]()
my_hostname, _, _ = socket.gethostname().partition('.')
if my_hostname == machine:
print "Not syncing master worker"
return
my_d4j = '/'.join(
get_property('d4j_path', my_hostname, 0)[0].split('/')[:-2])
dst_d4j = '/'.join(get_property('d4j_path', machine, 0)[0].split('/')[:-3])
print "RSYNCING FOR DEFECTS4J "
rsync = local['rsync']['-avz', '--exclude', '.git', '--exclude',
'project_repos'][my_d4j]
rsync('%s:%s' % (workers(machine)['hostname'], dst_d4j))
rem_d4j = rem.path(dst_d4j) / 'defects4j'
repos_dir = rem_d4j / 'project_repos'
if not repos_dir.exists():
with rem.cwd(rem_d4j):
print "GETTING REPOSITORIES..."
rem['./get-repos.sh']()
start_time = time.time()
print("Building wheel")
local["make"]["clean"]()
local["make"]["wheel"]()
PIP_PATH = Path("bin/pip")
print(f"Connecting to {args.hostname}")
remote = SshMachine(args.hostname)
venv_path = Path("/etc/virtualenvs") / args.venv
with remote.env(
PIP_INDEX_URL=local.env["ARTIFACTORY_URL"]), remote.tempdir() as temp:
wheel = next(file for file in local.path("dist").list()
if file.name.endswith("whl"))
print(f"Uploading {wheel} to {args.hostname}")
remote.upload(wheel, temp)
venv_exists = remote.path(venv_path).exists()
if not venv_exists:
print(f"No venv found at {venv_path}, creating venv")
remote.path(venv_path).mkdir()
remote["python3"]["-m"]["venv"][venv_path]()
pip_command = remote[str(venv_path / PIP_PATH)]["install"][str(
temp / wheel.name)]["--force-reinstall"]
if venv_exists:
pip_command = pip_command["--no-deps"]
print(f"Running {pip_command} on {args.hostname}")
pip_command & FG
print(f"Completed installation in {time.time() - start_time}s")
class VM:
"""
Manage a qemu-system virtual machine.
It will be started with `__init__` and the ssh connection will be
established with `__enter__`, so any ssh operation should be done
inside a `with` block.
A CPU allocation can also be provided, to bind the cores of the
virtual machine to physical cores.
:ivar ssh: plumbum.SshMachine object, useful to run commands on
the VM. It should only be used inside a `with` block.
:ivar process: popen process of qemu, useful to send signals
or input to qemu.
:example:
with VM('bzImage', 'debian.img', '~/.ssh/id_rsa') as vm:
vm.shh['ls']
"""
def __init__(self,
kernel_path,
filesystem_img_path,
keyfile,
cpu_allocation=None,
isolcpus=[]):
"""Start the qemu VM (non blocking)
:param kernel_path: Path of the kernel's bzImage
:param filesystem_img_path: Path of the filesystem image (.img)
:param keyfile: Path of rsa key that is authorized on the image
:param cpu_allocation: CpuAllocation for qemu and the vm's cores,
or None to not assign CPUs
:param isolcpus: list of CPU ID that should be isolated at boot time
"""
qemu_args = VM.__construct_qemu_args(
kernel_path=kernel_path,
filesystem_img_path=filesystem_img_path,
isolcpus=isolcpus)
self.process = local['qemu-system-x86_64'].popen(qemu_args)
self.ssh = None
self.key = keyfile
if cpu_allocation:
VM.__qemu_affinity_setup(self.process.pid, cpu_allocation)
def __enter__(self):
"""Initialize the ssh connection (blocks until success)"""
err = None
for _ in range(SSH_MAX_RETRY):
time.sleep(1)
try:
self.ssh = SshMachine('127.0.0.1',
user='******',
port=HOST_PORT,
keyfile=self.key)
break
except (EOFError, plumbum.machines.session.SSHCommsError) as e:
err = e
continue
else: # Reached maximum retries
raise VMException('SSH connection failed after too many retries',
err)
return self
def __exit__(self, type, value, traceback):
"""Stop the SSH connection and the VM"""
if self.ssh is not None:
self.ssh.close()
self.ssh = None
self.process.terminate()
def scp_to(self, src_local, dst_remote):
"""Send a file from the host to the VM
:param src_local: local path of the file to send
:param dst_remote: destination path on the vm
:raises ValueError: when the ssh connection is not established,
i.e. when not used inside a `with` block
"""
if self.ssh is None:
raise VMException(
'`VM.scp_to` must be used with an established SSH connection, '
'i.e. inside a `with` block.')
src = local.path(src_local)
dst = self.ssh.path(dst_remote)
plumbum.path.utils.copy(src, dst)
@staticmethod
def __construct_qemu_args(kernel_path, filesystem_img_path, isolcpus=[]):
"""Qemu arguments similar to what `vm start` produces"""
kernel_opt = ''
if isolcpus:
kernel_opt = ' isolcpus=' + ','.join(map(str, isolcpus))
return [
'-nographic', '-s', '-machine', 'accel=kvm', '-cpu', 'host',
'-device', 'e1000,netdev=net0', '-netdev',
'user,id=net0,hostfwd=tcp::%d-:22' % HOST_PORT, '-append',
'console=ttyS0,115200 root=/dev/sda rw nokaslr' + kernel_opt,
'-smp', '2', '-m', '4G', '-drive',
'if=none,id=hd,file=%s,format=raw' % filesystem_img_path,
'-device', 'virtio-scsi-pci,id=scsi', '-device',
'scsi-hd,drive=hd', '-device',
'virtio-rng-pci,max-bytes=1024,period=1000', '-qmp',
'tcp:localhost:4444,server,nowait', '-serial', 'mon:stdio',
'-kernel',
'%s' % kernel_path, '-name', 'lsm_perf_vm,debug-threads=on'
]
@staticmethod
def __qemu_affinity_setup(qemu_pid, cpu_alloc):
"""Run qemu_affinity.py to allocate CPUs based on the CpuAllocation"""
system_affinities = (
'-p %(sys)d -i *:%(sys)d -q %(sys)d -w *:%(sys)d' % {
'sys': cpu_alloc.qemu_sys
}).split(' ')
kvm_affinities = [
'-k', str(cpu_alloc.host_kvm0),
str(cpu_alloc.host_kvm1)
]
args = system_affinities + kvm_affinities + ['--', str(qemu_pid)]
cmd = plumbum.cmd.sudo[sys.executable][QEMU_AFFINITY_PATH][args]
return cmd()
def teardown(machine):
rem = SshMachine(workers(machine)['hostname'], ssh_opts=SSH_OPTS)
dir = rem.path(workers(machine)['rqdir'])
print "REMOVING DIR.."
rem["rm"]("-rf", dir)