def __init__(self, task, config, voting=False): """ :param Task task: :param dict config: job config """ self.config = config self._task = weakref.ref(task) self.task_id = task.id self.task_started_at = task.started_at self.task_local_config = task.local_config self.error = 256 self.voting = config.get("voting", voting) self.root = task.root self.log = task.root.log self.timeout = self.config.get("timeout", 90) * 60 self.id = utils.get_rnd_name(length=10) self.env = copy.deepcopy(self.task.event.env) self.env.update(config.get("env", {})) self.status = "__init__" self.log_path = os.path.join(self.task.id, config["name"]) self.log.debug("Job %s initialized." % self.id) self.vms = [] self.console_listeners = []
def __init__(self, stream, project, event): self.id = utils.get_rnd_name("EVNT", length=10) self.stream = stream self.root = stream.root self.log = stream.root.log self.event = event self.project = project self.jobs = {} self.jobs_list = [] self.finished_at = 0 self.cfg = self.root.config.data["project"][self.project] env = self._get_env() if event["type"] == "ref-updated": for job_name in self.cfg.get("on-ref-updated", []): job = Job(self, job_name) job.env.update(env) self.jobs_list.append(job) return for job_name in self.cfg.get("jobs", []): job = Job(self, job_name) job.env.update(env) self.jobs_list.append(job) for job_name in self.cfg.get("non-voting-jobs", []): job = Job(self, job_name) job.env.update(env) job.voting = False self.jobs_list.append(job)
def __init__(self, name=None, memory=1024): self.macs = [] if name is None: self.name = utils.get_rnd_name() else: self.name = name x = XMLElement(None, "domain", type="kvm") self.x = x x.se("name").x.text = self.name for mem in ("memory", "currentMemory"): x.se(mem, unit="MiB").x.text = str(memory) x.se("vcpu", placement="static").x.text = "1" cpu = x.se("cpu", mode="host-model") cpu.se("model", fallback="forbid") os = x.se("os") os.se("type", arch="x86_64", machine="pc-1.0").x.text = "hvm" features = x.se("features") features.se("acpi") features.se("apic") features.se("pae") self.devices = x.se("devices") self.devices.se("emulator").x.text = "/usr/bin/kvm" self.devices.se("controller", type="pci", index="0", model="pci-root") self.devices.se("graphics", type="spice", autoport="yes") mb = self.devices.se("memballoon", model="virtio") mb.se("address", type="pci", domain="0x0000", bus="0x00", slot="0x09", function="0x0")
def _get_vm(self, local_cfg, conf): """ :param local_cfg: config.job.runner.vms item :param conf: config.provider.vms item """ LOG.debug("Creating VM with conf %s" % conf) name = local_cfg["name"] image = conf.get("image") if image: yield from self.build_image(image) else: image = name rnd_name = utils.get_rnd_name(name) yield from self.storage.clone(image, rnd_name) vm = VM(self, conf, local_cfg) files = yield from self.storage.list_files(rnd_name) vm.disks.append(rnd_name) for f in files: vm.add_disk(f) for net in conf["net"]: net = net.split(" ") if len(net) == 1: vm.add_net(net[0]) else: vm.add_net(net[0], mac=net[1]) yield from vm.boot() self.vms.append(vm) return vm
def _get_vm(self, name, conf): """ :param conf: config.provider.vms item """ LOG.debug("Creating VM %s" % name) image = conf.get("image") if image: yield from self.build_image(image) else: image = name rnd_name = utils.get_rnd_name("rci_" + name) yield from self.storage.clone(image, rnd_name) vm = VM(self, name, conf) files = yield from self.storage.list_files(rnd_name) vm.disks.append(rnd_name) for f in files: vm.add_disk(f) for net in conf["net"]: net = net.split(" ") if len(net) == 1: vm.add_net(net[0]) else: vm.add_net(net[0], mac=net[1]) yield from vm.boot() return vm
def build(self, image_name=None): if image_name is None: image_name = self.runner_vm_conf.get("image") if image_name is None: # FIXME yield from asyncio.sleep(1) return self.job.set_status("building %s" % image_name) LOG.debug("Building image: %s" % image_name) build_key = (self.h_ssh.hostname, image_name) BUILDING_IMAGES.setdefault(build_key, asyncio.Lock()) with (yield from BUILDING_IMAGES[build_key]): self.dataset, image_source = self._get_source_image(image_name) cmd = "zfs list %s/%s@1" % (self.dataset, image_name) error = yield from self.h_ssh.run(cmd, raise_on_error=False) if not error: LOG.debug("Image %s already built." % image_name) return image_name # delete possibly stale image cmd = "zfs destroy %s/%s" % (self.dataset, image_name) yield from self.h_ssh.run(cmd, raise_on_error=False) # find source image image_conf = self.runner_conf["images"][image_name] parent = image_conf.get("parent") if parent: source = yield from self.build(parent) self.job.set_status("building %s" % image_name) else: source = image_source if "@" not in source: source += "@1" name = utils.get_rnd_name(prefix="rci_build_%s_" % image_name) xml = XML(name=name, memory=image_conf.get("memory", 1024)) target = "/".join([self.dataset, image_name]) try: cmd = "zfs clone %s/%s %s" % (self.dataset, source, target) yield from self.h_ssh.run(cmd) yield from self._add_disks(xml, target) mac = utils.get_rnd_mac() xml.add_net(image_conf.get("build-net", "virbr0"), mac) ssh = yield from self.boot(xml) image_conf = self.runner_conf["images"][image_name] LOG.debug("building image with image_conf %s" % image_conf) for script in image_conf.get("build-scripts", []): yield from self.run_script(ssh, script) yield from self._shutdown(ssh, xml) yield from self.h_ssh.run("zfs snapshot %s@1" % target) return image_name except: LOG.exception("Error while building %s" % image_name) yield from self.h_ssh.run("virsh destroy %s" % xml.name, raise_on_error=False) yield from asyncio.sleep(4) yield from self.h_ssh.run("zfs destroy %s" % target, raise_on_error=False) raise
def boot_vm(self): LOG.debug("Booting VM %s" % self.vm_conf["name"]) name = utils.get_rnd_name(prefix="rci_%s_" % self.vm_conf["name"]) self.xml = XML(name=name, memory=self.runner_vm_conf.get("memory", 1024)) image = self.runner_vm_conf.get("image") if image: dataset, src = self._get_source_image(image) src = image else: dataset = self.runner_vm_conf["dataset"] src = self.runner_vm_conf["source"] if "@" not in src: src += "@1" self.volume = "%s/%s" % (dataset, utils.get_rnd_name()) cmd = "zfs clone %s/%s %s" % (dataset, src, self.volume) yield from self.h_ssh.run(cmd) yield from self._add_disks(self.xml, self.volume) yield from self._setup_networks(self.xml, self.runner_vm_conf["net"]) self.ssh = yield from self.boot(self.xml)
def run_script(self, script): name = utils.get_rnd_name() LOG.debug("Starting script %s" % script) cmd = "docker run -i --name %s" % name for env in self.job.env.items(): cmd += " -e %s=%s" % (env) cmd += " %s %s" % (self.image, script["interpreter"]) result = yield from self.ssh.run(cmd, stdin=script["data"]) self.image = yield from self.ssh.run("docker commit %s" % name, return_output=True) self.images.append(self.image) self.containers.append(name) return result
def get_vm(self, image, job): yield from self._build_image(image, job) name = utils.get_rnd_name("rci_") cmd = ["lxc-clone", "-s", "-o", image, "-n", name] yield from self.ssh.run(cmd, stderr=print) cmd = ["lxc-start", "-d", "-n", name] yield from self.ssh.run(cmd, stderr=print) ip = yield from self._get_ip(name) vm = VM(self, job, ip, name) if job not in self.job_vm: self.job_vm[job] = [] self.job_vm[job].append(vm) return vm
def __init__(self, root, event): """ :param Root root: :param Event event: """ self.root = root self.event = event self.local_config = None self.jobs = [] self.finished_at = None self.started_at = time.time() self.id = utils.get_rnd_name(length=10) self._finished = asyncio.Event(loop=root.loop) self._job_futures = {}
def __init__(self, stream, raw_event): self.id = utils.get_rnd_name(prefix="", length=10) self.stream = stream self.root = stream.root self.raw_event = raw_event self.project_name = self.get_project_name(raw_event) self.jobs = {} self.jobs_list = [] self.cfg = self.root.config.data["project"][self.project_name] for job_name in self.cfg.get("jobs", []): self.jobs_list.append(Job(self, job_name)) for job_name in self.cfg.get("non-voting-jobs", []): job = Job(self, job_name) job.voting = False self.jobs_list.append(job)
def __init__(self, event, name): self.voting = True self.error = 254 # FIXME self.queued_at = time.time() self.event = event self.name = name self.root = event.root self.config = event.root.config.data["job"][name] self.timeout = self.config.get("timeout", 90) * 60 self.id = utils.get_rnd_name("JOB", length=10) self.env = self.config.get("env", {}).copy() self.status = "__init__" self.finished_at = 0 self.log_path = os.path.join(self.event.id, self.name) LOG.debug("Job %s initialized." % self.id)
def boot(self, xml): self.job.set_status("booting %s" % self.vm_conf["name"]) with xml.fd() as fd: conf = "/tmp/.conf.%s.xml" % utils.get_rnd_name() yield from self.h_ssh.run("cat > %s" % conf, stdin=fd) yield from self.h_ssh.run("virsh create %s" % conf) yield from self.h_ssh.run("rm %s" % conf) if not self.runner_vm_conf.get("no_ip"): ip = yield from asyncio.wait_for(self._get_ip(xml.macs), 120) LOG.debug("Got ip: %s" % ip) ip_env_var = self.vm_conf.get("ip_env_var") if ip_env_var: self.job.env[ip_env_var] = ip yield from asyncio.sleep(4) return asyncssh.AsyncSSH("root", ip, key=self.runner_conf.get("key"), cb=self.job.logger)
def __init__(self, event, name): self.voting = True self.error = 254 # FIXME self.queued_at = time.time() self.event = event self.name = name self.root = event.root self.config = event.root.config.data["job"][name] self.id = utils.get_rnd_name(prefix="", length=10) self.stream_number = 0 self.env = {} self.log_path = os.path.join(self.event.id, self.id) self.loggers = [] for logger in self.root.config.get_instances("logger"): logger.job = self self.loggers.append(logger) self.status = "queued" LOG.debug("Job %s initialized." % self.id)
def __init__(self, config, event): """Represent Change Request :param config: Config instance :param event: dict decoded from gerrit event json """ self.config = config self.event = event self.jobs = [] self.id = utils.get_rnd_name(prefix="", length=10) event_type = event["type"] LOG.debug("New event %s" % event_type) if event_type == "ref-updated": LOG.debug("Ref updated.") # TODO return project_name = self.event["change"]["project"] self.project = self.config.data["projects"].get(project_name) if not self.project: LOG.debug("Unknown project %s" % project_name) return if event_type == "patchset-created": LOG.debug("Patchset created for project %s" % self.project) self.prepare_jobs() return if event_type == "comment-added": regexp = self.config.data.get("recheck", {}).get("regexp") if not regexp: return LOG.debug("Matching comment with pattern '%s'" % regexp) m = re.search(regexp, event["comment"], re.MULTILINE) if m: LOG.info("Recheck requested.") self.prepare_jobs() else: LOG.debug("Ignoring comment '%s'" % event["comment"]) LOG.debug("Unknown event-type %s" % event_type)
def __init__(self, host, name, cfg=None): """Represent a VM. :param Host host: :param str name: :param dict cfg: config.provider.vms item """ self.host = host self.cfg = cfg or {} self._ssh = host.ssh self._ssh_cache = {} self.macs = [] self.disks = [] self.name = utils.get_rnd_name("rci_" + name) x = XMLElement(None, "domain", type="kvm") self.x = x x.se("name").x.text = self.name for mem in ("memory", "currentMemory"): x.se(mem, unit="MiB").x.text = str(self.cfg.get("memory", 1024)) x.se("vcpu", placement="static").x.text = str(self.cfg.get("vcpu", 1)) cpu = x.se("cpu", mode="host-model") cpu.se("model", fallback="forbid") os = x.se("os") os.se("type", arch="x86_64", machine="pc-1.0").x.text = "hvm" features = x.se("features") features.se("acpi") features.se("apic") features.se("pae") self.devices = x.se("devices") self.devices.se("emulator").x.text = "/usr/bin/kvm" self.devices.se("controller", type="pci", index="0", model="pci-root") self.devices.se("graphics", type="spice", autoport="yes") mb = self.devices.se("memballoon", model="virtio") mb.se("address", type="pci", domain="0x0000", bus="0x00", slot="0x09", function="0x0")