def _port_ready(self, port):
"""Add a port to the stage queue."""
assert not self._pending[port]
assert not self.ports[port].stack.failed
assert not port.dependency.check(self.stage)
del self._pending[port]
stagejob = self.ports[port]
if self._port_check(port):
if stagejob.complete():
stagejob.run()
else:
log.debug(
"StageBuilder._port_ready()",
"Port '%s': queuing job for stage %s" %
(port.origin, self.stage.name))
assert self.stage.prev in port.stages
self.update.emit(self, Builder.QUEUED, port)
stagejob.started.connect(self._started)
self.queue.add(stagejob)
else:
if not self.stage.check(port):
stagejob.run()
else:
stagejob.done()
log.debug(
"StageBuilder._port_ready()",
"Port '%s': skipping stage %s" %
(port.origin, self.stage.name))
def _post_make(self, status):
"""Process the results of make.target()."""
distfiles = set(self.port.attr["distfiles"])
self._fetch_lock.release(distfiles)
if status:
self._bad_checksum.difference_update(distfiles)
self._fetched.update(distfiles)
else:
files = ", ".join("'%s'" % i for i in distfiles)
log.debug(
"Fetch._post_make()",
"Fetch '%s': failed to fetch distfiles: %s" %
(self.port.origin, files))
self._bad_checksum.update(distfiles)
self._fetch_failed.update(distfiles)
# TODO:
# - extend queue to handle non-active "done" jobs
# - make queue finish via signal, not direct call to queue.done
# ? track which jobs are handling which distfiles (cleanup with done())
# Go through all the pending fetch jobs and see if any have been
# resolved due to this job:
for q in (queue.fetch.stalled, queue.fetch.queue):
for i in range(len(q) - 1, -1, -1):
j = q[i]
if (isinstance(j, Fetch)
and not distfiles.isdisjoint(j.port.attr["distfiles"])
and (not j.check(j.port) or j.complete())):
del q[i]
j.run()
return status
def _post_make(self, status):
"""Process the results of make.target()."""
distfiles = set(self.port.attr["distfiles"])
self._fetch_lock.release(distfiles)
if status:
self._bad_checksum.difference_update(distfiles)
self._fetched.update(distfiles)
else:
files = ", ".join("'%s'" % i for i in distfiles)
log.debug("Fetch._post_make()",
"Fetch '%s': failed to fetch distfiles: %s" %
(self.port.origin, files))
self._bad_checksum.update(distfiles)
self._fetch_failed.update(distfiles)
# TODO:
# - extend queue to handle non-active "done" jobs
# - make queue finish via signal, not direct call to queue.done
# ? track which jobs are handling which distfiles (cleanup with done())
# Go through all the pending fetch jobs and see if any have been
# resolved due to this job:
for q in (queue.fetch.stalled, queue.fetch.queue):
for i in range(len(q) - 1, -1, -1):
j = q[i]
if (isinstance(j, Fetch) and
not distfiles.isdisjoint(j.port.attr["distfiles"]) and
(not j.check(j.port) or j.complete())):
del q[i]
j.run()
return status
def _port_ready(self, port):
"""Add a port to the stage queue."""
assert not self._pending[port]
assert not self.ports[port].stack.failed
assert not port.dependency.check(self.stage)
del self._pending[port]
stagejob = self.ports[port]
if self._port_check(port):
if stagejob.complete():
stagejob.run()
else:
log.debug("StageBuilder._port_ready()",
"Port '%s': queuing job for stage %s" %
(port.origin, self.stage.name))
assert self.stage.prev in port.stages
self.update.emit(self, Builder.QUEUED, port)
stagejob.started.connect(self._started)
self.queue.add(stagejob)
else:
if not self.stage.check(port):
stagejob.run()
else:
stagejob.done()
log.debug("StageBuilder._port_ready()",
"Port '%s': skipping stage %s" %
(port.origin, self.stage.name))
def _add(self, port, pending=0):
"""Add a ports dependencies and prior stage to be built."""
from .env import flags
# Don't try and build a port that has already failed
# or cannot be built
if port.failed or port.dependency.failed:
self.ports[port].stage_done()
return
depends = port.dependency.check(self.stage)
# Add all outstanding ports to be installed
self._pending[port] = len(depends) + pending
for p in depends:
if p not in self._depends:
self._depends[p] = set()
depend_resolve(p).connect(self._depend_resolv)
self._depends[p].add(port)
# Build the previous stage if needed
if (self.prev_builder and (port.install_status <= flags["stage"] or
port.force) and port.stage < self.stage - 1):
self._pending[port] += 1
self.prev_builder.add(port).connect(self._stage_resolv)
log.debug("StageBuilder._add()",
"Port '%s': added job for stage %d, waiting on %d" %
(port.origin, self.stage, self._pending[port]))
# Build stage if port is ready
if not self._pending[port]:
self._port_ready(port)
def attr(origin):
"""Retrieve a ports attributes by using the attribute queue."""
# TODO inline function to caller
log.debug("attr()", "Port '%s': getting attribute" % origin)
attr_obj = Attr(origin)
queue.attr.add(job.AttrJob(attr_obj))
return attr_obj
def clean(self, force=False):
"""Remove port's working director and log files."""
if stacks.Build in self.stages or force:
mak = make.make_target(self, "clean", NOCLEANDEPENDS=True)
log.debug("Port.clean()", "Port '%s': full clean" % self.origin)
return mak.connect(self._post_clean)
else:
self._post_clean()
log.debug("Port.clean()", "Port '%s': quick clean" % self.origin)
return True
def clean(self, force=False):
"""Remove port's working director and log files."""
if stacks.Build in self.stages or force:
mak = make.make_target(self, "clean", NOCLEANDEPENDS=True)
log.debug("Port.clean()", "Port '%s': full clean" % self.origin)
return mak.connect(self._post_clean)
else:
self._post_clean()
log.debug("Port.clean()", "Port '%s': quick clean" % self.origin)
return True
def _depend_resolv(self, depend):
"""Update dependency structures for resolved dependency."""
if not self._port_failed(depend):
all_depends = ["'%s'" % i.origin for i in self._depends[depend]]
log.debug("StageBuilder._depend_resolv()",
"Port '%s': resolved stage %d for ports %s" %
(depend.origin, self.stage, ", ".join(all_depends)))
for port in self._depends.pop(depend):
if not self._port_failed(port):
self._pending[port] -= 1
if not self._pending[port]:
self._port_ready(port)
def _do_stage(self): # pylint: disable-msg=E0202
"""Issue a pkg.add() to install the package from a repo."""
log.debug(
"PackageInstaller._do_stage()",
"Port '%s': building stage %s" % (self.port.origin, self.name))
pkg_add = self._add_pkg()
# pkg_add may be False if installing `ports-mgmt/pkg` and
# env.flags["pkg_mgmt"] == "pkgng"
if pkg_add:
self.pid = pkg_add.connect(self._post_pkg_add).pid
else:
# Cannot call self._finalise from within self.work() ->
# self._do_stage()
event.post_event(self._finalise, False)
def _do_stage(self): # pylint: disable-msg=E0202
"""Issue a pkg.add() to install the package from a repo."""
log.debug("PackageInstaller._do_stage()",
"Port '%s': building stage %s" %
(self.port.origin, self.name))
pkg_add = self._add_pkg()
# pkg_add may be False if installing `ports-mgmt/pkg` and
# env.flags["pkg_mgmt"] == "pkgng"
if pkg_add:
self.pid = pkg_add.connect(self._post_pkg_add).pid
else:
# Cannot call self._finalise from within self.work() ->
# self._do_stage()
event.post_event(self._finalise, False)
def _finalise(self, status):
"""Finalise the stage."""
if not status:
log.error("Stage._finalise()", "Port '%s': failed stage %s" %
(self.port.origin, self.name))
if self.stack.name == "common":
for stack in self.port.stacks.values():
stack.failed = self.__class__
else:
self.stack.failed = self.__class__
self.failed = True
else:
log.debug("Stage._finalise()", "Port '%s': finished stage %s" %
(self.port.origin, self.name))
self.stack.working = False
self.port.stages.add(self.__class__)
self.done()
def _post_pkg_add(self, pkg_add):
"""Process the results of pkg.add()."""
self.pid = None
if pkg_add.wait() == make.SUCCESS:
log.debug("PackageInstaller._post_pkg_add()",
"Port '%s': finished stage %s" %
(self.port.origin, self.name))
if "explicit" not in self.port.flags:
pkg_change = self.pid = pkg.change(self.port, "explicit", False)
if pkg_change:
self.pid = pkg_change.connect(self._post_pkg_change).pid
return
self._finalise(True)
else:
log.error("PackageInstaller._port_pkg_add()",
"Port '%s': failed stage %s" %
(self.port.origin, self.name))
self._finalise(False)
def _depend_resolv(self, port):
"""Update dependency structures for resolved dependency."""
if not port.dependent.failed and env.flags["mode"] != "clean":
all_depends = ["'%s'" % i.origin for i in self._depends[port]]
resolved_ports = ", ".join(all_depends)
log.debug("StageBuilder._depend_resolv()",
"Port '%s': resolved stage %s for ports %s" %
(port.origin, self.stage.name, resolved_ports))
for port in self._depends.pop(port):
if port not in self.failed:
if not port.dependency.failed and env.flags["mode"] != "clean":
self._pending[port] -= 1
if not self._pending[port]:
self._port_ready(port)
else:
if not self.ports[port].stack.failed:
self.ports[port].stack.failed = True
self._port_failed(port)
def _port_ready(self, port):
"""Add a port to the stage queue."""
assert not self._pending[port]
assert not port.failed or port.dependency.fail
assert not port.dependency.check(self.stage)
del self._pending[port]
if self._port_check(port):
log.debug("StageBuilder._port_ready()",
"Port '%s': queuing job for stage %d" %
(port.origin, self.stage))
assert port.stage == self.stage - 1 or self.stage > Port.PACKAGE
self.update.emit(self, Builder.QUEUED, port)
self.ports[port].started.connect(self._started)
self.queue.add(self.ports[port])
else:
log.debug("StageBuilder._port_ready()",
"Port '%s': skipping stage %d" %
(port.origin, self.stage))
def _depend_resolv(self, port):
"""Update dependency structures for resolved dependency."""
if not port.dependent.failed and env.flags["mode"] != "clean":
all_depends = ["'%s'" % i.origin for i in self._depends[port]]
resolved_ports = ", ".join(all_depends)
log.debug(
"StageBuilder._depend_resolv()",
"Port '%s': resolved stage %s for ports %s" %
(port.origin, self.stage.name, resolved_ports))
for port in self._depends.pop(port):
if port not in self.failed:
if not port.dependency.failed and env.flags["mode"] != "clean":
self._pending[port] -= 1
if not self._pending[port]:
self._port_ready(port)
else:
if not self.ports[port].stack.failed:
self.ports[port].stack.failed = True
self._port_failed(port)
def _finalise(self, status):
"""Finalise the stage."""
if not status:
log.error(
"Stage._finalise()",
"Port '%s': failed stage %s" % (self.port.origin, self.name))
if self.stack.name == "common":
for stack in self.port.stacks.values():
stack.failed = self.__class__
else:
self.stack.failed = self.__class__
self.failed = True
else:
log.debug(
"Stage._finalise()",
"Port '%s': finished stage %s" % (self.port.origin, self.name))
self.stack.working = False
self.port.stages.add(self.__class__)
self.done()
def _post_pkg_add(self, pkg_add):
"""Process the results of pkg.add()."""
self.pid = None
if pkg_add.wait() == make.SUCCESS:
log.debug(
"PackageInstaller._post_pkg_add()",
"Port '%s': finished stage %s" % (self.port.origin, self.name))
if "explicit" not in self.port.flags:
pkg_change = self.pid = pkg.change(self.port, "explicit",
False)
if pkg_change:
self.pid = pkg_change.connect(self._post_pkg_change).pid
return
self._finalise(True)
else:
log.error(
"PackageInstaller._port_pkg_add()",
"Port '%s': failed stage %s" % (self.port.origin, self.name))
self._finalise(False)
def work(self):
assert not self.stack.working
assert not self.failed
log.debug("Stage.work()", "Port '%s': starting stage %s" %
(self.port.origin, self.name))
if not self.check(self.port):
# Cannot call self._finalise(True) directly as self.done() cannot
# be called from within the scope of self.work()
event.post_event(self._finalise, False)
return
assert self.prev in self.port.stages
assert (not self.port.dependency or
not self.port.dependency.check(self.__class__))
if self.complete():
# Cannot call self._finalise(True) directly as self.done() cannot
# be called from within the scope of self.work()
event.post_event(self._finalise, True)
else:
self._do_stage() # May throw job.JobStalled()
self.stack.working = time.time()
def _cleanup(self, stagejob):
"""Cleanup after the port has completed its stage."""
port = stagejob.port
log.debug("StageBuilder._cleanup()",
"Port '%s': completed job for stage %s" %
(stagejob.port.origin, self.stage.name))
failed = stagejob.stack.failed or env.flags["mode"] == "clean"
del self.ports[port]
if port in self.cleanup and not env.flags["mode"] == "clean":
self.cleanup.remove(port)
if not failed:
self.done.append(port)
self.update.emit(self, Builder.DONE, port)
if env.flags["target"][-1] == "clean":
queue.clean.add(job.CleanJob(port))
elif not failed:
self.succeeded.append(port)
self.update.emit(self, Builder.SUCCEEDED, port)
if failed:
self.update.emit(self, Builder.FAILED, port)
def _find_method(self, port):
"""Find a method to resolve the port."""
while True:
method = self.method[port]
if not method:
# No method left, port failed to resolve
del self.method[port]
for stack in port.stacks.values():
if stack.failed and stack.failed is not True:
port.flags.add("failed")
break
port.dependent.status_changed(exhausted=True)
if port.dependent.failed and not port.dependency.failed:
log.debug("DependLoader._find_method()",
"Port '%s': no viable resolve method found" %
(port.origin,))
return False
else:
self.method[port] = self._next(self.method[port])
if self._resolve(port, method):
log.debug("DependLoader._find_method()",
"Port '%s': resolving using method '%s'" %
(port.origin, method))
return True
else:
log.debug("DependLoader._find_method()",
"Port '%s': skipping resolve method '%s'" %
(port.origin, method))
def _find_method(self, port):
"""Find a method to resolve the port."""
while True:
method = self.method[port]
if not method:
# No method left, port failed to resolve
del self.method[port]
for stack in port.stacks.values():
if stack.failed and stack.failed is not True:
port.flags.add("failed")
break
port.dependent.status_changed(exhausted=True)
if port.dependent.failed and not port.dependency.failed:
log.debug(
"DependLoader._find_method()",
"Port '%s': no viable resolve method found" %
(port.origin, ))
return False
else:
self.method[port] = self._next(self.method[port])
if self._resolve(port, method):
log.debug(
"DependLoader._find_method()",
"Port '%s': resolving using method '%s'" %
(port.origin, method))
return True
else:
log.debug(
"DependLoader._find_method()",
"Port '%s': skipping resolve method '%s'" %
(port.origin, method))
def _cleanup(self, stagejob):
"""Cleanup after the port has completed its stage."""
port = stagejob.port
log.debug(
"StageBuilder._cleanup()",
"Port '%s': completed job for stage %s" %
(stagejob.port.origin, self.stage.name))
failed = stagejob.stack.failed or env.flags["mode"] == "clean"
del self.ports[port]
if port in self.cleanup and not env.flags["mode"] == "clean":
self.cleanup.remove(port)
if not failed:
self.done.append(port)
self.update.emit(self, Builder.DONE, port)
if env.flags["target"][-1] == "clean":
queue.clean.add(job.CleanJob(port))
elif not failed:
self.succeeded.append(port)
self.update.emit(self, Builder.SUCCEEDED, port)
if failed:
self.update.emit(self, Builder.FAILED, port)
def work(self):
assert not self.stack.working
assert not self.failed
log.debug(
"Stage.work()",
"Port '%s': starting stage %s" % (self.port.origin, self.name))
if not self.check(self.port):
# Cannot call self._finalise(True) directly as self.done() cannot
# be called from within the scope of self.work()
event.post_event(self._finalise, False)
return
assert self.prev in self.port.stages
assert (not self.port.dependency
or not self.port.dependency.check(self.__class__))
if self.complete():
# Cannot call self._finalise(True) directly as self.done() cannot
# be called from within the scope of self.work()
event.post_event(self._finalise, True)
else:
self._do_stage() # May throw job.JobStalled()
self.stack.working = time.time()
def _cleanup(self, job):
"""Cleanup after the port has completed its stage."""
from .env import flags
log.debug("StageBuilder._cleanup()",
"Port '%s': completed job for stage %d" %
(job.port.origin, self.stage))
del self.ports[job.port]
failed = self._port_failed(job.port)
if job.port in self.cleanup and not flags["mode"] == "clean":
self.cleanup.remove(job.port)
if not failed:
self.done.append(job.port)
self.update.emit(self, Builder.DONE, job.port)
if env.flags["target"][-1] == "clean":
queue.clean.add(CleanJob(job.port))
elif not failed:
self.succeeded.append(job.port)
self.update.emit(self, Builder.SUCCEEDED, job.port)
if failed:
self.update.emit(self, Builder.FAILED, job.port)
def _find_method(self, port):
"""Find a method to resolve the port."""
while True:
method = self.method[port]
if not method:
# No method left, port failed to resolve
del self.method[port]
port.failed = True
port.dependent.status_changed()
log.debug("DependLoader._find_method()",
"Port '%s': no viable resolve method found" % (port,))
return False
else:
self.method[port] = self._next(self.method[port])
port.dependent.propogate = not self.method[port]
if self._resolve(port, method):
log.debug("DependLoader._find_method()",
"Port '%s': resolving using method '%s'" %
(port, method))
return True
else:
log.debug("DependLoader._find_method()",
"Port '%s': skipping resolve method '%s'" %
(port, method))