def handle_delivery(self, channel, method, header, body):
"""Message received callback."""
global producer
message = pickle.loads(body)
p = message["package"]
arch = message["arch"]
branch = message["branch"]
buildid = message["buildid"]
torrents = message["torrents"]
reply_to = message["reply_to"]
self.buildname = "{0}/{1}/{2}".format(arch, branch, buildid)
if buildid in ready:
util.log("Building {0} for build {1}".format(p.name, self.buildname))
channel.basic_ack(delivery_tag=method.delivery_tag)
else:
time.sleep(5)
channel.basic_reject(delivery_tag=method.delivery_tag)
util.log("Setting up build {0}".format(self.buildname))
ts = torrent.TorrentSession()
for t in torrents:
t.dest="." # XXX - Temporary while I'm testing locally.
ts.add(t)
while not ts.all_seeding():
ts.status()
time.sleep(1)
util.debug("Finished downloading components.")
ts.terminate()
producer.notify(reply_to)
ready.append(buildid)
def makerestr(self):
"""Create restricted.sh file."""
environ = { }
environ.update(os.environ)
environ.update(self.buildenv)
environ["__MAKE_SHELL"] = "/rescue/sh"
environ["LOCALBASE"] = "/nonexistentlocal"
environ["LINUXBASE"] = "/nonexistentlinux"
environ["PKG_DBDIR"] = "/nonexistentpkg"
environ["PORT_DBDIR"] = "/nonexistentport"
error = None
cmdf = "{0}/scripts/makerestr {1} {2} {3} {4}"
cmd = cmdf.format(pbc, self.arch, self.branch, self.buildid, \
self.subsetfile if self.subsetfile else "")
util.log("Creating restricted.sh file...")
try:
f = util.shell_cmd(cmd, cwd=self.portsdir)
if f.returncode != 0:
error = "Failed to build restricted.sh file."
except KeyboardInterrupt:
error = "Failed to build restricted.sh file."
if error:
raise BuildMissingMetadata(error)
def seed_start(self):
"""Start seeding the build tarballs."""
self.ts = torrent.TorrentSession()
util.log("Start seeding...")
for t in self.tarballs:
t.torrent = torrent.Torrent.create(t.realpath)
t.torrent.dump()
self.ts.add(t.torrent)
def notify(self, queue):
"""Send notification back to the head node."""
message = "Finished" # FIXME
util.log("Sending notification on {0}".format(queue))
props = BasicProperties(content_type="text/plain", delivery_mode=1)
self.channel.basic_publish(exchange="",
routing_key=queue,
mandatory=True,
immediate=True,
body=pickle.dumps(message),
properties=props)
def create(builddir, component):
"""Create a new tarball."""
cachedir = os.path.join(pbd, "tarballs")
if os.path.isdir(cachedir):
destdir = cachedir
else:
destdir = builddir
prefix = component + "-"
(f, tmp) = tempfile.mkstemp(dir=destdir, prefix=prefix, suffix=".tbz")
util.log("Creating {0} tarball...".format(component))
cmd = "/usr/bin/tar -C {0} -cjf {1} {2}".format(builddir, tmp, component)
try:
util.shell_cmd(cmd)
except KeyboardInterrupt:
print "Cleaning up temporary tarball..."
os.unlink(tmp)
raise
else:
return Tarball(builddir, component, tmp)
def setup(self):
changed = False
util.log("Setting up build...")
self.buildenv()
# Load the configuration files.
try:
self.config = config.Config(self.arch, self.branch)
except config.InvalidConfig as e:
raise BuildInvalidConfig(e)
try:
self.__setup_bindist()
changed |= self.__setup_ports()
changed |= self.__setup_src()
except BuildMissingComponent:
raise
return changed
def __setup_src(self):
"""Create new src tarball and Tarball object."""
changed = False
st_orig = None
try:
st_orig = tarball.SrcTarball(self.builddir)
except:
pass
try:
st = tarball.SrcTarball.create(self.builddir)
except KeyboardInterrupt:
error = "Src tarball creation was interrupted"
raise BuildMissingComponent(error)
if not st == st_orig:
changed = True
st.promote()
self.tarballs.append(st)
else:
util.log("Src tarball unchanged.")
self.tarballs.append(st_orig)
st.delete()
return changed
def __setup_ports(self):
"""Create new ports tarball and Tarball object."""
changed = False
pt_orig = None
try:
pt_orig = tarball.PortsTarball(self.builddir)
except:
pass
try:
pt = tarball.PortsTarball.create(self.builddir)
except KeyboardInterrupt:
error = "Ports tarball creation was interrupted"
raise BuildMissingComponent(error)
if not pt == pt_orig:
changed = True
pt.promote()
self.tarballs.append(pt)
else:
util.log("Ports tarball unchanged.")
self.tarballs.append(pt_orig)
pt.delete()
return changed
def makeindex(self):
"""Create INDEX file."""
# Set a few environment variables. Add whatever is in buildenv and then
# makeindex specific variables.
environ = { }
environ.update(os.environ)
environ.update(self.buildenv)
environ["INDEXDIR"] = self.builddir
environ["INDEX_PRISTINE"] = "1"
environ["INDEX_QUIET"] = "1"
environ["INDEX_JOBS"] = "6"
environ["LOCALBASE"] = "/nonexistentlocal"
error = None
cmdf = "{0}/scripts/makeindex {1} {2} {3} {4}"
cmd = cmdf.format(pbc, self.arch, self.branch, self.buildid, \
self.subsetfile if self.subsetfile else "")
util.log("Creating INDEX file...")
try:
f = util.pipe_cmd(cmd, env=environ, cwd=self.portsdir)
index = os.path.basename(self.indexfile)
success = "^Generating %s - please wait.. Done.$".format(index)
for i in f.stdout.readlines():
if re.match(success, i.rstrip()):
break
else:
print i.rstrip()
f.wait()
except KeyboardInterrupt:
pass
if f.returncode != 0:
error = "Failed to build INDEX file."
if error:
raise BuildMissingMetadata(error)
def promote(self, dest=None):
"""Put the tarball where it should be."""
# dest might be used as a compatibility shim.
if dest == None:
dest = os.path.join(self.builddir, self.component + ".tbz")
cachedir = os.path.join(pbd, "tarballs")
if os.path.isdir(cachedir):
fullname = "{0}-{1}.tbz".format(self.component, self.checksum[0:16])
self.realpath = os.path.join(cachedir, fullname)
if not os.path.exists(self.realpath):
util.log("Tarball cached as {0}".format(self.realpath))
shutil.move(self.path, self.realpath)
else:
os.unlink(self.path)
self.path = dest
if os.path.lexists(self.path):
os.unlink(self.path)
os.symlink(self.realpath, self.path)
else:
if self.path != dest:
shutil.move(self.path, dest)
self.path = dest
self.realpath = dest
def dispatch(self, package):
util.log("Dispatching {0}...".format(package.name))
self.producer.dispatch(package)
def finish(self):
self.seed_stop()
self.end_time = time.time()
util.log("Build ended after %d seconds." %
(self.end_time - self.start_time))
def seed_stop(self):
"""Stop seeding the build tarballs."""
util.log("Stop seeding...")
self.ts.terminate()