def start_jobs(self, overrun=0):
"""Start waiting jobs"""
running_threads = self.count_running_jobs()
max_threads = max(overrun, self.cfg.getint('build', 'max_threads'))
jobs_started = 0
with self.jobs_locker:
for job in self.jobs:
if running_threads >= max_threads:
break
with job.status_lock:
if job.status == JobStatus.WAIT_LOCKED and not job.isAlive():
RebuilddLog.info("Starting new thread for job %s" % job.id)
job.notify = self.job_finished
job.setDaemon(True)
job.start()
jobs_started += 1
running_threads = running_threads + 1
RebuilddLog.info("Running threads: [ build %s/%s ] [ real %s ]" %
(running_threads, max_threads, threading.activeCount()))
return jobs_started
def add_job(self, name, version, priority, dist, mailto=None, arch=None):
"""Add a job"""
if not arch:
arch = self.cfg.arch[0]
if not Dists().get_dist(dist, arch):
RebuilddLog.error("Couldn't find dist/arch in the config file for %s_%s on %s/%s, don't adding it" \
% (name, version, dist, arch))
return False
pkgs = Package.selectBy(name=name, version=version)
if pkgs.count():
# If several packages exists, just take the first
pkg = pkgs[0]
else:
# Maybe we found no packages, so create a brand new one!
pkg = Package(name=name, version=version, priority=priority)
jobs_count = Job.selectBy(package=pkg, dist=dist, arch=arch, mailto=mailto, status=JobStatus.WAIT).count()
if jobs_count:
RebuilddLog.error("Job already existing for %s_%s on %s/%s, don't adding it" \
% (pkg.name, pkg.version, dist, arch))
return False
job = Job(package=pkg, dist=dist, arch=arch)
job.status = JobStatus.WAIT
job.arch = arch
job.mailto = mailto
log = Log(job=job)
RebuilddLog.info("Added job for %s_%s on %s/%s for %s" \
% (name, version, dist, arch, mailto))
return True
def start_jobs(self, overrun=0):
"""Start waiting jobs"""
running_threads = self.count_running_jobs()
max_threads = max(overrun, self.cfg.getint('build', 'max_threads'))
jobs_started = 0
with self.jobs_locker:
for job in self.jobs:
if running_threads >= max_threads:
break
with job.status_lock:
if job.status == JobStatus.WAIT_LOCKED and not job.isAlive(
):
RebuilddLog.info("Starting new thread for job %s" %
job.id)
job.notify = self.job_finished
job.setDaemon(True)
job.start()
jobs_started += 1
running_threads = running_threads + 1
RebuilddLog.info(
"Running threads: [ build %s/%s ] [ real %s ]" %
(running_threads, max_threads, threading.activeCount()))
return jobs_started
def add_dep(self, dep):
"""Add a job dependency on another job"""
for existing_dep in self.deps:
if existing_dep.id == dep.id:
RebuilddLog.error("Already existing dependency between job %s and job %s" % (self.id, dep.id))
return
self.addJob(dep)
RebuilddLog.info("Dependency added between job %s and job %s" % (self.id, dep.id))
def add_dep(self, dep):
"""Add a job dependency on another job"""
for existing_dep in self.deps:
if existing_dep.id == dep.id:
RebuilddLog.error("Already existing dependency between job %s and job %s" % (self.id, dep.id))
return
self.addJob(dep)
RebuilddLog.info("Dependency added between job %s and job %s" % (self.id, dep.id))
def __setattr__(self, name, value):
"""Override setattr to log build status changes"""
if name == "status":
RebuilddLog.info("Job %s for %s_%s on %s/%s changed status from %s to %s"\
% (self.id, self.package.name, self.package.version,
self.dist, self.arch,
JobStatus.whatis(self.status),
JobStatus.whatis(value)))
self.status_changed = sqlobject.DateTimeCol.now()
sqlobject.SQLObject.__setattr__(self, name, value)
def __setattr__(self, name, value):
"""Override setattr to log build status changes"""
if name == "status":
RebuilddLog.info("Job %s for %s_%s on %s/%s changed status from %s to %s"\
% (self.id, self.package.name, self.package.version,
self.dist, self.arch,
JobStatus.whatis(self.status),
JobStatus.whatis(value)))
self.status_changed = sqlobject.DateTimeCol.now()
sqlobject.SQLObject.__setattr__(self, name, value)
def stop_all_jobs(self):
"""Stop all running jobs"""
with self.jobs_locker:
for job in self.jobs:
if job.status == JobStatus.BUILDING and job.isAlive():
job.do_quit.set()
RebuilddLog.info("Sending stop to job %s" % job.id)
for job in self.jobs:
if job.isAlive():
RebuilddLog.info("Waiting for job %s to terminate" % job.id)
job.join(60)
return True
def stop_all_jobs(self):
"""Stop all running jobs"""
with self.jobs_locker:
for job in self.jobs:
if job.status == JobStatus.BUILDING and job.isAlive():
job.do_quit.set()
RebuilddLog.info("Sending stop to job %s" % job.id)
for job in self.jobs:
if job.isAlive():
RebuilddLog.info("Waiting for job %s to terminate" %
job.id)
job.join(60)
return True
def cancel_job(self, jobid):
"""Cancel a job"""
with self.jobs_locker:
job = self.get_job(jobid)
if job != None:
if job.isAlive():
job.do_quit.set()
job.join()
with job.status_lock:
job.status = JobStatus.CANCELED
self.jobs.remove(job)
RebuilddLog.info("Canceled job %s for %s_%s on %s/%s for %s" \
% (job.id, job.package.name, job.package.version,
job.dist, job.arch, job.mailto))
return True
return False
def cancel_job(self, jobid):
"""Cancel a job"""
with self.jobs_locker:
job = self.get_job(jobid)
if job != None:
if job.isAlive():
job.do_quit.set()
job.join()
with job.status_lock:
job.status = JobStatus.CANCELED
self.jobs.remove(job)
RebuilddLog.info("Canceled job %s for %s_%s on %s/%s for %s" \
% (job.id, job.package.name, job.package.version,
job.dist, job.arch, job.mailto))
return True
return False
def add_job(self, name, version, priority, dist, mailto=None, arch=None):
"""Add a job"""
if not arch:
arch = self.cfg.arch[0]
if not Dists().get_dist(dist, arch):
RebuilddLog.error("Couldn't find dist/arch in the config file for %s_%s on %s/%s, not adding it" \
% (name, version, dist, arch))
return False
pkgs = Package.selectBy(name=name, version=version)
if pkgs.count():
# If several packages exists, just take the first
pkg = pkgs[0]
else:
# Maybe we found no packages, so create a brand new one!
pkg = Package(name=name, version=version, priority=priority)
jobs_count = Job.selectBy(package=pkg,
dist=dist,
arch=arch,
mailto=mailto,
status=JobStatus.WAIT).count()
if jobs_count:
RebuilddLog.error("Job already existing for %s_%s on %s/%s, not adding it" \
% (pkg.name, pkg.version, dist, arch))
return False
job = Job(package=pkg, dist=dist, arch=arch)
job.status = JobStatus.WAIT
job.arch = arch
job.mailto = mailto
log = Log(job=job)
RebuilddLog.info("Added job for %s_%s on %s/%s for %s" \
% (name, version, dist, arch, mailto))
return True
def daemon(self):
RebuilddLog.info("Starting rebuildd %s" % __version__)
self.daemonize()
# Run the network server thread
RebuilddLog.info("Launching network server")
self.netserv = RebuilddNetworkServer(self)
self.netserv.setDaemon(True)
self.netserv.start()
# Run main loop
RebuilddLog.info("Running main loop")
self.loop()
# On exit
RebuilddLog.info("Cleaning finished and canceled jobs")
self.clean_jobs()
RebuilddLog.info("Stopping all jobs")
self.stop_all_jobs()
RebuilddLog.info("Releasing wait-locked jobs")
self.release_jobs()
self.netserv.join(10)
RebuilddLog.info("Exiting rebuildd")
def handle_sigterm(self, signum, stack):
RebuilddLog.info("Receiving transmission... it's a signal %s capt'ain! EVERYONE OUT!" % signum)
self.do_quit.set()
def handle_sigterm(self, signum, stack):
RebuilddLog.info(
"Receiving transmission... it's a signal %s capt'ain! EVERYONE OUT!"
% signum)
self.do_quit.set()
def daemon(self):
RebuilddLog.info("Starting rebuildd %s" % __version__)
self.daemonize()
# Run the network server thread
RebuilddLog.info("Launching network server")
self.netserv = RebuilddNetworkServer(self)
self.netserv.setDaemon(True)
self.netserv.start()
# Run main loop
RebuilddLog.info("Running main loop")
self.loop()
# On exit
RebuilddLog.info("Cleaning finished and canceled jobs")
self.clean_jobs()
RebuilddLog.info("Stopping all jobs")
self.stop_all_jobs()
RebuilddLog.info("Releasing wait-locked jobs")
self.release_jobs()
self.netserv.join(10)
RebuilddLog.info("Exiting rebuildd")
state = 1
break
state = proc.poll()
while not self.do_quit.isSet() and state == None:
state = proc.poll()
self.do_quit.wait(1)
if self.do_quit.isSet():
break
if state != 0:
with self.status_lock:
self.status = failed_status
break
if self.do_quit.isSet():
# Kill gently the process
RebuilddLog.info("Killing job %s with SIGINT" % self.id)
try:
os.killpg(os.getpgid(proc.pid), signal.SIGINT)
except OSError, error:
RebuilddLog.error("Error killing job %s: %s" % (self.id, error))
# If after 60s it's not dead, KILL HIM
counter = 0
timemax = RebuilddConfig().get('build', 'kill_timeout')
while proc.poll() == None and counter < timemax:
time.sleep(1)
counter += 1
if proc.poll() == None:
RebuilddLog.error("Killing job %s timed out, killing with SIGKILL" \
% self.id)
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
