def start_jobs(self, overrun=0):
"""Start waiting jobs"""
running_threads = self.count_running_jobs()
max_threads = max(overrun, self.cfg.getint('build', 'max_threads'))
jobs_started = 0
with self.jobs_locker:
for job in self.jobs:
if running_threads >= max_threads:
break
with job.status_lock:
if job.status == JobStatus.WAIT_LOCKED and not job.isAlive():
RebuilddLog.info("Starting new thread for job %s" % job.id)
job.notify = self.job_finished
job.setDaemon(True)
job.start()
jobs_started += 1
running_threads = running_threads + 1
RebuilddLog.info("Running threads: [ build %s/%s ] [ real %s ]" %
(running_threads, max_threads, threading.activeCount()))
return jobs_started
def get_build_cmd(self):
"""Return command used for building source for this distribution
Substitutions are done in the command strings:
$d => The distro's name
$a => the target architecture
$p => the package's name
$v => the package's version
$j => rebuildd job id
"""
# Strip epochs (x:) away
try:
index = self.package.version.index(":")
args = { 'd': self.dist, 'a': self.arch, \
'v': self.package.version[index+1:], 'p': self.package.name, \
'j': str(self.id) }
t = Template(RebuilddConfig().get('build', 'build_cmd'))
return t.safe_substitute(**args)
except ValueError:
pass
try:
args = { 'd': self.dist, 'a': self.arch, \
'v': self.package.version, 'p': self.package.name, \
'j': str(self.id) }
t = Template(RebuilddConfig().get('build', 'build_cmd'))
return t.safe_substitute(**args)
except TypeError, error:
RebuilddLog.error("get_build_cmd has invalid format: %s" % error)
return None
def start_jobs(self, overrun=0):
"""Start waiting jobs"""
running_threads = self.count_running_jobs()
max_threads = max(overrun, self.cfg.getint('build', 'max_threads'))
jobs_started = 0
with self.jobs_locker:
for job in self.jobs:
if running_threads >= max_threads:
break
with job.status_lock:
if job.status == JobStatus.WAIT_LOCKED and not job.isAlive(
):
RebuilddLog.info("Starting new thread for job %s" %
job.id)
job.notify = self.job_finished
job.setDaemon(True)
job.start()
jobs_started += 1
running_threads = running_threads + 1
RebuilddLog.info(
"Running threads: [ build %s/%s ] [ real %s ]" %
(running_threads, max_threads, threading.activeCount()))
return jobs_started
def add_dep(self, dep):
"""Add a job dependency on another job"""
for existing_dep in self.deps:
if existing_dep.id == dep.id:
RebuilddLog.error("Already existing dependency between job %s and job %s" % (self.id, dep.id))
return
self.addJob(dep)
RebuilddLog.info("Dependency added between job %s and job %s" % (self.id, dep.id))
def send_build_log(self):
"""When job is built, send logs by mail"""
try:
with open(self.logfile, "r") as build_log:
log = build_log.read()
except IOError, error:
RebuilddLog.error("Unable to open logfile for job %d" % self.id)
return False
def send_build_log(self):
"""When job is built, send logs by mail"""
try:
with open(self.logfile, "r") as build_log:
log = build_log.read()
except IOError, error:
RebuilddLog.error("Unable to open logfile for job %d" % self.id)
return False
def add_dep(self, dep):
"""Add a job dependency on another job"""
for existing_dep in self.deps:
if existing_dep.id == dep.id:
RebuilddLog.error("Already existing dependency between job %s and job %s" % (self.id, dep.id))
return
self.addJob(dep)
RebuilddLog.info("Dependency added between job %s and job %s" % (self.id, dep.id))
def __setattr__(self, name, value):
"""Override setattr to log build status changes"""
if name == "status":
RebuilddLog.info("Job %s for %s_%s on %s/%s changed status from %s to %s"\
% (self.id, self.package.name, self.package.version,
self.dist, self.arch,
JobStatus.whatis(self.status),
JobStatus.whatis(value)))
self.status_changed = sqlobject.DateTimeCol.now()
sqlobject.SQLObject.__setattr__(self, name, value)
def __setattr__(self, name, value):
"""Override setattr to log build status changes"""
if name == "status":
RebuilddLog.info("Job %s for %s_%s on %s/%s changed status from %s to %s"\
% (self.id, self.package.name, self.package.version,
self.dist, self.arch,
JobStatus.whatis(self.status),
JobStatus.whatis(value)))
self.status_changed = sqlobject.DateTimeCol.now()
sqlobject.SQLObject.__setattr__(self, name, value)
def get_source_cmd(self, package):
"""Return command used for grabing source for this distribution"""
try:
args = { 'd': self.name, 'a': self.arch, 'v': package.version, \
'p': package.name, 'r': package.repo }
t = Template(RebuilddConfig().get('build', 'source_cmd'))
return t.safe_substitute(**args)
except TypeError, error:
RebuilddLog.error("get_source_cmd has invalid format: %s" % error)
return None
def requeue_job(self, job_id):
"""Requeue a failed job"""
if Job.selectBy(id=job_id).count() == 0:
RebuilddLog.error("There is no job related to %s that is in the job list" % job_id)
return False
job = Job.selectBy(id=job_id)[0]
if job.status in FailedStatus:
job.status = JobStatus.WAIT
job.host = ""
return True
def get_post_build_cmd(self, package):
"""Return command used after building source for this distribution"""
cmd = RebuilddConfig().get('build', 'post_build_cmd')
if cmd == '':
return None
try:
args = { 'd': self.name, 'a': self.arch, \
'v': package.version, 'p': package.name, 'r': package.repo }
t = Template(cmd)
return t.safe_substitute(**args)
except TypeError, error:
RebuilddLog.error("post_build_cmd has invalid format: %s" % error)
return None
def stop_all_jobs(self):
"""Stop all running jobs"""
with self.jobs_locker:
for job in self.jobs:
if job.status == JobStatus.BUILDING and job.isAlive():
job.do_quit.set()
RebuilddLog.info("Sending stop to job %s" % job.id)
for job in self.jobs:
if job.isAlive():
RebuilddLog.info("Waiting for job %s to terminate" % job.id)
job.join(60)
return True
def requeue_job(self, job_id):
"""Requeue a failed job"""
if Job.selectBy(id=job_id).count() == 0:
RebuilddLog.error(
"There is no job related to %s that is in the job list" %
job_id)
return False
job = Job.selectBy(id=job_id)[0]
if job.status in FailedStatus:
job.status = JobStatus.WAIT
job.host = ""
return True
def stop_all_jobs(self):
"""Stop all running jobs"""
with self.jobs_locker:
for job in self.jobs:
if job.status == JobStatus.BUILDING and job.isAlive():
job.do_quit.set()
RebuilddLog.info("Sending stop to job %s" % job.id)
for job in self.jobs:
if job.isAlive():
RebuilddLog.info("Waiting for job %s to terminate" %
job.id)
job.join(60)
return True
def add_deps(self, job_id, dependency_ids):
if Job.selectBy(id=job_id).count() == 0:
RebuilddLog.error("There is no job related to %s that is in the job list" % job_id)
return False
job = Job.selectBy(id=job_id)[0]
deps = []
for dep in dependency_ids:
if Job.selectBy(id=dep).count() == 0:
RebuilddLog.error("There is no job related to %s that is in the job list" % dep)
return False
dep_job = Job.selectBy(id=dep)[0]
deps.append(dep_job)
job.add_deps(deps)
return True
def cancel_job(self, jobid):
"""Cancel a job"""
with self.jobs_locker:
job = self.get_job(jobid)
if job != None:
if job.isAlive():
job.do_quit.set()
job.join()
with job.status_lock:
job.status = JobStatus.CANCELED
self.jobs.remove(job)
RebuilddLog.info("Canceled job %s for %s_%s on %s/%s for %s" \
% (job.id, job.package.name, job.package.version,
job.dist, job.arch, job.mailto))
return True
return False
def cancel_job(self, jobid):
"""Cancel a job"""
with self.jobs_locker:
job = self.get_job(jobid)
if job != None:
if job.isAlive():
job.do_quit.set()
job.join()
with job.status_lock:
job.status = JobStatus.CANCELED
self.jobs.remove(job)
RebuilddLog.info("Canceled job %s for %s_%s on %s/%s for %s" \
% (job.id, job.package.name, job.package.version,
job.dist, job.arch, job.mailto))
return True
return False
def add_deps(self, job_id, dependency_ids):
if Job.selectBy(id=job_id).count() == 0:
RebuilddLog.error(
"There is no job related to %s that is in the job list" %
job_id)
return False
job = Job.selectBy(id=job_id)[0]
deps = []
for dep in dependency_ids:
if Job.selectBy(id=dep).count() == 0:
RebuilddLog.error(
"There is no job related to %s that is in the job list" %
dep)
return False
dep_job = Job.selectBy(id=dep)[0]
deps.append(dep_job)
job.add_deps(deps)
return True
def get_build_cmd(self, package):
"""Return command used for building source for this distribution"""
# Strip epochs (x:) away
try:
index = package.version.index(":")
args = { 'd': self.name, 'a': self.arch, \
'v': package.version[index + 1:], 'p': package.name, 'r': package.repo }
t = Template(RebuilddConfig().get('build', 'build_cmd'))
return t.safe_substitute(**args)
except ValueError:
pass
try:
args = { 'd': self.name, 'a': self.arch, \
'v': package.version, 'p': package.name, 'r': package.repo }
t = Template(RebuilddConfig().get('build', 'build_cmd'))
return t.safe_substitute(**args)
except TypeError, error:
RebuilddLog.error("get_build_cmd has invalid format: %s" % error)
return None
def add_job(self, name, version, priority, dist, mailto=None, arch=None):
"""Add a job"""
if not arch:
arch = self.cfg.arch[0]
if not Dists().get_dist(dist, arch):
RebuilddLog.error("Couldn't find dist/arch in the config file for %s_%s on %s/%s, don't adding it" \
% (name, version, dist, arch))
return False
pkgs = Package.selectBy(name=name, version=version)
if pkgs.count():
# If several packages exists, just take the first
pkg = pkgs[0]
else:
# Maybe we found no packages, so create a brand new one!
pkg = Package(name=name, version=version, priority=priority)
jobs_count = Job.selectBy(package=pkg, dist=dist, arch=arch, mailto=mailto, status=JobStatus.WAIT).count()
if jobs_count:
RebuilddLog.error("Job already existing for %s_%s on %s/%s, don't adding it" \
% (pkg.name, pkg.version, dist, arch))
return False
job = Job(package=pkg, dist=dist, arch=arch)
job.status = JobStatus.WAIT
job.arch = arch
job.mailto = mailto
log = Log(job=job)
RebuilddLog.info("Added job for %s_%s on %s/%s for %s" \
% (name, version, dist, arch, mailto))
return True
def get_post_build_cmd(self):
"""Return command used after building source for this distribution
Substitutions are done in the command strings:
$d => The distro's name
$a => the target architecture
$p => the package's name
$v => the package's version
$j => rebuildd job id
"""
cmd = RebuilddConfig().get('build', 'post_build_cmd')
if cmd == '':
return None
try:
args = { 'd': self.dist, 'a': self.arch, \
'v': self.package.version, 'p': self.package.name, \
'j': str(self.id) }
t = Template(cmd)
return t.safe_substitute(**args)
except TypeError, error:
RebuilddLog.error("post_build_cmd has invalid format: %s" % error)
return None
def init(self):
self.cfg = RebuilddConfig()
# Init log system
RebuilddLog()
self._sqlconnection = sqlobject.connectionForURI(
self.cfg.get('build', 'database_uri'))
sqlobject.sqlhub.processConnection = self._sqlconnection
# Create distributions
for dist in self.cfg.get('build', 'dists').split(' '):
for arch in self.cfg.arch:
Dists().add_dist(Distribution(dist, arch))
self.do_quit = threading.Event()
self.jobs_locker = threading.Lock()
self.job_finished = threading.Event()
def add_job(self, name, version, priority, dist, mailto=None, arch=None):
"""Add a job"""
if not arch:
arch = self.cfg.arch[0]
if not Dists().get_dist(dist, arch):
RebuilddLog.error("Couldn't find dist/arch in the config file for %s_%s on %s/%s, not adding it" \
% (name, version, dist, arch))
return False
pkgs = Package.selectBy(name=name, version=version)
if pkgs.count():
# If several packages exists, just take the first
pkg = pkgs[0]
else:
# Maybe we found no packages, so create a brand new one!
pkg = Package(name=name, version=version, priority=priority)
jobs_count = Job.selectBy(package=pkg,
dist=dist,
arch=arch,
mailto=mailto,
status=JobStatus.WAIT).count()
if jobs_count:
RebuilddLog.error("Job already existing for %s_%s on %s/%s, not adding it" \
% (pkg.name, pkg.version, dist, arch))
return False
job = Job(package=pkg, dist=dist, arch=arch)
job.status = JobStatus.WAIT
job.arch = arch
job.mailto = mailto
log = Log(job=job)
RebuilddLog.info("Added job for %s_%s on %s/%s for %s" \
% (name, version, dist, arch, mailto))
return True
state = 1
break
state = proc.poll()
while not self.do_quit.isSet() and state == None:
state = proc.poll()
self.do_quit.wait(1)
if self.do_quit.isSet():
break
if state != 0:
with self.status_lock:
self.status = failed_status
break
if self.do_quit.isSet():
# Kill gently the process
RebuilddLog.info("Killing job %s with SIGINT" % self.id)
try:
os.killpg(os.getpgid(proc.pid), signal.SIGINT)
except OSError, error:
RebuilddLog.error("Error killing job %s: %s" % (self.id, error))
# If after 60s it's not dead, KILL HIM
counter = 0
timemax = RebuilddConfig().get('build', 'kill_timeout')
while proc.poll() == None and counter < timemax:
time.sleep(1)
counter += 1
if proc.poll() == None:
RebuilddLog.error("Killing job %s timed out, killing with SIGKILL" \
% self.id)
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
msg.set_payload(log)
try:
smtp = smtplib.SMTP()
smtp.connect(RebuilddConfig().get('mail', 'smtp_host'),
RebuilddConfig().get('mail', 'smtp_port'))
smtp.sendmail(RebuilddConfig().get('mail', 'from'),
[m.strip() for m in msg['To'].split(",")],
msg.as_string())
except Exception, error:
try:
process = Popen("sendmail", shell=True, stdin=PIPE)
process.communicate(input=msg.as_string())
except:
RebuilddLog.error("Unable to send build log mail for job %d: %s" % (self.id, error))
return True
def __str__(self):
return "I: Job %s for %s_%s is status %s on %s for %s/%s" % \
(self.id, self.package.name, self.package.version, self.host,
JobStatus.whatis(self.status), self.dist, self.arch)
def is_allowed_to_build(self):
""" Check if job is allowed to build """
for dep in Job.selectBy(id=self)[0].deps:
if Job.selectBy(id=dep)[0].status != JobStatus.BUILD_OK:
return False
return True
def daemon(self):
RebuilddLog.info("Starting rebuildd %s" % __version__)
self.daemonize()
# Run the network server thread
RebuilddLog.info("Launching network server")
self.netserv = RebuilddNetworkServer(self)
self.netserv.setDaemon(True)
self.netserv.start()
# Run main loop
RebuilddLog.info("Running main loop")
self.loop()
# On exit
RebuilddLog.info("Cleaning finished and canceled jobs")
self.clean_jobs()
RebuilddLog.info("Stopping all jobs")
self.stop_all_jobs()
RebuilddLog.info("Releasing wait-locked jobs")
self.release_jobs()
self.netserv.join(10)
RebuilddLog.info("Exiting rebuildd")
def handle_sigterm(self, signum, stack):
RebuilddLog.info("Receiving transmission... it's a signal %s capt'ain! EVERYONE OUT!" % signum)
self.do_quit.set()
def handle_sigterm(self, signum, stack):
RebuilddLog.info(
"Receiving transmission... it's a signal %s capt'ain! EVERYONE OUT!"
% signum)
self.do_quit.set()
def daemon(self):
RebuilddLog.info("Starting rebuildd %s" % __version__)
self.daemonize()
# Run the network server thread
RebuilddLog.info("Launching network server")
self.netserv = RebuilddNetworkServer(self)
self.netserv.setDaemon(True)
self.netserv.start()
# Run main loop
RebuilddLog.info("Running main loop")
self.loop()
# On exit
RebuilddLog.info("Cleaning finished and canceled jobs")
self.clean_jobs()
RebuilddLog.info("Stopping all jobs")
self.stop_all_jobs()
RebuilddLog.info("Releasing wait-locked jobs")
self.release_jobs()
self.netserv.join(10)
RebuilddLog.info("Exiting rebuildd")
