def init(self):
cfg = RebuilddConfig()
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s',
filename=cfg.get('log', 'file'),
datefmt=cfg.get('log', 'time_format'),
filemode='a')
def GET(self, name=None, version=None):
jobs = []
if version:
title = "%s %s" % (name, version)
package = "%s/%s" % (name, version)
query = Job.select(
IN(
Job.q.package,
Select(
Package.q.id,
AND(Package.q.name == name,
Package.q.version == version))))
else:
title = package = name
query = Job.select(
IN(Job.q.package, Select(Package.q.id,
Package.q.name == name)))
result, page, nb_pages = _paginate_query(
query.orderBy(DESC(Job.q.package)))
jobs.extend(result)
return render.base(page=render.tab(jobs=jobs, page=page, nb_pages=nb_pages), \
hostname=socket.gethostname(), \
title=title, \
package=package, \
archs=RebuilddConfig().arch, \
dists=RebuilddConfig().get('build', 'dists').split(' '))
def get_build_cmd(self):
"""Return command used for building source for this distribution
Substitutions are done in the command strings:
$d => The distro's name
$a => the target architecture
$p => the package's name
$v => the package's version
$j => rebuildd job id
"""
# Strip epochs (x:) away
try:
index = self.package.version.index(":")
args = { 'd': self.dist, 'a': self.arch, \
'v': self.package.version[index+1:], 'p': self.package.name, \
'j': str(self.id) }
t = Template(RebuilddConfig().get('build', 'build_cmd'))
return t.safe_substitute(**args)
except ValueError:
pass
try:
args = { 'd': self.dist, 'a': self.arch, \
'v': self.package.version, 'p': self.package.name, \
'j': str(self.id) }
t = Template(RebuilddConfig().get('build', 'build_cmd'))
return t.safe_substitute(**args)
except TypeError, error:
RebuilddLog.error("get_build_cmd has invalid format: %s" % error)
return None
def init(self):
cfg = RebuilddConfig()
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s',
filename=cfg.get('log', 'file'),
datefmt=cfg.get('log', 'time_format'),
filemode='a')
def GET(self):
jobs = []
query = Job.select(orderBy=DESC(Job.q.creation_date))
result, page, nb_pages = _paginate_query(query)
jobs.extend(result)
return render.base(\
page=render.tab(jobs=jobs, page=page, nb_pages=nb_pages), \
title="all builds", \
hostname=socket.gethostname(), \
archs=RebuilddConfig().arch, \
dists=RebuilddConfig().get('build', 'dists').split(' '))
def start(self):
"""Run main HTTP server thread"""
web.webapi.internalerror = web.debugerror
import sys
sys.argv.append(RebuilddConfig().get('http', 'ip') + ":" +
RebuilddConfig().get('http', 'port'))
app = web.application(self.urls, globals())
app.run()
class RequestJob:
def GET(self, jobid=None):
job = Job.selectBy(id=jobid)[0]
try:
with open(job.logfile, "r") as build_logfile:
build_log = build_logfile.read()
except IOError, error:
build_log = job.log.text
return render.base(page=render.job(job=job, build_log=build_log), \
hostname=socket.gethostname(), \
title="job %s" % job.id, \
archs=RebuilddConfig().arch, \
dists=RebuilddConfig().get('build', 'dists').split(' '))
def GET(self, dist, arch=None):
jobs = []
result, page, nb_pages = _paginate_query(
Job.select(AND(Job.q.arch == arch, Job.q.dist == dist),
orderBy=DESC(Job.q.creation_date)))
jobs.extend(result)
return render.base(page=render.tab(jobs=jobs, page=page, nb_pages=nb_pages), \
arch=arch, \
dist=dist, \
title="%s/%s" % (dist, arch), \
hostname=socket.gethostname(), \
archs=RebuilddConfig().arch, \
dists=RebuilddConfig().get('build', 'dists').split(' '))
def init(self):
self.cfg = RebuilddConfig()
# Init log system
RebuilddLog()
self._sqlconnection = sqlobject.connectionForURI(
self.cfg.get('build', 'database_uri'))
sqlobject.sqlhub.processConnection = self._sqlconnection
# Create distributions
for dist in self.cfg.get('build', 'dists').split(' '):
for arch in self.cfg.arch:
Dists().add_dist(Distribution(dist, arch))
self.do_quit = threading.Event()
self.jobs_locker = threading.Lock()
self.job_finished = threading.Event()
def logfile(self):
"""Compute and return logfile name"""
build_log_file = "%s/%s_%s-%s-%s-%s.%s.log" % (RebuilddConfig().get('log', 'logs_dir'),
self.package.name, self.package.version,
self.dist, self.arch,
self.creation_date.strftime("%Y%m%d-%H%M%S"),
self.id)
return build_log_file
def exec_cmd_config(self, *args):
"""Manipulate configuration file"""
if len(args) < 1:
return "E: usage: config [reload|dump|save]\n"
if args[0] == "reload":
if RebuilddConfig().reload():
return "I: config reloaded\n"
return "E: config not reloded\n"
if args[0] == "dump":
return RebuilddConfig().dump()
if args[0] == "save":
if RebuilddConfig().save():
return "I: config saved\n"
return "E: config not saved\n"
return "E: usage: config [reload|dump|save]\n"
def run(self):
"""Run main network server thread"""
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.settimeout(1)
self.socket.bind((RebuilddConfig().get('telnet', 'ip'),
RebuilddConfig().getint('telnet', 'port')))
self.socket.listen(2)
while not self.rebuildd.do_quit.isSet():
try:
(client_socket, client_info) = self.socket.accept()
if client_socket:
interface = RebuilddNetworkClient(client_socket,
self.rebuildd)
interface.setDaemon(True)
interface.start()
except socket.timeout:
pass
self.socket.close()
def run(self):
"""Run client thread"""
self.socket.settimeout(1)
self.socket.send(RebuilddConfig().get('telnet', 'motd') + "\n")
prompt = RebuilddConfig().get('telnet', 'prompt') + " "
line = ""
has_timeout = False
while line != "exit" and not self.rebuildd.do_quit.isSet():
if not has_timeout:
try:
self.socket.send(self.exec_cmd(line))
self.socket.send(prompt)
except Exception:
break
try:
line = ""
line = self.socket.recv(512).strip()
has_timeout = False
except socket.timeout:
has_timeout = True
self.socket.close()
def _paginate_query(query):
# Retrieve requested page
page = int(web.input(p=1).p)
if page < 1:
page = 1
# Validate and calculate jobs range
max_jobs = RebuilddConfig().getint('http', 'jobs_per_page')
nb_pages = int(ceil(query.count() / float(max_jobs)))
if page > nb_pages:
page = nb_pages
start = (page - 1) * (max_jobs - 1)
end = page * (max_jobs - 1)
# Return formatted result
return (query[start:end], page, nb_pages)
def init(self):
self.cfg = RebuilddConfig()
# Init log system
RebuilddLog()
self._sqlconnection = sqlobject.connectionForURI(self.cfg.get('build', 'database_uri'))
sqlobject.sqlhub.processConnection = self._sqlconnection
# Create distributions
for dist in self.cfg.get('build', 'dists').split(' '):
for arch in self.cfg.arch:
Dists().add_dist(Distribution(dist, arch))
self.do_quit = threading.Event()
self.jobs_locker = threading.Lock()
self.job_finished = threading.Event()
def get_post_build_cmd(self):
"""Return command used after building source for this distribution
Substitutions are done in the command strings:
$d => The distro's name
$a => the target architecture
$p => the package's name
$v => the package's version
$j => rebuildd job id
"""
cmd = RebuilddConfig().get('build', 'post_build_cmd')
if cmd == '':
return None
try:
args = { 'd': self.dist, 'a': self.arch, \
'v': self.package.version, 'p': self.package.name, \
'j': str(self.id) }
t = Template(cmd)
return t.safe_substitute(**args)
except TypeError, error:
RebuilddLog.error("post_build_cmd has invalid format: %s" % error)
return None
class Rebuildd(object):
jobs = []
_instance = None
def __new__(cls):
if cls._instance is None:
cls._instance = object.__new__(cls)
cls._instance.init()
return cls._instance
def init(self):
self.cfg = RebuilddConfig()
# Init log system
RebuilddLog()
self._sqlconnection = sqlobject.connectionForURI(self.cfg.get('build', 'database_uri'))
sqlobject.sqlhub.processConnection = self._sqlconnection
# Create distributions
for dist in self.cfg.get('build', 'dists').split(' '):
for arch in self.cfg.arch:
Dists().add_dist(Distribution(dist, arch))
self.do_quit = threading.Event()
self.jobs_locker = threading.Lock()
self.job_finished = threading.Event()
def daemon(self):
RebuilddLog.info("Starting rebuildd %s" % __version__)
self.daemonize()
# Run the network server thread
RebuilddLog.info("Launching network server")
self.netserv = RebuilddNetworkServer(self)
self.netserv.setDaemon(True)
self.netserv.start()
# Run main loop
RebuilddLog.info("Running main loop")
self.loop()
# On exit
RebuilddLog.info("Cleaning finished and canceled jobs")
self.clean_jobs()
RebuilddLog.info("Stopping all jobs")
self.stop_all_jobs()
RebuilddLog.info("Releasing wait-locked jobs")
self.release_jobs()
self.netserv.join(10)
RebuilddLog.info("Exiting rebuildd")
def daemonize(self):
"""Do daemon stuff"""
signal.signal(signal.SIGTERM, self.handle_sigterm)
signal.signal(signal.SIGINT, self.handle_sigterm)
try:
os.chdir(self.cfg.get('build', 'work_dir'))
except Exception, error:
print "E: unable to chdir to work_dir: %s" % error
sys.exit(1)
try:
sys.stdout = sys.stderr = file(self.cfg.get('log', 'file'), "a")
except Exception, error:
print "E: unable to open logfile: %s" % error
class Rebuildd(object):
jobs = []
_instance = None
def __new__(cls):
if cls._instance is None:
cls._instance = object.__new__(cls)
cls._instance.init()
return cls._instance
def init(self):
self.cfg = RebuilddConfig()
# Init log system
RebuilddLog()
self._sqlconnection = sqlobject.connectionForURI(
self.cfg.get('build', 'database_uri'))
sqlobject.sqlhub.processConnection = self._sqlconnection
# Create distributions
for dist in self.cfg.get('build', 'dists').split(' '):
for arch in self.cfg.arch:
Dists().add_dist(Distribution(dist, arch))
self.do_quit = threading.Event()
self.jobs_locker = threading.Lock()
self.job_finished = threading.Event()
def daemon(self):
RebuilddLog.info("Starting rebuildd %s" % __version__)
self.daemonize()
# Run the network server thread
RebuilddLog.info("Launching network server")
self.netserv = RebuilddNetworkServer(self)
self.netserv.setDaemon(True)
self.netserv.start()
# Run main loop
RebuilddLog.info("Running main loop")
self.loop()
# On exit
RebuilddLog.info("Cleaning finished and canceled jobs")
self.clean_jobs()
RebuilddLog.info("Stopping all jobs")
self.stop_all_jobs()
RebuilddLog.info("Releasing wait-locked jobs")
self.release_jobs()
self.netserv.join(10)
RebuilddLog.info("Exiting rebuildd")
def daemonize(self):
"""Do daemon stuff"""
signal.signal(signal.SIGTERM, self.handle_sigterm)
signal.signal(signal.SIGINT, self.handle_sigterm)
try:
os.chdir(self.cfg.get('build', 'work_dir'))
except Exception, error:
print "E: unable to chdir to work_dir: %s" % error
sys.exit(1)
try:
sys.stdout = sys.stderr = file(self.cfg.get('log', 'file'), "a")
except Exception, error:
print "E: unable to open logfile: %s" % error
if state != 0:
with self.status_lock:
self.status = failed_status
break
if self.do_quit.isSet():
# Kill gently the process
RebuilddLog.info("Killing job %s with SIGINT" % self.id)
try:
os.killpg(os.getpgid(proc.pid), signal.SIGINT)
except OSError, error:
RebuilddLog.error("Error killing job %s: %s" % (self.id, error))
# If after 60s it's not dead, KILL HIM
counter = 0
timemax = RebuilddConfig().get('build', 'kill_timeout')
while proc.poll() == None and counter < timemax:
time.sleep(1)
counter += 1
if proc.poll() == None:
RebuilddLog.error("Killing job %s timed out, killing with SIGKILL" \
% self.id)
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
with self.status_lock:
self.status = JobStatus.WAIT_LOCKED
# Reset host
self.host = None
build_log.write("\nBuild job killed on request\n")
from __future__ import with_statement
from RebuilddConfig import RebuilddConfig
from Rebuildd import Rebuildd
from Package import Package
from Job import Job
from JobStatus import JobStatus
from JobStatus import FailedStatus
import tempfile, socket
import web
import gdchart
from sqlobject.sqlbuilder import IN, AND, DESC, Select
from math import ceil
render = web.template.render(RebuilddConfig().get('http', 'templates_dir'), \
cache=RebuilddConfig().getboolean('http', 'cache'))
def _paginate_query(query):
# Retrieve requested page
page = int(web.input(p=1).p)
if page < 1:
page = 1
# Validate and calculate jobs range
max_jobs = RebuilddConfig().getint('http', 'jobs_per_page')
nb_pages = int(ceil(query.count() / float(max_jobs)))
if page > nb_pages:
page = nb_pages
start = (page - 1) * (max_jobs - 1)
