def init(self):
     cfg = RebuilddConfig()
     logging.basicConfig(level=logging.DEBUG,
                         format='%(asctime)s %(levelname)s %(message)s',
                         filename=cfg.get('log', 'file'),
                         datefmt=cfg.get('log', 'time_format'),
                         filemode='a')
    def GET(self, name=None, version=None):
        jobs = []

        if version:
            title = "%s %s" % (name, version)
            package = "%s/%s" % (name, version)
            query = Job.select(
                IN(
                    Job.q.package,
                    Select(
                        Package.q.id,
                        AND(Package.q.name == name,
                            Package.q.version == version))))
        else:
            title = package = name
            query = Job.select(
                IN(Job.q.package, Select(Package.q.id,
                                         Package.q.name == name)))

        result, page, nb_pages = _paginate_query(
            query.orderBy(DESC(Job.q.package)))

        jobs.extend(result)
        return render.base(page=render.tab(jobs=jobs, page=page, nb_pages=nb_pages), \
                hostname=socket.gethostname(), \
                title=title, \
                package=package, \
                archs=RebuilddConfig().arch, \
                dists=RebuilddConfig().get('build', 'dists').split(' '))
Beispiel #3
0
    def get_build_cmd(self):
        """Return command used for building source for this distribution

        Substitutions are done in the command strings:

        $d => The distro's name
        $a => the target architecture
        $p => the package's name
        $v => the package's version
        $j => rebuildd job id
        """

        # Strip epochs (x:) away
        try:
            index = self.package.version.index(":")
            args = { 'd': self.dist, 'a': self.arch, \
                     'v': self.package.version[index+1:], 'p': self.package.name, \
                     'j': str(self.id) }
            t = Template(RebuilddConfig().get('build', 'build_cmd'))
            return t.safe_substitute(**args)
        except ValueError:
            pass

        try:
            args = { 'd': self.dist, 'a': self.arch, \
                     'v': self.package.version, 'p': self.package.name, \
                     'j': str(self.id) }
            t = Template(RebuilddConfig().get('build', 'build_cmd'))
            return t.safe_substitute(**args)
        except TypeError, error:
            RebuilddLog.error("get_build_cmd has invalid format: %s" % error)
            return None
Beispiel #4
0
 def init(self):
     cfg = RebuilddConfig()
     logging.basicConfig(level=logging.DEBUG,
                         format='%(asctime)s %(levelname)s %(message)s',
                         filename=cfg.get('log', 'file'),
                         datefmt=cfg.get('log', 'time_format'),
                         filemode='a')
 def GET(self):
     jobs = []
     query = Job.select(orderBy=DESC(Job.q.creation_date))
     result, page, nb_pages = _paginate_query(query)
     jobs.extend(result)
     return render.base(\
                 page=render.tab(jobs=jobs, page=page, nb_pages=nb_pages), \
                 title="all builds", \
                 hostname=socket.gethostname(), \
                 archs=RebuilddConfig().arch, \
                 dists=RebuilddConfig().get('build', 'dists').split(' '))
    def start(self):
        """Run main HTTP server thread"""

        web.webapi.internalerror = web.debugerror

        import sys
        sys.argv.append(RebuilddConfig().get('http', 'ip') + ":" +
                        RebuilddConfig().get('http', 'port'))

        app = web.application(self.urls, globals())
        app.run()
class RequestJob:
    def GET(self, jobid=None):
        job = Job.selectBy(id=jobid)[0]

        try:
            with open(job.logfile, "r") as build_logfile:
                build_log = build_logfile.read()
        except IOError, error:
            build_log = job.log.text

        return render.base(page=render.job(job=job, build_log=build_log), \
                hostname=socket.gethostname(), \
                title="job %s" % job.id, \
                archs=RebuilddConfig().arch, \
                dists=RebuilddConfig().get('build', 'dists').split(' '))
    def GET(self, dist, arch=None):
        jobs = []

        result, page, nb_pages = _paginate_query(
            Job.select(AND(Job.q.arch == arch, Job.q.dist == dist),
                       orderBy=DESC(Job.q.creation_date)))

        jobs.extend(result)
        return render.base(page=render.tab(jobs=jobs, page=page, nb_pages=nb_pages), \
                arch=arch, \
                dist=dist, \
                title="%s/%s" % (dist, arch), \
                hostname=socket.gethostname(), \
                archs=RebuilddConfig().arch, \
                dists=RebuilddConfig().get('build', 'dists').split(' '))
Beispiel #9
0
    def init(self):
        self.cfg = RebuilddConfig()

        # Init log system
        RebuilddLog()

        self._sqlconnection = sqlobject.connectionForURI(
            self.cfg.get('build', 'database_uri'))
        sqlobject.sqlhub.processConnection = self._sqlconnection

        # Create distributions
        for dist in self.cfg.get('build', 'dists').split(' '):
            for arch in self.cfg.arch:
                Dists().add_dist(Distribution(dist, arch))

        self.do_quit = threading.Event()
        self.jobs_locker = threading.Lock()
        self.job_finished = threading.Event()
Beispiel #10
0
    def logfile(self):
        """Compute and return logfile name"""

        build_log_file = "%s/%s_%s-%s-%s-%s.%s.log" % (RebuilddConfig().get('log', 'logs_dir'),
                                           self.package.name, self.package.version,
                                           self.dist, self.arch,
                                           self.creation_date.strftime("%Y%m%d-%H%M%S"),
                                           self.id)
        return build_log_file
    def exec_cmd_config(self, *args):
        """Manipulate configuration file"""

        if len(args) < 1:
            return "E: usage: config [reload|dump|save]\n"

        if args[0] == "reload":
            if RebuilddConfig().reload():
                return "I: config reloaded\n"
            return "E: config not reloded\n"

        if args[0] == "dump":
            return RebuilddConfig().dump()

        if args[0] == "save":
            if RebuilddConfig().save():
                return "I: config saved\n"
            return "E: config not saved\n"

        return "E: usage: config [reload|dump|save]\n"
Beispiel #12
0
    def run(self):
        """Run main network server thread"""

        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.socket.settimeout(1)
        self.socket.bind((RebuilddConfig().get('telnet', 'ip'),
                          RebuilddConfig().getint('telnet', 'port')))
        self.socket.listen(2)
        while not self.rebuildd.do_quit.isSet():
            try:
                (client_socket, client_info) = self.socket.accept()
                if client_socket:
                    interface = RebuilddNetworkClient(client_socket,
                                                      self.rebuildd)
                    interface.setDaemon(True)
                    interface.start()
            except socket.timeout:
                pass

        self.socket.close()
    def run(self):
        """Run client thread"""

        self.socket.settimeout(1)
        self.socket.send(RebuilddConfig().get('telnet', 'motd') + "\n")
        prompt = RebuilddConfig().get('telnet', 'prompt') + " "
        line = ""
        has_timeout = False
        while line != "exit" and not self.rebuildd.do_quit.isSet():
            if not has_timeout:
                try:
                    self.socket.send(self.exec_cmd(line))
                    self.socket.send(prompt)
                except Exception:
                    break
            try:
                line = ""
                line = self.socket.recv(512).strip()
                has_timeout = False
            except socket.timeout:
                has_timeout = True

        self.socket.close()
Beispiel #14
0
def _paginate_query(query):
    # Retrieve requested page
    page = int(web.input(p=1).p)
    if page < 1:
        page = 1

    # Validate and calculate jobs range
    max_jobs = RebuilddConfig().getint('http', 'jobs_per_page')
    nb_pages = int(ceil(query.count() / float(max_jobs)))
    if page > nb_pages:
        page = nb_pages
    start = (page - 1) * (max_jobs - 1)
    end = page * (max_jobs - 1)

    # Return formatted result
    return (query[start:end], page, nb_pages)
Beispiel #15
0
    def init(self):
        self.cfg = RebuilddConfig()

        # Init log system
        RebuilddLog()

        self._sqlconnection = sqlobject.connectionForURI(self.cfg.get('build', 'database_uri'))
        sqlobject.sqlhub.processConnection = self._sqlconnection

        # Create distributions
        for dist in self.cfg.get('build', 'dists').split(' '):
            for arch in self.cfg.arch:
                Dists().add_dist(Distribution(dist, arch))

        self.do_quit = threading.Event()
        self.jobs_locker = threading.Lock()
        self.job_finished = threading.Event()
Beispiel #16
0
    def get_post_build_cmd(self):
        """Return command used after building source for this distribution

        Substitutions are done in the command strings:

        $d => The distro's name
        $a => the target architecture
        $p => the package's name
        $v => the package's version
        $j => rebuildd job id
        """

        cmd = RebuilddConfig().get('build', 'post_build_cmd')
        if cmd == '':
            return None
        try:
            args = { 'd': self.dist, 'a': self.arch, \
                     'v': self.package.version, 'p': self.package.name, \
                     'j': str(self.id) }
            t = Template(cmd)
	    return t.safe_substitute(**args)
        except TypeError, error:
            RebuilddLog.error("post_build_cmd has invalid format: %s" % error)
            return None
Beispiel #17
0
class Rebuildd(object):
    jobs = []
    _instance = None 
         
    def __new__(cls):  
        if cls._instance is None:  
           cls._instance = object.__new__(cls)  
           cls._instance.init()
        return cls._instance  

    def init(self):
        self.cfg = RebuilddConfig()

        # Init log system
        RebuilddLog()

        self._sqlconnection = sqlobject.connectionForURI(self.cfg.get('build', 'database_uri'))
        sqlobject.sqlhub.processConnection = self._sqlconnection

        # Create distributions
        for dist in self.cfg.get('build', 'dists').split(' '):
            for arch in self.cfg.arch:
                Dists().add_dist(Distribution(dist, arch))

        self.do_quit = threading.Event()
        self.jobs_locker = threading.Lock()
        self.job_finished = threading.Event()

    def daemon(self):
        RebuilddLog.info("Starting rebuildd %s" % __version__)
        self.daemonize()

        # Run the network server thread
        RebuilddLog.info("Launching network server")
        self.netserv = RebuilddNetworkServer(self)
        self.netserv.setDaemon(True)
        self.netserv.start()

        # Run main loop
        RebuilddLog.info("Running main loop")
        self.loop()

        # On exit
        RebuilddLog.info("Cleaning finished and canceled jobs")
        self.clean_jobs()
        RebuilddLog.info("Stopping all jobs")
        self.stop_all_jobs()
        RebuilddLog.info("Releasing wait-locked jobs")
        self.release_jobs()
        self.netserv.join(10)
        RebuilddLog.info("Exiting rebuildd")

    def daemonize(self):
        """Do daemon stuff"""

        signal.signal(signal.SIGTERM, self.handle_sigterm)
        signal.signal(signal.SIGINT, self.handle_sigterm)

        try:
            os.chdir(self.cfg.get('build', 'work_dir'))
        except Exception, error:
            print "E: unable to chdir to work_dir: %s" % error
            sys.exit(1)

        try:
            sys.stdout = sys.stderr = file(self.cfg.get('log', 'file'), "a")
        except Exception, error:
            print "E: unable to open logfile: %s" % error
Beispiel #18
0
class Rebuildd(object):
    jobs = []
    _instance = None

    def __new__(cls):
        if cls._instance is None:
            cls._instance = object.__new__(cls)
            cls._instance.init()
        return cls._instance

    def init(self):
        self.cfg = RebuilddConfig()

        # Init log system
        RebuilddLog()

        self._sqlconnection = sqlobject.connectionForURI(
            self.cfg.get('build', 'database_uri'))
        sqlobject.sqlhub.processConnection = self._sqlconnection

        # Create distributions
        for dist in self.cfg.get('build', 'dists').split(' '):
            for arch in self.cfg.arch:
                Dists().add_dist(Distribution(dist, arch))

        self.do_quit = threading.Event()
        self.jobs_locker = threading.Lock()
        self.job_finished = threading.Event()

    def daemon(self):
        RebuilddLog.info("Starting rebuildd %s" % __version__)
        self.daemonize()

        # Run the network server thread
        RebuilddLog.info("Launching network server")
        self.netserv = RebuilddNetworkServer(self)
        self.netserv.setDaemon(True)
        self.netserv.start()

        # Run main loop
        RebuilddLog.info("Running main loop")
        self.loop()

        # On exit
        RebuilddLog.info("Cleaning finished and canceled jobs")
        self.clean_jobs()
        RebuilddLog.info("Stopping all jobs")
        self.stop_all_jobs()
        RebuilddLog.info("Releasing wait-locked jobs")
        self.release_jobs()
        self.netserv.join(10)
        RebuilddLog.info("Exiting rebuildd")

    def daemonize(self):
        """Do daemon stuff"""

        signal.signal(signal.SIGTERM, self.handle_sigterm)
        signal.signal(signal.SIGINT, self.handle_sigterm)

        try:
            os.chdir(self.cfg.get('build', 'work_dir'))
        except Exception, error:
            print "E: unable to chdir to work_dir: %s" % error
            sys.exit(1)

        try:
            sys.stdout = sys.stderr = file(self.cfg.get('log', 'file'), "a")
        except Exception, error:
            print "E: unable to open logfile: %s" % error
Beispiel #19
0
            if state != 0:
                with self.status_lock:
                    self.status = failed_status
                break

        if self.do_quit.isSet():
            # Kill gently the process
            RebuilddLog.info("Killing job %s with SIGINT" % self.id)
            try:
                os.killpg(os.getpgid(proc.pid), signal.SIGINT)
            except OSError, error:
                RebuilddLog.error("Error killing job %s: %s" % (self.id, error))

            # If after 60s it's not dead, KILL HIM
            counter = 0
            timemax = RebuilddConfig().get('build', 'kill_timeout')
            while proc.poll() == None and counter < timemax:
                time.sleep(1)
                counter += 1
            if proc.poll() == None:
                RebuilddLog.error("Killing job %s timed out, killing with SIGKILL" \
                           % self.id)
                os.killpg(os.getpgid(proc.pid), signal.SIGKILL)

            with self.status_lock:
                self.status = JobStatus.WAIT_LOCKED

            # Reset host
            self.host = None

            build_log.write("\nBuild job killed on request\n")
Beispiel #20
0
from __future__ import with_statement

from RebuilddConfig import RebuilddConfig
from Rebuildd import Rebuildd
from Package import Package
from Job import Job
from JobStatus import JobStatus
from JobStatus import FailedStatus

import tempfile, socket
import web
import gdchart
from sqlobject.sqlbuilder import IN, AND, DESC, Select
from math import ceil

render = web.template.render(RebuilddConfig().get('http', 'templates_dir'), \
         cache=RebuilddConfig().getboolean('http', 'cache'))


def _paginate_query(query):
    # Retrieve requested page
    page = int(web.input(p=1).p)
    if page < 1:
        page = 1

    # Validate and calculate jobs range
    max_jobs = RebuilddConfig().getint('http', 'jobs_per_page')
    nb_pages = int(ceil(query.count() / float(max_jobs)))
    if page > nb_pages:
        page = nb_pages
    start = (page - 1) * (max_jobs - 1)