示例#1
0
class Manager(Thread):
    def __init__(self, app):
        Thread.__init__(self)
        self.must_run = False
        self.app = app
        self.build_list = {}
        self.carrier = Carrier(pecan.conf.rabbit_server,
                               pecan.conf.rabbit_port, pecan.conf.rabbit_user,
                               pecan.conf.rabbit_password,
                               pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
        self.supported_build_type = pecan.conf.supported_build_type
        for build_type in self.supported_build_type:
            self.carrier.declare_queue('%s.queue' % build_type)

        self.logger = None

    def shutdown(self):
        logging.debug("Stopping Manager")
        self.carrier.closing = True
        self.must_run = False

    def run(self):
        self.must_run = True
        logging.debug("Starting Manager")

        while self.must_run:
            time.sleep(0.1)
            for build_type in self.supported_build_type:
                new_build = self.carrier.get_message('%s.queue' % build_type)

                build = None
                if new_build is not None:
                    distro_name = new_build['distro_name']
                    build_conf = new_build['build_conf']
                    root_folder = new_build['root_folder']
                    build_path = new_build['build_path']
                    build = Build(new_build['build'])
                    job_id = new_build['job_id']
                    path = os.path.dirname(get_logger_path(build))
                    try:
                        os.makedirs(path)
                    except Exception:
                        pass
                    self.logger = get_logger(build, distro_name)
                    self.logger.debug(build.dumps())
                    builder_class = globals().get(build_type.title() +
                                                  'Builder')
                    builder = builder_class(distro_name, build_conf,
                                            root_folder, self.logger, build,
                                            build_path, job_id)
                    builder.run()
                    """
                    build = Build(new_build)
                    build.user = User.fetch(new_build['username'],
                                            sub_objects=False)
                    build.project = Project.fetch(build.user,
                                                  new_build['project_name'],
                                                  sub_objects=False)
                    """
                """if build:
示例#2
0
    def post(self, send_build):
        """ launch build """
        project_name = pecan.request.context['project_name']
        user = User.fetch(pecan.request.context['username'])
        project = Project.fetch(user.username, project_name)

        if project is None:
            # The project doesn't exist
            # We have to create it
            # TODO Maybe it's better to force users to create project before
            # they can create builds
            sent_project = {"name": project_name, "username": user.username}
            project = Project(sent_project, sub_objects=False)
            if not project.create():
                # Handle error
                return {"result": "Error creating %s project" % project_name}

        build = Build(send_build)
        build.username = user.username
        build.project_name = project.name
        build.create()
        carrier = Carrier(
            pecan.conf.rabbit_server,
            pecan.conf.rabbit_port,
            pecan.conf.rabbit_user,
            pecan.conf.rabbit_password,
            pecan.conf.rabbit_vhost,
            pecan.conf.rabbit_db
        )
        carrier.declare_queue('builds.queue')
        # carrier.declare_builds()
        if not carrier.send_message(build.dumps(), 'builds.queue'):
            return None
        return {"result": {"build": int(build.id_)}}
示例#3
0
class Manager(Thread):
    def __init__(self, app):
        Thread.__init__(self)
        self.must_run = False
        self.app = app
        self.build_list = {}
        self.carrier = Carrier(
            pecan.conf.rabbit_server,
            pecan.conf.rabbit_port,
            pecan.conf.rabbit_user,
            pecan.conf.rabbit_password,
            pecan.conf.rabbit_vhost,
            pecan.conf.rabbit_db
        )
        self.supported_build_type = pecan.conf.supported_build_type
        for build_type in self.supported_build_type:
            self.carrier.declare_queue('%s.queue' % build_type)

        self.logger = None

    def shutdown(self):
        logging.debug("Stopping Manager")
        self.carrier.closing = True
        self.must_run = False

    def run(self):
        self.must_run = True
        logging.debug("Starting Manager")

        while self.must_run:
            time.sleep(0.1)
            for build_type in self.supported_build_type:
                new_build = self.carrier.get_message('%s.queue' % build_type)

                build = None
                if new_build is not None:
                    distro_name = new_build['distro_name']
                    build_conf = new_build['build_conf']
                    root_folder = new_build['root_folder']
                    build_path = new_build['build_path']
                    build = Build(new_build['build'])
                    path = os.path.dirname(get_logger_path(build))
                    try:
                        os.makedirs(path)
                    except Exception:
                        pass
                    self.logger = get_logger(build, distro_name)
                    self.logger.debug(build.dumps())
                    builder_class = globals().get(build_type.title() + 'Builder')
                    builder = builder_class(distro_name, build_conf, root_folder, self.logger, build, build_path)
                    builder.run()
                    """
                    build = Build(new_build)
                    build.user = User.fetch(new_build['username'],
                                            sub_objects=False)
                    build.project = Project.fetch(build.user,
                                                  new_build['project_name'],
                                                  sub_objects=False)
                    """
                """if build:
示例#4
0
    def post(self, send_build):
        """ launch build """
        project_name = pecan.request.context['project_name']
        user = User.fetch(pecan.request.context['username'])
        project = Project.fetch(user.username, project_name)

        if project is None:
            # The project doesn't exist
            # We have to create it
            # TODO Maybe it's better to force users to create project before
            # they can create builds
            sent_project = {"name": project_name, "username": user.username}
            project = Project(sent_project, sub_objects=False)
            if not project.create():
                # Handle error
                return {"result": "Error creating %s project" % project_name}

        build = Build(send_build)
        build.username = user.username
        build.project_name = project.name
        build.create()
        carrier = Carrier(pecan.conf.rabbit_server, pecan.conf.rabbit_port,
                          pecan.conf.rabbit_user, pecan.conf.rabbit_password,
                          pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
        carrier.declare_queue('builds.queue')
        # carrier.declare_builds()
        if not carrier.send_message(build.dumps(), 'builds.queue'):
            return None
        return {"result": {"build": int(build.id_)}}
示例#5
0
 def __init__(self, app):
     self.must_run = False
     self.app = app
     self.build_list = {}
     self.carrier = Carrier(pecan.conf.rabbit_server,
                            pecan.conf.rabbit_port, pecan.conf.rabbit_user,
                            pecan.conf.rabbit_password,
                            pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
     self.carrier.declare_queue('builds.queue')
示例#6
0
    def dispatch(self, packer_conf, root_folder, global_conf=None):

        carrier = Carrier(pecan.conf.rabbit_server,
                          pecan.conf.rabbit_port,
                          pecan.conf.rabbit_user,
                          pecan.conf.rabbit_password,
                          pecan.conf.rabbit_vhost,
                          pecan.conf.rabbit_db
                          )
        # carrier.declare_queue('docker.queue')
        for distro_name, build_conf in packer_conf.items():
            if not hasattr(self.build, "forced_distro") or (
                    self.build.forced_distro == distro_name or
                    not self.build.forced_distro):
                if 'type' not in build_conf:
                    # Default type is docker to keep backward compat
                    build_conf['type'] = 'docker'

                # if not build_conf['type'] == 'docker':
                queue = "%s.queue" % build_conf['type']
                carrier.declare_queue(queue)

                # Add global external repos
                if not build_conf.get('repos'):
                    build_conf['repos'] = {}
                if not build_conf.get('repos').get('rpm'):
                    build_conf['repos']['rpm'] = []
                if global_conf:
                    build_conf['repos']['rpm'] += global_conf['repos']['rpm']
                #
                build_conf['distro'] = distro_name
                build_conf['branch'] = self.branch
                build_conf['root_folder'] = root_folder
                job_data = {
                    'distro': distro_name,
                    'username': self.build.username,
                    'project_name': self.build.project_name,
                    'build_id': self.build.id_,
                }
                job = Job(job_data)
                job.create()
                message = {
                    'distro_name': distro_name,
                    'build_conf': build_conf,
                    'root_folder': root_folder,
                    'log_path': get_logger_path(self.build),
                    'id_': self.id_,
                    'job_id': job.id_,
                    'build': self.build.dumps(),
                    'build_path': self.build.get_folder_path()
                }
                if not carrier.send_message(message, queue):
                    self.logger.error("Can't post message to rabbitmq")
                else:
                    self.logger.info("Posted build to %s" % queue)
                    self.build.inc_job_count()
示例#7
0
    def dispatch(self, packer_conf, root_folder, global_conf=None):

        carrier = Carrier(pecan.conf.rabbit_server, pecan.conf.rabbit_port,
                          pecan.conf.rabbit_user, pecan.conf.rabbit_password,
                          pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
        # carrier.declare_queue('docker.queue')
        for distro_name, build_conf in packer_conf.items():
            if not hasattr(self.build, "forced_distro") or (
                    self.build.forced_distro == distro_name
                    or not self.build.forced_distro):
                if 'type' not in build_conf:
                    # Default type is docker to keep backward compat
                    build_conf['type'] = 'docker'

                # if not build_conf['type'] == 'docker':
                queue = "%s.queue" % build_conf['type']
                carrier.declare_queue(queue)

                # Add global external repos
                if not build_conf.get('repos'):
                    build_conf['repos'] = {}
                if not build_conf.get('repos').get('rpm'):
                    build_conf['repos']['rpm'] = []
                if global_conf:
                    build_conf['repos']['rpm'] += global_conf['repos']['rpm']
                #
                build_conf['distro'] = distro_name
                build_conf['branch'] = self.branch
                build_conf['root_folder'] = root_folder
                job_data = {
                    'distro': distro_name,
                    'username': self.build.username,
                    'project_name': self.build.project_name,
                    'build_id': self.build.id_,
                }
                job = Job(job_data)
                job.create()
                message = {
                    'distro_name': distro_name,
                    'build_conf': build_conf,
                    'root_folder': root_folder,
                    'log_path': get_logger_path(self.build),
                    'id_': self.id_,
                    'job_id': job.id_,
                    'build': self.build.dumps(),
                    'build_path': self.build.get_folder_path()
                }
                if not carrier.send_message(message, queue):
                    self.logger.error("Can't post message to rabbitmq")
                else:
                    self.logger.info("Posted build to %s" % queue)
                    self.build.inc_job_count()
示例#8
0
    def __init__(self, app):
        Thread.__init__(self)
        self.must_run = False
        self.app = app
        self.build_list = {}
        self.carrier = Carrier(pecan.conf.rabbit_server,
                               pecan.conf.rabbit_port, pecan.conf.rabbit_user,
                               pecan.conf.rabbit_password,
                               pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
        self.supported_build_type = pecan.conf.supported_build_type
        for build_type in self.supported_build_type:
            self.carrier.declare_queue('%s.queue' % build_type)

        self.logger = None
示例#9
0
    def __init__(self, app):
        Thread.__init__(self)
        self.must_run = False
        self.app = app
        self.build_list = {}
        self.carrier = Carrier(
            pecan.conf.rabbit_server,
            pecan.conf.rabbit_port,
            pecan.conf.rabbit_user,
            pecan.conf.rabbit_password,
            pecan.conf.rabbit_vhost,
            pecan.conf.rabbit_db
        )
        self.supported_build_type = pecan.conf.supported_build_type
        for build_type in self.supported_build_type:
            self.carrier.declare_queue('%s.queue' % build_type)

        self.logger = None
示例#10
0
    def post(self):
        """ launch build from gitlab webhook"""
        body = pecan.request.json
        # Get use
        if not body.get('user_id'):
            abort(403)

        # Get token
        token = pecan.request.GET.get('token')
        if token is None:
            abort(403)

        # Get project
        project = Project.fetch_from_token(token, False)
        if project is None:
            abort(403)
        if body.get('project_id') != project.gitlab_project_id:
            abort(403)

        if body.get('object_kind') not in ['push', 'tag']:
            abort(403)

        else:
            # If it's a TAG event we DON'T make snaphot
            snapshot = False
            if body.get('object_kind') == 'push':
                # If it's a PUSH event we make snapshot
                snapshot = True

            if not body.get('repository'):
                abort(403)
            repository = body.get('repository')
            project_name = repository.get('name')
            if project_name != project.name:
                abort(403)

            new_build = {
                "source_url": repository.get('git_http_url'),
                #"source_type": "gitlab",
                "source_type": "git",
                "commit": body.get('after'),
                # TODO Find how decide if is a snapshot or not
                "snapshot": snapshot,
                # TODO Check if branch ~= ref
                "branch": body.get('ref'),
            }
            build = Build(new_build)
            build.username = project.username
            build.project_name = project.name
            build.create()
            carrier = Carrier(pecan.conf.rabbit_server, pecan.conf.rabbit_port,
                              pecan.conf.rabbit_user,
                              pecan.conf.rabbit_password,
                              pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
            carrier.declare_queue('builds.queue')
            # carrier.declare_builds()
            if not carrier.send_message(build.dumps(), 'builds.queue'):
                return None
            return json.dumps({"result": True, "build": int(build.id_)})

        abort(403)
示例#11
0
    def post(self):
        """Launch build from github webhook"""
        body = pecan.request.json
        # Get user
        if not body.get('sender'):
            abort(403)
        # Get username
        username = body.get('repository').get('owner').get('name')
        if username is None:
            username = body.get('repository').get('owner').get('login')
        user = User.fetch(username)
        if user is None:
            abort(403)
        # Check signature
        signature = pecan.request.headers.get('X-Hub-Signature')
        sha_name, signature = signature.split("=")
        if sha_name != 'sha1':
            abort(403)
        mac = hmac.new(user.token.encode("utf-8"), pecan.request.text, digestmod=hashlib.sha1)
        if not hmac.compare_digest(mac.hexdigest(), signature):
            abort(403)

        # Ping event
        if pecan.request.headers.get('X-Github-Event') == 'ping':
            return json.dumps({"result": True, "event": "ping"})

        # TODO handle tag event
        # Push Event
        if pecan.request.headers.get('X-Github-Event') == 'push':
            if not body.get('repository'):
                abort(403)
            repository = body.get('repository')
            project_name = repository.get('name')
            if not project_name:
                abort(403)

            project = Project.fetch(user.username, project_name)
            if project is None:
                # Error project doesn't exits
                # Maybe We should create it
                return json.dumps({"result": False , "error": "project not found"})
            new_build = {"source_url": repository.get('clone_url'),
                         #"source_type": "github",
                         "source_type": "git",
                         "commit": repository.get('commit'),
                         # TODO Find how decide if is a snapshot or not
                         # Answer: on tag event => NOT
                         "snapshot": True,
                         # TODO Check if branch ~= ref
                         "branch": repository.get('ref'),
                         }
            build = Build(new_build)
            build.username = user.username
            build.project_name = project.name
            build.create()
            carrier = Carrier(
                pecan.conf.rabbit_server,
                pecan.conf.rabbit_port,
                pecan.conf.rabbit_user,
                pecan.conf.rabbit_password,
                pecan.conf.rabbit_vhost,
                pecan.conf.rabbit_db
            )
            carrier.declare_queue('builds.queue')
            # carrier.declare_builds()
            if not carrier.send_message(build.dumps(), 'builds.queue'):
                return None
            return json.dumps({"result": True, "build": int(build.id_)})

        abort(403)
示例#12
0
class Manager(object):
    def __init__(self, app):
        self.must_run = False
        self.app = app
        self.build_list = {}
        self.carrier = Carrier(pecan.conf.rabbit_server,
                               pecan.conf.rabbit_port, pecan.conf.rabbit_user,
                               pecan.conf.rabbit_password,
                               pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
        self.carrier.declare_queue('builds.queue')

    def shutdown(self):
        logging.debug("Stopping Manager")
        self.carrier.closing = True
        self.must_run = False

    def run(self):
        self.must_run = True
        logging.debug("Starting Manager")

        while self.must_run:
            time.sleep(0.1)
            new_build = self.carrier.get_message('builds.queue')
            build = None
            if new_build is not None:
                build = Build(new_build)
                if build:
                    build.user = User.fetch(new_build['username'],
                                            sub_objects=False)
                    build.project = Project.fetch(build.username,
                                                  new_build['project_name'],
                                                  sub_objects=False)
                    logging.debug("Task received")
                    build.set_status("dispatching")
                    dispatcher = Dispatcher(build)
                    self.build_list[dispatcher.uuid2] = dispatcher
                    dispatcher.start()

            self.check_builds_status()

    def check_builds_status(self):
        builds = mongo.builds.find(
            {"status": {
                "$nin": ["succeeded", "failed"]
            }})
        for b in builds:
            finished = 0
            build = Build(b)
            jobs = build.get_jobs()

            if len(jobs) == build.job_count and build.job_count > 0:
                for job in jobs:
                    if job.status in ['succeeded', 'failed']:
                        finished += 1

                if finished == len(jobs):
                    if all([
                            True if j.status == 'succeeded' else False
                            for j in jobs
                    ]):
                        build.set_status('succeeded')
                    else:
                        build.set_status('failed')
                    build.finishing()

        # collect old jammed build
        time_limit = time.time() - pecan.conf.build_lifetime
        builds = mongo.builds.find({
            "status": {
                "$nin": ["succeeded", "failed"]
            },
            "created": {
                "$lte": time_limit
            }
        })
        for b in builds:
            build = Build(b)
            build.set_status('failed')
            build.finishing()
            jobs = build.get_jobs()
            for job in jobs:
                if job.status not in ['succeeded', 'failed']:
                    job.set_status('failed')
示例#13
0
    def post(self):
        """ launch build from gitlab webhook"""
        body = pecan.request.json
        # Get use
        if not body.get('user_id'):
            abort(403)

        # Get token
        token = pecan.request.GET.get('token')
        if token is None:
            abort(403)

        # Get project
        project = Project.fetch_from_token(token, False)
        if project is None:
            abort(403)
        if body.get('project_id') != project.gitlab_project_id:
            abort(403)

        if body.get('object_kind') not in ['push', 'tag']:
            abort(403)

        else:
            # If it's a TAG event we DON'T make snaphot
            snapshot = False
            if body.get('object_kind') == 'push':
                # If it's a PUSH event we make snapshot
                snapshot = True

            if not body.get('repository'):
                abort(403)
            repository = body.get('repository')
            project_name = repository.get('name')
            if project_name != project.name:
                abort(403)

            new_build = {"source_url": repository.get('git_http_url'),
                         #"source_type": "gitlab",
                         "source_type": "git",
                         "commit": body.get('after'),
                         # TODO Find how decide if is a snapshot or not
                         "snapshot": snapshot,
                         # TODO Check if branch ~= ref
                         "branch": body.get('ref'),
                         }
            build = Build(new_build)
            build.username = project.username
            build.project_name = project.name
            build.create()
            carrier = Carrier(
                pecan.conf.rabbit_server,
                pecan.conf.rabbit_port,
                pecan.conf.rabbit_user,
                pecan.conf.rabbit_password,
                pecan.conf.rabbit_vhost,
                pecan.conf.rabbit_db
            )
            carrier.declare_queue('builds.queue')
            # carrier.declare_builds()
            if not carrier.send_message(build.dumps(), 'builds.queue'):
                return None
            return json.dumps({"result": True, "build": int(build.id_)})

        abort(403)