コード例 #1
0
 def _clobber_slave(self, slavename, builder_name):
     data = [{"slave": slavename, "buildername": builder_name}]
     log.info("Clobbering %s %s at %s %s", slavename, builder_name,
              self.clobberer_url, data)
     req = urllib2.Request(self.clobberer_url)
     req.add_header('Content-Type', 'application/json')
     self._authorize(req, self.clobberer_auth)
     return urllib2.urlopen(req, json.dumps(data))
コード例 #2
0
 def _clobber_slave(self, slavename, builder_name):
     data = [{
         "slave": slavename,
         "buildername": builder_name
     }]
     log.info("Clobbering %s %s at %s %s", slavename, builder_name, self.clobberer_url, data)
     req = urllib2.Request(self.clobberer_url)
     req.add_header('Content-Type', 'application/json')
     urllib2.urlopen(req, json.dumps(data))
コード例 #3
0
ファイル: mq.py プロジェクト: kartikgupta0909/build-buildapi
 def receive(self, message_data, message):
     """Handles new job completion messages.  Marks them as finished in the
     DB by setting the complete_at and complete_data columsn."""
     try:
         log.info("Got %s", message_data)
         now = self._clock()
         s = self.session()
         r = s.query(buildapidb.JobRequest).get(message_data['request_id'])
         if not r:
             log.warn("Couldn't find message %s", message_data['request_id'])
         else:
             log.info("Updating complete data")
             r.completed_at = now
             r.complete_data = json.dumps(message_data)
             s.commit()
         message.ack()
     except:
         log.exception("Unable to process message %s", message_data)
 def receive(self, message_data, message):
     """Handles new job completion messages.  Marks them as finished in the
     DB by setting the complete_at and complete_data columsn."""
     try:
         log.info("Got %s", message_data)
         now = self._clock()
         s = self.session()
         r = s.query(buildapidb.JobRequest).get(message_data['request_id'])
         if not r:
             log.warn("Couldn't find message %s",
                      message_data['request_id'])
         else:
             log.info("Updating complete data")
             r.completed_at = now
             r.complete_data = json.dumps(message_data)
             s.commit()
         message.ack()
     except:
         log.exception("Unable to process message %s", message_data)
コード例 #5
0
ファイル: mq.py プロジェクト: kartikgupta0909/build-buildapi
    def send_msg(self, action, who, **kwargs):
        """Wrap JobRequestPublisher.send_msg by first logging the request in
        the DB, and then sending the message to the broker."""
        try:
            what = json.dumps(kwargs)
            when = self._clock()
            r = buildapidb.JobRequest(action=action, who=who, when=when, what=what)
            s = self.session()
            s.add(r)
            s.commit()
        except:
            log.exception("Couldn't create JobRequest row")
            return {"status": "FAILED", "msg": "Couldn't create JobRequest row"}

        try:
            JobRequestPublisher.send_msg(self, action, who, when=r.when,
                    request_id=r.id, **kwargs)
        except:
            log.exception("Couldn't send message")
            # TODO: What do we do with r?  Mark it as done/failed, or try and
            # pick it up again later?  Return job id anyway?
            return {"status": "FAILED", "msg": "Couldn't send message to broker"}
        return {"status": "OK", "request_id": r.id}
    def send_msg(self, action, who, **kwargs):
        """Wrap JobRequestPublisher.send_msg by first logging the request in
        the DB, and then sending the message to the broker."""
        try:
            what = json.dumps(kwargs)
            when = self._clock()
            r = buildapidb.JobRequest(action=action,
                                      who=who,
                                      when=when,
                                      what=what)
            s = self.session()
            s.add(r)
            s.commit()
        except:
            log.exception("Couldn't create JobRequest row")
            return {
                "status": "FAILED",
                "msg": "Couldn't create JobRequest row"
            }

        try:
            JobRequestPublisher.send_msg(self,
                                         action,
                                         who,
                                         when=r.when,
                                         request_id=r.id,
                                         **kwargs)
        except:
            log.exception("Couldn't send message")
            # TODO: What do we do with r?  Mark it as done/failed, or try and
            # pick it up again later?  Return job id anyway?
            return {
                "status": "FAILED",
                "msg": "Couldn't send message to broker"
            }
        return {"status": "OK", "request_id": r.id}
コード例 #7
0
ファイル: selfserve.py プロジェクト: hwine/build-buildapi
 def _htmlify(self, obj):
     return "<pre>%s</pre>" % e(json.dumps(obj, indent=2))
コード例 #8
0
 def _htmlify(self, obj):
     return "<pre>%s</pre>" % e(json.dumps(obj, indent=2))
コード例 #9
0
    def do_new_build_for_builder(self, message_data, message):
        who = message_data['who']
        branch = message_data['body']['branch']
        revision = message_data['body']['revision']
        priority = message_data['body']['priority']
        builder_name = message_data['body']['builder_name']
        files = message_data['body']['files']
        log.info("New build for %s by %s of %s %s", builder_name, who, branch, revision)

        # Create a sourcestamp
        real_branch = branch.split("-selfserve")[0]
        q = text("""INSERT INTO sourcestamps
                (`branch`, `revision`, `patchid`, `repository`, `project`)
                VALUES
                (:real_branch, :revision, NULL, '', '')
                """)
        log.debug(q)
        r = self.db.execute(q, branch=branch, revision=revision)
        ssid = r.lastrowid # SourcestampID
        log.debug("Created sourcestamp %s", ssid)

        # Create change object
        when = time.time()
        q = text("""INSERT INTO changes
                (`author`, `comments`, `is_dir`, `branch`,
                `revision`, `revlink`, `when_timestamp`, `category`,
                `repository`, `project`)
                VALUES
                (:who, '', 0, :branch, :revision, NULL, :when, NULL, '', '')
                """)
        log.debug(q)
        r = self.db.execute(q, who=who, branch=branch, revision=revision, when=when)
        cid = r.lastrowid
        log.debug("Created change %s", cid)

        # Create change-files
        for f in files:
            q = text("""INSERT INTO change_files
                    (`changeid`, `filename`)
                    VALUES
                    (:cid, :f)
                    """)
            log.debug(q)
            r = self.db.execute(q, cid=cid, f=f) 
        log.debug("Created change_file for change object %s", cid)

        # Create sourcestamp_changes
        q = text("""INSERT INTO sourcestamp_changes
                (`sourcestampid`, `changeid`)
                VALUES
                (:ssid, :cid)
                """)
        log.debug(q)
        r = self.db.execute(q, ssid=ssid, cid=cid) 
        log.debug("Created sourcestamp_changes for sourcestamp %s, change object %s", ssid, cid)

        # Create a new buildset
        now = time.time()
        buildsetid = create_buildset(
            self.db,
            idstring=None,
            reason='Self-serve: Requested by %s' % who,
            ssid=ssid,
            submitted_at=now,
        )

        # Create buildset properties (buildid, builduid)
        q = text("""INSERT INTO buildset_properties
                (`buildsetid`, `property_name`, `property_value`)
                VALUES
                (:buildsetid, :key, :value)
                """)
        props = {
            'buildid': json.dumps((genBuildID(now), "self-serve")),
            'builduid': json.dumps((genBuildUID(), "self-serve")),
        }
        # Making a tuple of each key and a tuple of it's associated property and the source "self-serve"
        props.update(((k, json.dumps((v, "self-serve"))) for (k,v) in message_data['body']['properties'].iteritems()))
        log.debug(q)
        for key, value in props.items():
            r = self.db.execute(q, buildsetid=buildsetid, key=key, value=value)
            log.debug("Created buildset_property %s=%s", key, value)

        # Create buildrequests
        q = text("""INSERT INTO buildrequests
                (`buildsetid`, `buildername`, `submitted_at`, `priority`, `claimed_at`, `claimed_by_name`, `claimed_by_incarnation`, `complete`, `results`, `complete_at`)
                VALUES
                (:buildsetid, :builder_name, :submitted_at, :priority, 0, NULL, NULL, 0, NULL, NULL)""")
        log.debug(q)
        r = self.db.execute(
            q,
            buildsetid=buildsetid,
            builder_name=builder_name,
            submitted_at=now,
            priority=priority)
        log.debug("Created buildrequest %s: %i", builder_name, r.lastrowid)
        return {"errors": False, "msg": "Ok"}
コード例 #10
0
    def _create_build_for_revision(self, who, branch, revision, priority, builder_expression, builder_exclusions=None):
        if builder_exclusions is None:
            builder_exclusions = ['%l10n nightly']
        now = time.time()
        repo_path = self._get_repo_path(branch)

        # Find builders that have been active in the past 2 weeks
        q = """SELECT DISTINCT buildername FROM buildrequests WHERE
                buildername LIKE :buildername AND
            """
        for i, bx in enumerate(builder_exclusions):
            q = q + "buildername NOT LIKE :buildername_exclusion_%i AND " % i
        q = q + """
          submitted_at > :submitted_at"""
        qparams = {
            'buildername': builder_expression,
            'submitted_at': time.time() - 14 * 24 * 3600,
        }
        for i, bx in enumerate(builder_exclusions):
            qparams['buildername_exclusion_%i' % i] = builder_exclusions[i]
        result = self.db.execute(text(q), qparams)

        buildernames = [r[0] for r in result]
        log.debug("buildernames are %s", buildernames)

        # Create a sourcestamp
        q = text("""INSERT INTO sourcestamps
                (`branch`, `revision`, `patchid`, `repository`, `project`)
                VALUES
                (:branch, :revision, NULL, '', '')
                """)
        log.debug(q)
        r = self.db.execute(q, branch=repo_path, revision=revision)
        ssid = r.lastrowid
        log.debug("Created sourcestamp %s", ssid)

        # Create a new buildset
        buildsetid = create_buildset(
            self.db,
            idstring=None,
            reason='Self-serve: Requested by %s' % who,
            ssid=ssid,
            submitted_at=now,
        )

        # Create buildset properties (buildid, builduid)
        q = text("""INSERT INTO buildset_properties
                (`buildsetid`, `property_name`, `property_value`)
                VALUES
                (:buildsetid, :key, :value)
                """)
        props = {
            'buildid': json.dumps((genBuildID(now), "self-serve")),
            'builduid': json.dumps((genBuildUID(), "self-serve")),
        }
        log.debug(q)
        for key, value in props.items():
            r = self.db.execute(q, buildsetid=buildsetid, key=key, value=value)
            log.debug("Created buildset_property %s=%s", key, value)

        # Create buildrequests
        q = text("""INSERT INTO buildrequests
                (`buildsetid`, `buildername`, `submitted_at`, `priority`, `claimed_at`, `claimed_by_name`, `claimed_by_incarnation`, `complete`, `results`, `complete_at`)
                VALUES
                (:buildsetid, :buildername, :submitted_at, :priority, 0, NULL, NULL, 0, NULL, NULL)""")
        log.debug(q)
        for buildername in buildernames:
            r = self.db.execute(
                q,
                buildsetid=buildsetid,
                buildername=buildername,
                submitted_at=now,
                priority=priority)
            log.debug("Created buildrequest %s: %i", buildername, r.lastrowid)
        return {"errors": False, "msg": "Ok"}
コード例 #11
0
    def do_new_build_for_builder(self, message_data, message):
        who = message_data['who']
        branch = message_data['body']['branch']
        revision = message_data['body']['revision']
        priority = message_data['body']['priority']
        builder_name = message_data['body']['builder_name']
        files = message_data['body']['files']
        log.info("New build for %s by %s of %s %s", builder_name, who, branch,
                 revision)

        # Create a sourcestamp
        real_branch = branch.split("-selfserve")[0]
        q = text("""INSERT INTO sourcestamps
                (`branch`, `revision`, `patchid`, `repository`, `project`)
                VALUES
                (:real_branch, :revision, NULL, '', '')
                """)
        log.debug(q)
        r = self.db.execute(q, real_branch=real_branch, revision=revision)
        ssid = r.lastrowid  # SourcestampID
        log.debug("Created sourcestamp %s", ssid)

        # Create change object
        when = time.time()
        q = text("""INSERT INTO changes
                (`author`, `comments`, `is_dir`, `branch`,
                `revision`, `revlink`, `when_timestamp`, `category`,
                `repository`, `project`)
                VALUES
                (:who, '', 0, :branch, :revision, NULL, :when, NULL, '', '')
                """)
        log.debug(q)
        r = self.db.execute(q,
                            who=who,
                            branch=branch,
                            revision=revision,
                            when=when)
        cid = r.lastrowid
        log.debug("Created change %s", cid)

        # Create change-files
        for f in files:
            q = text("""INSERT INTO change_files
                    (`changeid`, `filename`)
                    VALUES
                    (:cid, :f)
                    """)
            log.debug(q)
            r = self.db.execute(q, cid=cid, f=f)
        log.debug("Created change_file for change object %s", cid)

        # Create sourcestamp_changes
        q = text("""INSERT INTO sourcestamp_changes
                (`sourcestampid`, `changeid`)
                VALUES
                (:ssid, :cid)
                """)
        log.debug(q)
        r = self.db.execute(q, ssid=ssid, cid=cid)
        log.debug(
            "Created sourcestamp_changes for sourcestamp %s, change object %s",
            ssid, cid)

        # Create a new buildset
        now = time.time()
        buildsetid = create_buildset(
            self.db,
            idstring=None,
            reason='Self-serve: Requested by %s' % who,
            ssid=ssid,
            submitted_at=now,
        )

        # Create buildset properties (buildid, builduid)
        q = text("""INSERT INTO buildset_properties
                (`buildsetid`, `property_name`, `property_value`)
                VALUES
                (:buildsetid, :key, :value)
                """)
        props = {
            'buildid': json.dumps((genBuildID(now), "self-serve")),
            'builduid': json.dumps((genBuildUID(), "self-serve")),
        }
        # Making a tuple of each key and a tuple of it's associated property and the source "self-serve"
        props.update(
            ((k, json.dumps((v, "self-serve")))
             for (k, v) in message_data['body']['properties'].iteritems()))
        log.debug(q)
        for key, value in props.items():
            r = self.db.execute(q, buildsetid=buildsetid, key=key, value=value)
            log.debug("Created buildset_property %s=%s", key, value)

        # Create buildrequests
        q = text("""INSERT INTO buildrequests
                (`buildsetid`, `buildername`, `submitted_at`, `priority`, `claimed_at`, `claimed_by_name`, `claimed_by_incarnation`, `complete`, `results`, `complete_at`)
                VALUES
                (:buildsetid, :builder_name, :submitted_at, :priority, 0, NULL, NULL, 0, NULL, NULL)"""
                 )
        log.debug(q)
        r = self.db.execute(q,
                            buildsetid=buildsetid,
                            builder_name=builder_name,
                            submitted_at=now,
                            priority=priority)
        log.debug("Created buildrequest %s: %i", builder_name, r.lastrowid)
        return {"errors": False, "msg": "Ok"}
コード例 #12
0
    def _create_build_for_revision(self,
                                   who,
                                   branch,
                                   revision,
                                   priority,
                                   builder_expression,
                                   builder_exclusions=None):
        now = time.time()
        repo_path = self._get_repo_path(branch)

        # Figure out our set of builders
        buildernames = get_buildernames(branch, self.builders,
                                        builder_expression, builder_exclusions)
        log.debug("buildernames are %s", buildernames)
        if not buildernames:
            log.info("no builders found")
            return {"errors": False, "msg": "No builds to create"}

        # Create a sourcestamp
        q = text("""INSERT INTO sourcestamps
                (`branch`, `revision`, `patchid`, `repository`, `project`)
                VALUES
                (:branch, :revision, NULL, '', '')
                """)
        log.debug(q)
        r = self.db.execute(q, branch=repo_path, revision=revision)
        ssid = r.lastrowid
        log.debug("Created sourcestamp %s", ssid)

        # Create a new buildset
        buildsetid = create_buildset(
            self.db,
            idstring=None,
            reason='Self-serve: Requested by %s' % who,
            ssid=ssid,
            submitted_at=now,
        )

        # Create buildset properties (buildid, builduid)
        q = text("""INSERT INTO buildset_properties
                (`buildsetid`, `property_name`, `property_value`)
                VALUES
                (:buildsetid, :key, :value)
                """)
        props = {
            'buildid': json.dumps((genBuildID(now), "self-serve")),
            'builduid': json.dumps((genBuildUID(), "self-serve")),
        }
        log.debug(q)
        for key, value in props.items():
            r = self.db.execute(q, buildsetid=buildsetid, key=key, value=value)
            log.debug("Created buildset_property %s=%s", key, value)

        # Create buildrequests
        q = text("""INSERT INTO buildrequests
                (`buildsetid`, `buildername`, `submitted_at`, `priority`, `claimed_at`, `claimed_by_name`, `claimed_by_incarnation`, `complete`, `results`, `complete_at`)
                VALUES
                (:buildsetid, :buildername, :submitted_at, :priority, 0, NULL, NULL, 0, NULL, NULL)"""
                 )
        log.debug(q)
        for buildername in buildernames:
            r = self.db.execute(q,
                                buildsetid=buildsetid,
                                buildername=buildername,
                                submitted_at=now,
                                priority=priority)
            log.debug("Created buildrequest %s: %i", buildername, r.lastrowid)
        return {"errors": False, "msg": "Ok"}
コード例 #13
0
    def _create_build_for_revision(self, who, branch, revision, priority,
                                   builder_expression,
                                   builder_exclusions=None):
        now = time.time()
        repo_path = self._get_repo_path(branch)

        # Figure out our set of builders
        buildernames = get_buildernames(branch, self.builders,
                                        builder_expression, builder_exclusions)
        log.debug("buildernames are %s", buildernames)
        if not buildernames:
            log.info("no builders found")
            return {"errors": False, "msg": "No builds to create"}

        # Create a sourcestamp
        q = text("""INSERT INTO sourcestamps
                (`branch`, `revision`, `patchid`, `repository`, `project`)
                VALUES
                (:branch, :revision, NULL, '', '')
                """)
        log.debug(q)
        r = self.db.execute(q, branch=repo_path, revision=revision)
        ssid = r.lastrowid
        log.debug("Created sourcestamp %s", ssid)

        # Create a new buildset
        buildsetid = create_buildset(
            self.db,
            idstring=None,
            reason='Self-serve: Requested by %s' % who,
            ssid=ssid,
            submitted_at=now,
        )

        # Create buildset properties (buildid, builduid)
        q = text("""INSERT INTO buildset_properties
                (`buildsetid`, `property_name`, `property_value`)
                VALUES
                (:buildsetid, :key, :value)
                """)
        props = {
            'buildid': json.dumps((genBuildID(now), "self-serve")),
            'builduid': json.dumps((genBuildUID(), "self-serve")),
        }
        log.debug(q)
        for key, value in props.items():
            r = self.db.execute(q, buildsetid=buildsetid, key=key, value=value)
            log.debug("Created buildset_property %s=%s", key, value)

        # Create buildrequests
        q = text("""INSERT INTO buildrequests
                (`buildsetid`, `buildername`, `submitted_at`, `priority`, `claimed_at`, `claimed_by_name`, `claimed_by_incarnation`, `complete`, `results`, `complete_at`)
                VALUES
                (:buildsetid, :buildername, :submitted_at, :priority, 0, NULL, NULL, 0, NULL, NULL)""")
        log.debug(q)
        for buildername in buildernames:
            r = self.db.execute(
                q,
                buildsetid=buildsetid,
                buildername=buildername,
                submitted_at=now,
                priority=priority)
            log.debug("Created buildrequest %s: %i", buildername, r.lastrowid)
        return {"errors": False, "msg": "Ok"}
 def jsonify(self, data):
     response.headers['Content-Type'] = 'application/json'
     return json.dumps(data)
コード例 #15
0
    def _create_build_for_revision(self, who, branch, revision, priority, builder_expression, builder_exclusions=None):
        if builder_exclusions is None:
            builder_exclusions = ['%l10n nightly']
        now = time.time()
        repo_path = self._get_repo_path(branch)

        # Find builders that have been active in the past 2 weeks
        q = """SELECT DISTINCT buildername FROM buildrequests WHERE
                buildername LIKE :buildername AND
            """
        for i, bx in enumerate(builder_exclusions):
            q = q + "buildername NOT LIKE :buildername_exclusion_%i AND " % i
        q = q + """
          submitted_at > :submitted_at"""
        qparams = {
            'buildername': builder_expression,
            'submitted_at': time.time() - 14 * 24 * 3600,
        }
        for i, bx in enumerate(builder_exclusions):
            qparams['buildername_exclusion_%i' % i] = builder_exclusions[i]
        result = self.db.execute(text(q), qparams)

        buildernames = [r[0] for r in result]
        log.debug("buildernames are %s", buildernames)

        # Create a sourcestamp
        q = text("""INSERT INTO sourcestamps
                (`branch`, `revision`, `patchid`, `repository`, `project`)
                VALUES
                (:branch, :revision, NULL, '', '')
                """)
        log.debug(q)
        r = self.db.execute(q, branch=repo_path, revision=revision)
        ssid = r.lastrowid
        log.debug("Created sourcestamp %s", ssid)

        # Create a new buildset
        buildsetid = create_buildset(
            self.db,
            idstring=None,
            reason='Self-serve: Requested by %s' % who,
            ssid=ssid,
            submitted_at=now,
        )

        # Create buildset properties (buildid, builduid)
        q = text("""INSERT INTO buildset_properties
                (`buildsetid`, `property_name`, `property_value`)
                VALUES
                (:buildsetid, :key, :value)
                """)
        props = {
            'buildid': json.dumps((genBuildID(now), "self-serve")),
            'builduid': json.dumps((genBuildUID(), "self-serve")),
        }
        log.debug(q)
        for key, value in props.items():
            r = self.db.execute(q, buildsetid=buildsetid, key=key, value=value)
            log.debug("Created buildset_property %s=%s", key, value)

        # Create buildrequests
        q = text("""INSERT INTO buildrequests
                (`buildsetid`, `buildername`, `submitted_at`, `priority`, `claimed_at`, `claimed_by_name`, `claimed_by_incarnation`, `complete`, `results`, `complete_at`)
                VALUES
                (:buildsetid, :buildername, :submitted_at, :priority, 0, NULL, NULL, 0, NULL, NULL)""")
        log.debug(q)
        for buildername in buildernames:
            r = self.db.execute(
                q,
                buildsetid=buildsetid,
                buildername=buildername,
                submitted_at=now,
                priority=priority)
            log.debug("Created buildrequest %s: %i", buildername, r.lastrowid)
        return {"errors": False, "msg": "Ok"}
コード例 #16
0
ファイル: base.py プロジェクト: mozilla-releng/build-buildapi
 def jsonify(self, data):
     response.headers["Content-Type"] = "application/json"
     return json.dumps(data)