def _clobber_slave(self, slavename, builder_name):
data = [{"slave": slavename, "buildername": builder_name}]
log.info("Clobbering %s %s at %s %s", slavename, builder_name,
self.clobberer_url, data)
req = urllib2.Request(self.clobberer_url)
req.add_header('Content-Type', 'application/json')
self._authorize(req, self.clobberer_auth)
return urllib2.urlopen(req, json.dumps(data))
def _clobber_slave(self, slavename, builder_name):
data = [{
"slave": slavename,
"buildername": builder_name
}]
log.info("Clobbering %s %s at %s %s", slavename, builder_name, self.clobberer_url, data)
req = urllib2.Request(self.clobberer_url)
req.add_header('Content-Type', 'application/json')
urllib2.urlopen(req, json.dumps(data))
def receive(self, message_data, message):
"""Handles new job completion messages. Marks them as finished in the
DB by setting the complete_at and complete_data columsn."""
try:
log.info("Got %s", message_data)
now = self._clock()
s = self.session()
r = s.query(buildapidb.JobRequest).get(message_data['request_id'])
if not r:
log.warn("Couldn't find message %s", message_data['request_id'])
else:
log.info("Updating complete data")
r.completed_at = now
r.complete_data = json.dumps(message_data)
s.commit()
message.ack()
except:
log.exception("Unable to process message %s", message_data)
def receive(self, message_data, message):
"""Handles new job completion messages. Marks them as finished in the
DB by setting the complete_at and complete_data columsn."""
try:
log.info("Got %s", message_data)
now = self._clock()
s = self.session()
r = s.query(buildapidb.JobRequest).get(message_data['request_id'])
if not r:
log.warn("Couldn't find message %s",
message_data['request_id'])
else:
log.info("Updating complete data")
r.completed_at = now
r.complete_data = json.dumps(message_data)
s.commit()
message.ack()
except:
log.exception("Unable to process message %s", message_data)
def send_msg(self, action, who, **kwargs):
"""Wrap JobRequestPublisher.send_msg by first logging the request in
the DB, and then sending the message to the broker."""
try:
what = json.dumps(kwargs)
when = self._clock()
r = buildapidb.JobRequest(action=action, who=who, when=when, what=what)
s = self.session()
s.add(r)
s.commit()
except:
log.exception("Couldn't create JobRequest row")
return {"status": "FAILED", "msg": "Couldn't create JobRequest row"}
try:
JobRequestPublisher.send_msg(self, action, who, when=r.when,
request_id=r.id, **kwargs)
except:
log.exception("Couldn't send message")
# TODO: What do we do with r? Mark it as done/failed, or try and
# pick it up again later? Return job id anyway?
return {"status": "FAILED", "msg": "Couldn't send message to broker"}
return {"status": "OK", "request_id": r.id}
def send_msg(self, action, who, **kwargs):
"""Wrap JobRequestPublisher.send_msg by first logging the request in
the DB, and then sending the message to the broker."""
try:
what = json.dumps(kwargs)
when = self._clock()
r = buildapidb.JobRequest(action=action,
who=who,
when=when,
what=what)
s = self.session()
s.add(r)
s.commit()
except:
log.exception("Couldn't create JobRequest row")
return {
"status": "FAILED",
"msg": "Couldn't create JobRequest row"
}
try:
JobRequestPublisher.send_msg(self,
action,
who,
when=r.when,
request_id=r.id,
**kwargs)
except:
log.exception("Couldn't send message")
# TODO: What do we do with r? Mark it as done/failed, or try and
# pick it up again later? Return job id anyway?
return {
"status": "FAILED",
"msg": "Couldn't send message to broker"
}
return {"status": "OK", "request_id": r.id}
def _htmlify(self, obj):
return "<pre>%s</pre>" % e(json.dumps(obj, indent=2))
def _htmlify(self, obj):
return "<pre>%s</pre>" % e(json.dumps(obj, indent=2))
def do_new_build_for_builder(self, message_data, message):
who = message_data['who']
branch = message_data['body']['branch']
revision = message_data['body']['revision']
priority = message_data['body']['priority']
builder_name = message_data['body']['builder_name']
files = message_data['body']['files']
log.info("New build for %s by %s of %s %s", builder_name, who, branch, revision)
# Create a sourcestamp
real_branch = branch.split("-selfserve")[0]
q = text("""INSERT INTO sourcestamps
(`branch`, `revision`, `patchid`, `repository`, `project`)
VALUES
(:real_branch, :revision, NULL, '', '')
""")
log.debug(q)
r = self.db.execute(q, branch=branch, revision=revision)
ssid = r.lastrowid # SourcestampID
log.debug("Created sourcestamp %s", ssid)
# Create change object
when = time.time()
q = text("""INSERT INTO changes
(`author`, `comments`, `is_dir`, `branch`,
`revision`, `revlink`, `when_timestamp`, `category`,
`repository`, `project`)
VALUES
(:who, '', 0, :branch, :revision, NULL, :when, NULL, '', '')
""")
log.debug(q)
r = self.db.execute(q, who=who, branch=branch, revision=revision, when=when)
cid = r.lastrowid
log.debug("Created change %s", cid)
# Create change-files
for f in files:
q = text("""INSERT INTO change_files
(`changeid`, `filename`)
VALUES
(:cid, :f)
""")
log.debug(q)
r = self.db.execute(q, cid=cid, f=f)
log.debug("Created change_file for change object %s", cid)
# Create sourcestamp_changes
q = text("""INSERT INTO sourcestamp_changes
(`sourcestampid`, `changeid`)
VALUES
(:ssid, :cid)
""")
log.debug(q)
r = self.db.execute(q, ssid=ssid, cid=cid)
log.debug("Created sourcestamp_changes for sourcestamp %s, change object %s", ssid, cid)
# Create a new buildset
now = time.time()
buildsetid = create_buildset(
self.db,
idstring=None,
reason='Self-serve: Requested by %s' % who,
ssid=ssid,
submitted_at=now,
)
# Create buildset properties (buildid, builduid)
q = text("""INSERT INTO buildset_properties
(`buildsetid`, `property_name`, `property_value`)
VALUES
(:buildsetid, :key, :value)
""")
props = {
'buildid': json.dumps((genBuildID(now), "self-serve")),
'builduid': json.dumps((genBuildUID(), "self-serve")),
}
# Making a tuple of each key and a tuple of it's associated property and the source "self-serve"
props.update(((k, json.dumps((v, "self-serve"))) for (k,v) in message_data['body']['properties'].iteritems()))
log.debug(q)
for key, value in props.items():
r = self.db.execute(q, buildsetid=buildsetid, key=key, value=value)
log.debug("Created buildset_property %s=%s", key, value)
# Create buildrequests
q = text("""INSERT INTO buildrequests
(`buildsetid`, `buildername`, `submitted_at`, `priority`, `claimed_at`, `claimed_by_name`, `claimed_by_incarnation`, `complete`, `results`, `complete_at`)
VALUES
(:buildsetid, :builder_name, :submitted_at, :priority, 0, NULL, NULL, 0, NULL, NULL)""")
log.debug(q)
r = self.db.execute(
q,
buildsetid=buildsetid,
builder_name=builder_name,
submitted_at=now,
priority=priority)
log.debug("Created buildrequest %s: %i", builder_name, r.lastrowid)
return {"errors": False, "msg": "Ok"}
def _create_build_for_revision(self, who, branch, revision, priority, builder_expression, builder_exclusions=None):
if builder_exclusions is None:
builder_exclusions = ['%l10n nightly']
now = time.time()
repo_path = self._get_repo_path(branch)
# Find builders that have been active in the past 2 weeks
q = """SELECT DISTINCT buildername FROM buildrequests WHERE
buildername LIKE :buildername AND
"""
for i, bx in enumerate(builder_exclusions):
q = q + "buildername NOT LIKE :buildername_exclusion_%i AND " % i
q = q + """
submitted_at > :submitted_at"""
qparams = {
'buildername': builder_expression,
'submitted_at': time.time() - 14 * 24 * 3600,
}
for i, bx in enumerate(builder_exclusions):
qparams['buildername_exclusion_%i' % i] = builder_exclusions[i]
result = self.db.execute(text(q), qparams)
buildernames = [r[0] for r in result]
log.debug("buildernames are %s", buildernames)
# Create a sourcestamp
q = text("""INSERT INTO sourcestamps
(`branch`, `revision`, `patchid`, `repository`, `project`)
VALUES
(:branch, :revision, NULL, '', '')
""")
log.debug(q)
r = self.db.execute(q, branch=repo_path, revision=revision)
ssid = r.lastrowid
log.debug("Created sourcestamp %s", ssid)
# Create a new buildset
buildsetid = create_buildset(
self.db,
idstring=None,
reason='Self-serve: Requested by %s' % who,
ssid=ssid,
submitted_at=now,
)
# Create buildset properties (buildid, builduid)
q = text("""INSERT INTO buildset_properties
(`buildsetid`, `property_name`, `property_value`)
VALUES
(:buildsetid, :key, :value)
""")
props = {
'buildid': json.dumps((genBuildID(now), "self-serve")),
'builduid': json.dumps((genBuildUID(), "self-serve")),
}
log.debug(q)
for key, value in props.items():
r = self.db.execute(q, buildsetid=buildsetid, key=key, value=value)
log.debug("Created buildset_property %s=%s", key, value)
# Create buildrequests
q = text("""INSERT INTO buildrequests
(`buildsetid`, `buildername`, `submitted_at`, `priority`, `claimed_at`, `claimed_by_name`, `claimed_by_incarnation`, `complete`, `results`, `complete_at`)
VALUES
(:buildsetid, :buildername, :submitted_at, :priority, 0, NULL, NULL, 0, NULL, NULL)""")
log.debug(q)
for buildername in buildernames:
r = self.db.execute(
q,
buildsetid=buildsetid,
buildername=buildername,
submitted_at=now,
priority=priority)
log.debug("Created buildrequest %s: %i", buildername, r.lastrowid)
return {"errors": False, "msg": "Ok"}
def do_new_build_for_builder(self, message_data, message):
who = message_data['who']
branch = message_data['body']['branch']
revision = message_data['body']['revision']
priority = message_data['body']['priority']
builder_name = message_data['body']['builder_name']
files = message_data['body']['files']
log.info("New build for %s by %s of %s %s", builder_name, who, branch,
revision)
# Create a sourcestamp
real_branch = branch.split("-selfserve")[0]
q = text("""INSERT INTO sourcestamps
(`branch`, `revision`, `patchid`, `repository`, `project`)
VALUES
(:real_branch, :revision, NULL, '', '')
""")
log.debug(q)
r = self.db.execute(q, real_branch=real_branch, revision=revision)
ssid = r.lastrowid # SourcestampID
log.debug("Created sourcestamp %s", ssid)
# Create change object
when = time.time()
q = text("""INSERT INTO changes
(`author`, `comments`, `is_dir`, `branch`,
`revision`, `revlink`, `when_timestamp`, `category`,
`repository`, `project`)
VALUES
(:who, '', 0, :branch, :revision, NULL, :when, NULL, '', '')
""")
log.debug(q)
r = self.db.execute(q,
who=who,
branch=branch,
revision=revision,
when=when)
cid = r.lastrowid
log.debug("Created change %s", cid)
# Create change-files
for f in files:
q = text("""INSERT INTO change_files
(`changeid`, `filename`)
VALUES
(:cid, :f)
""")
log.debug(q)
r = self.db.execute(q, cid=cid, f=f)
log.debug("Created change_file for change object %s", cid)
# Create sourcestamp_changes
q = text("""INSERT INTO sourcestamp_changes
(`sourcestampid`, `changeid`)
VALUES
(:ssid, :cid)
""")
log.debug(q)
r = self.db.execute(q, ssid=ssid, cid=cid)
log.debug(
"Created sourcestamp_changes for sourcestamp %s, change object %s",
ssid, cid)
# Create a new buildset
now = time.time()
buildsetid = create_buildset(
self.db,
idstring=None,
reason='Self-serve: Requested by %s' % who,
ssid=ssid,
submitted_at=now,
)
# Create buildset properties (buildid, builduid)
q = text("""INSERT INTO buildset_properties
(`buildsetid`, `property_name`, `property_value`)
VALUES
(:buildsetid, :key, :value)
""")
props = {
'buildid': json.dumps((genBuildID(now), "self-serve")),
'builduid': json.dumps((genBuildUID(), "self-serve")),
}
# Making a tuple of each key and a tuple of it's associated property and the source "self-serve"
props.update(
((k, json.dumps((v, "self-serve")))
for (k, v) in message_data['body']['properties'].iteritems()))
log.debug(q)
for key, value in props.items():
r = self.db.execute(q, buildsetid=buildsetid, key=key, value=value)
log.debug("Created buildset_property %s=%s", key, value)
# Create buildrequests
q = text("""INSERT INTO buildrequests
(`buildsetid`, `buildername`, `submitted_at`, `priority`, `claimed_at`, `claimed_by_name`, `claimed_by_incarnation`, `complete`, `results`, `complete_at`)
VALUES
(:buildsetid, :builder_name, :submitted_at, :priority, 0, NULL, NULL, 0, NULL, NULL)"""
)
log.debug(q)
r = self.db.execute(q,
buildsetid=buildsetid,
builder_name=builder_name,
submitted_at=now,
priority=priority)
log.debug("Created buildrequest %s: %i", builder_name, r.lastrowid)
return {"errors": False, "msg": "Ok"}
def _create_build_for_revision(self,
who,
branch,
revision,
priority,
builder_expression,
builder_exclusions=None):
now = time.time()
repo_path = self._get_repo_path(branch)
# Figure out our set of builders
buildernames = get_buildernames(branch, self.builders,
builder_expression, builder_exclusions)
log.debug("buildernames are %s", buildernames)
if not buildernames:
log.info("no builders found")
return {"errors": False, "msg": "No builds to create"}
# Create a sourcestamp
q = text("""INSERT INTO sourcestamps
(`branch`, `revision`, `patchid`, `repository`, `project`)
VALUES
(:branch, :revision, NULL, '', '')
""")
log.debug(q)
r = self.db.execute(q, branch=repo_path, revision=revision)
ssid = r.lastrowid
log.debug("Created sourcestamp %s", ssid)
# Create a new buildset
buildsetid = create_buildset(
self.db,
idstring=None,
reason='Self-serve: Requested by %s' % who,
ssid=ssid,
submitted_at=now,
)
# Create buildset properties (buildid, builduid)
q = text("""INSERT INTO buildset_properties
(`buildsetid`, `property_name`, `property_value`)
VALUES
(:buildsetid, :key, :value)
""")
props = {
'buildid': json.dumps((genBuildID(now), "self-serve")),
'builduid': json.dumps((genBuildUID(), "self-serve")),
}
log.debug(q)
for key, value in props.items():
r = self.db.execute(q, buildsetid=buildsetid, key=key, value=value)
log.debug("Created buildset_property %s=%s", key, value)
# Create buildrequests
q = text("""INSERT INTO buildrequests
(`buildsetid`, `buildername`, `submitted_at`, `priority`, `claimed_at`, `claimed_by_name`, `claimed_by_incarnation`, `complete`, `results`, `complete_at`)
VALUES
(:buildsetid, :buildername, :submitted_at, :priority, 0, NULL, NULL, 0, NULL, NULL)"""
)
log.debug(q)
for buildername in buildernames:
r = self.db.execute(q,
buildsetid=buildsetid,
buildername=buildername,
submitted_at=now,
priority=priority)
log.debug("Created buildrequest %s: %i", buildername, r.lastrowid)
return {"errors": False, "msg": "Ok"}
def _create_build_for_revision(self, who, branch, revision, priority,
builder_expression,
builder_exclusions=None):
now = time.time()
repo_path = self._get_repo_path(branch)
# Figure out our set of builders
buildernames = get_buildernames(branch, self.builders,
builder_expression, builder_exclusions)
log.debug("buildernames are %s", buildernames)
if not buildernames:
log.info("no builders found")
return {"errors": False, "msg": "No builds to create"}
# Create a sourcestamp
q = text("""INSERT INTO sourcestamps
(`branch`, `revision`, `patchid`, `repository`, `project`)
VALUES
(:branch, :revision, NULL, '', '')
""")
log.debug(q)
r = self.db.execute(q, branch=repo_path, revision=revision)
ssid = r.lastrowid
log.debug("Created sourcestamp %s", ssid)
# Create a new buildset
buildsetid = create_buildset(
self.db,
idstring=None,
reason='Self-serve: Requested by %s' % who,
ssid=ssid,
submitted_at=now,
)
# Create buildset properties (buildid, builduid)
q = text("""INSERT INTO buildset_properties
(`buildsetid`, `property_name`, `property_value`)
VALUES
(:buildsetid, :key, :value)
""")
props = {
'buildid': json.dumps((genBuildID(now), "self-serve")),
'builduid': json.dumps((genBuildUID(), "self-serve")),
}
log.debug(q)
for key, value in props.items():
r = self.db.execute(q, buildsetid=buildsetid, key=key, value=value)
log.debug("Created buildset_property %s=%s", key, value)
# Create buildrequests
q = text("""INSERT INTO buildrequests
(`buildsetid`, `buildername`, `submitted_at`, `priority`, `claimed_at`, `claimed_by_name`, `claimed_by_incarnation`, `complete`, `results`, `complete_at`)
VALUES
(:buildsetid, :buildername, :submitted_at, :priority, 0, NULL, NULL, 0, NULL, NULL)""")
log.debug(q)
for buildername in buildernames:
r = self.db.execute(
q,
buildsetid=buildsetid,
buildername=buildername,
submitted_at=now,
priority=priority)
log.debug("Created buildrequest %s: %i", buildername, r.lastrowid)
return {"errors": False, "msg": "Ok"}
def jsonify(self, data):
response.headers['Content-Type'] = 'application/json'
return json.dumps(data)
def _create_build_for_revision(self, who, branch, revision, priority, builder_expression, builder_exclusions=None):
if builder_exclusions is None:
builder_exclusions = ['%l10n nightly']
now = time.time()
repo_path = self._get_repo_path(branch)
# Find builders that have been active in the past 2 weeks
q = """SELECT DISTINCT buildername FROM buildrequests WHERE
buildername LIKE :buildername AND
"""
for i, bx in enumerate(builder_exclusions):
q = q + "buildername NOT LIKE :buildername_exclusion_%i AND " % i
q = q + """
submitted_at > :submitted_at"""
qparams = {
'buildername': builder_expression,
'submitted_at': time.time() - 14 * 24 * 3600,
}
for i, bx in enumerate(builder_exclusions):
qparams['buildername_exclusion_%i' % i] = builder_exclusions[i]
result = self.db.execute(text(q), qparams)
buildernames = [r[0] for r in result]
log.debug("buildernames are %s", buildernames)
# Create a sourcestamp
q = text("""INSERT INTO sourcestamps
(`branch`, `revision`, `patchid`, `repository`, `project`)
VALUES
(:branch, :revision, NULL, '', '')
""")
log.debug(q)
r = self.db.execute(q, branch=repo_path, revision=revision)
ssid = r.lastrowid
log.debug("Created sourcestamp %s", ssid)
# Create a new buildset
buildsetid = create_buildset(
self.db,
idstring=None,
reason='Self-serve: Requested by %s' % who,
ssid=ssid,
submitted_at=now,
)
# Create buildset properties (buildid, builduid)
q = text("""INSERT INTO buildset_properties
(`buildsetid`, `property_name`, `property_value`)
VALUES
(:buildsetid, :key, :value)
""")
props = {
'buildid': json.dumps((genBuildID(now), "self-serve")),
'builduid': json.dumps((genBuildUID(), "self-serve")),
}
log.debug(q)
for key, value in props.items():
r = self.db.execute(q, buildsetid=buildsetid, key=key, value=value)
log.debug("Created buildset_property %s=%s", key, value)
# Create buildrequests
q = text("""INSERT INTO buildrequests
(`buildsetid`, `buildername`, `submitted_at`, `priority`, `claimed_at`, `claimed_by_name`, `claimed_by_incarnation`, `complete`, `results`, `complete_at`)
VALUES
(:buildsetid, :buildername, :submitted_at, :priority, 0, NULL, NULL, 0, NULL, NULL)""")
log.debug(q)
for buildername in buildernames:
r = self.db.execute(
q,
buildsetid=buildsetid,
buildername=buildername,
submitted_at=now,
priority=priority)
log.debug("Created buildrequest %s: %i", buildername, r.lastrowid)
return {"errors": False, "msg": "Ok"}
def jsonify(self, data):
response.headers["Content-Type"] = "application/json"
return json.dumps(data)