def generate_request_body(self):
"""
Create the data structure required by tbpl's submitBuildStar.php script
It's used by both the bug_job_map endpoint and the job note endpoint.
"""
jm = JobsModel(self.project)
try:
buildapi_artifact = jm.get_job_artifact_list(0, 1, {
'job_id': set([("=", self.job_id)]),
'name': set([("=", "buildapi_complete")])
})[0]
job_data = jm.get_job(self.job_id)[0]
finally:
jm.disconnect()
note = ""
if self.bug_id:
note = "Bug {0}".format(self.bug_id)
if self.classification_id:
if note:
note += " - "
note += FailureClassification.objects.get(
id=self.classification_id).name
if self.note:
if note:
note += " - "
note += self.note
self.body = {
"id": buildapi_artifact["blob"]["id"],
"machinename": job_data["machine_name"],
"starttime": int(job_data["start_timestamp"]),
"note": note,
"who": self.who
}
def cycle_data(max_iterations=50, debug=False):
projects = Repository.objects.all().values_list('name', flat=True)
for project in projects:
jm = JobsModel(project)
sql_targets = {}
if debug:
print "Cycling Database: {0}".format(project)
cycle_iterations = max_iterations
while cycle_iterations > 0:
sql_targets = jm.cycle_data(sql_targets)
if debug:
print "Iterations: {0}".format(str(cycle_iterations))
print "sql_targets"
print sql_targets
cycle_iterations -= 1
# No more items to delete
if sql_targets['total_count'] == 0:
cycle_iterations = 0
jm.disconnect()
def cycle_data(max_iterations=50, debug=False):
projects = Repository.objects.all().values_list('name', flat=True)
for project in projects:
jm = JobsModel(project)
sql_targets = {}
if debug:
print "Cycling Database: {0}".format(project)
cycle_iterations = max_iterations
while cycle_iterations > 0:
sql_targets = jm.cycle_data(sql_targets)
if debug:
print "Iterations: {0}".format(str(cycle_iterations))
print "sql_targets"
print sql_targets
cycle_iterations -= 1
# No more items to delete
if sql_targets['total_count'] == 0:
cycle_iterations = 0
jm.disconnect()
def generate_request_body(self):
"""
Create the data structure that will be sent to ElasticSearch.
"""
jm = JobsModel(self.project)
try:
job_data = jm.get_job(self.job_id)[0]
option_collection = jm.refdata_model.get_all_option_collections()
revision_list = jm.get_resultset_revisions_list(
job_data["result_set_id"])
buildapi_artifact = jm.get_job_artifact_list(
0, 1, {
'job_id': set([("=", self.job_id)]),
'name': set([("=", "buildapi")])
})
if buildapi_artifact:
buildname = buildapi_artifact[0]["blob"]["buildername"]
else:
# OrangeFactor needs a buildname to be set or it skips the failure
# classification, so we make one up for non-buildbot jobs.
buildname = 'non-buildbot %s test %s' % (
job_data["platform"], job_data["job_type_name"])
finally:
jm.disconnect()
self.body = {
"buildname":
buildname,
"machinename":
job_data["machine_name"],
"os":
job_data["platform"],
# I'm using the request time date here, as start time is not
# available for pending jobs
"date":
datetime.fromtimestamp(int(
job_data["submit_timestamp"])).strftime("%Y-%m-%d"),
"type":
job_data["job_type_name"],
"buildtype":
option_collection[job_data["option_collection_hash"]]["opt"],
# Intentionally using strings for starttime, bug, timestamp for compatibility
# with TBPL's legacy output format.
"starttime":
str(job_data["start_timestamp"]),
"tree":
self.project,
"rev":
revision_list[0]["revision"],
"bug":
str(self.bug_id),
"who":
self.who,
"timestamp":
str(self.submit_timestamp),
"logfile":
"00000000"
}
def use_jobs_model(*args, **kwargs):
project = kwargs["project"]
jm = JobsModel(project)
try:
return model_func(*args, jm=jm, **kwargs)
finally:
jm.disconnect()
def populate_performance_series(project, series_type, series_data):
jm = JobsModel(project)
for t_range in settings.TREEHERDER_PERF_SERIES_TIME_RANGES:
for signature in series_data:
jm.store_performance_series(t_range['seconds'], series_type,
signature, series_data[signature])
jm.disconnect()
def use_jobs_model(*args, **kwargs):
project = kwargs["project"]
jm = JobsModel(project)
try:
return model_func(*args, jm=jm, **kwargs)
finally:
jm.disconnect()
def populate_performance_series(project, series_type, series_data):
jm = JobsModel(project)
for t_range in settings.TREEHERDER_PERF_SERIES_TIME_RANGES:
for signature in series_data:
jm.store_performance_series(
t_range['seconds'], series_type, signature,
series_data[signature]
)
jm.disconnect()
def publish_to_pulse(project, ids, data_type):
jm = JobsModel(project)
# Get appropriate data for data_type
# using the ids provided
data = []
if data_type == 'result_set':
data = jm.get_result_set_list_by_ids(ids)
jm.disconnect()
def calculate_eta(sample_window_seconds=21600, debug=False):
projects = Repository.objects.all().values_list('name', flat=True)
for project in projects:
jm = JobsModel(project)
jm.calculate_eta(sample_window_seconds, debug)
jm.disconnect()
def calculate_eta(sample_window_seconds=21600, debug=False):
projects = Repository.objects.all().values_list('name', flat=True)
for project in projects:
jm = JobsModel(project)
jm.calculate_eta(sample_window_seconds, debug)
jm.disconnect()
def process_objects(limit=None):
"""
Process a number of objects from the objectstore
and load them to the jobs store
"""
# default limit to 100
limit = limit or 100
for ds in Datasource.objects.all():
jm = JobsModel(ds.project)
try:
jm.process_objects(limit)
finally:
jm.disconnect()
def process_objects(limit=None):
"""
Process a number of objects from the objectstore
and load them to the jobs store
"""
# default limit to 100
limit = limit or 100
for ds in Datasource.objects.all():
jm = JobsModel(ds.project)
try:
jm.process_objects(limit)
finally:
jm.disconnect()
def generate_request_body(self):
"""
Create the data structure required by tbpl's starcomment.php script
"""
jm = JobsModel(self.project)
try:
buildapi_artifact = jm.get_job_artifact_list(
0, 1, {
'job_id': set([("=", self.job_id)]),
'name': set([("=", "buildapi")])
})[0]
job_data = jm.get_job(self.job_id)[0]
option_collection = jm.refdata_model.get_all_option_collections()
revision_list = jm.get_resultset_revisions_list(
job_data["result_set_id"])
finally:
jm.disconnect()
self.body = {
"buildname":
buildapi_artifact["blob"]["buildername"],
"machinename":
job_data["machine_name"],
"os":
job_data["platform"],
# I'm using the request time date here, as start time is not
# available for pending jobs
"date":
datetime.fromtimestamp(int(
job_data["submit_timestamp"])).strftime("%Y-%m-%d"),
"type":
job_data["job_type_name"],
"buildtype":
option_collection[job_data["option_collection_hash"]]["opt"],
"starttime":
int(job_data["start_timestamp"]),
# "logfile": "",
"tree":
self.project,
"rev":
revision_list[0]["revision"],
"comment":
"Bug {0}".format(self.bug_id),
"who":
self.who,
"timestamp":
self.submit_timestamp,
"logfile":
"00000000"
}
def unclassified_failure_count(projects=None):
if not projects:
projects = Repository.objects.all().values_list('name', flat=True)
unclassified_failure_publisher = UnclassifiedFailureCountPublisher(settings.BROKER_URL)
for project in projects:
jm = JobsModel(project)
count = jm.get_unclassified_failure_count()
count_excluded = jm.get_unclassified_failure_count_excluded()
unclassified_failure_publisher.publish(project, count, count_excluded)
jm.disconnect()
unclassified_failure_publisher.disconnect()
def publish_to_pulse(project, ids, data_type):
# If we don't have a publisher (because of missing configs), then we can't
# publish any pulse messages. This is okay, local installs etc. doesn't
# need to publish on pulse, and requiring a pulse user is adding more
# overhead to an already large development setup process.
if not publisher:
return
jm = JobsModel(project)
try:
# Publish messages with new result-sets
if data_type == 'result_set':
# Get appropriate data for data_type
# using the ids provided
for entry in jm.get_result_set_list_by_ids(ids):
repository = jm.refdata_model.get_repository_info(entry['repository_id'])
entry['repository_url'] = repository['url']
# Don't expose these properties, they are internal, at least that's
# what I think without documentation I have no clue... what any of
# this is
del entry['revisions'] # Not really internal, but too big
del entry['repository_id']
# Set required properties
entry['version'] = 1
entry['project'] = project
# Property revision_hash should already be there, I suspect it is the
# result-set identifier...
# publish the data to pulse
publisher.new_result_set(
message = entry,
revision_hash = entry['revision_hash'],
project = project
)
# Basically, I have no idea what context this runs and was inherently
# unable to make kombu with or without pyamqp, etc. confirm-publish,
# so we're stuck with this super ugly hack where we just close the
# connection so that if the process context is destroyed then at least
# messages will still get published... Well, assuming nothing goes
# wrong, because we're not using confirm channels for publishing...
publisher.connection.release()
finally:
jm.disconnect()
def get_context_data(self, **kwargs):
assert "repository" in kwargs and "revision" in kwargs
context = super(ResultsetStatusView, self).get_context_data(**kwargs)
jobs_model = JobsModel(kwargs['repository'])
try:
resultset_list = jobs_model.get_revision_resultset_lookup(
[kwargs['revision']])
if not resultset_list:
raise Http404("No resultset found for revision {0}".format(
kwargs['revision']))
result_set_id = resultset_list[kwargs['revision']]['id']
resultset_status_dict = jobs_model.get_resultset_status(
result_set_id)
context['resultset_status_dict'] = resultset_status_dict
return context
finally:
jobs_model.disconnect()
def publish_to_pulse(project, ids, data_type):
# If we don't have a publisher (because of missing configs), then we can't
# publish any pulse messages. This is okay, local installs etc. doesn't
# need to publish on pulse, and requiring a pulse user is adding more
# overhead to an already large development setup process.
if not publisher:
return
jm = JobsModel(project)
try:
# Publish messages with new result-sets
if data_type == 'result_set':
# Get appropriate data for data_type
# using the ids provided
for entry in jm.get_result_set_list_by_ids(ids):
# Don't expose these properties, they are internal, at least that's
# what I think without documentation I have no clue... what any of
# this is
del entry['revisions'] # Not really internal, but too big
del entry['repository_id']
# Set required properties
entry['version'] = 1
entry['project'] = project
# Property revision_hash should already be there, I suspect it is the
# result-set identifier...
# publish the data to pulse
publisher.new_result_set(
message = entry,
revision_hash = entry['revision_hash'],
project = project
)
# Basically, I have no idea what context this runs and was inherently
# unable to make kombu with or without pyamqp, etc. confirm-publish,
# so we're stuck with this super ugly hack where we just close the
# connection so that if the process context is destroyed then at least
# messages will still get published... Well, assuming nothing goes
# wrong, because we're not using confirm channels for publishing...
publisher.connection.release()
finally:
jm.disconnect()
def process_objects(limit=None, project=None):
"""
Process a number of objects from the objectstore
and load them to the jobs store
"""
# default limit to 100
limit = limit or 100
if project:
projects_to_process = [project]
else:
projects_to_process = Datasource.objects.values_list(
'project', flat=True).distinct()
for project in projects_to_process:
jm = JobsModel(project)
try:
jm.process_objects(limit)
finally:
jm.disconnect()
def use_jobs_model(*args, **kwargs):
project = kwargs["project"]
try:
jm = JobsModel(project)
return model_func(*args, jm=jm, **kwargs)
except DatasetNotFoundError as e:
return Response(
{"message": "No project with name {0}".format(project)},
status=404,
)
except ObjectNotFoundException as e:
return Response({"message": unicode(e)}, status=404)
except Exception as e: # pragma nocover
msg = {"message": unicode(e)}
if settings.DEBUG:
import traceback
msg["traceback"] = traceback.format_exc()
return Response(msg, status=500)
finally:
jm.disconnect()
def generate_request_body(self):
"""
Create the data structure required by tbpl's starcomment.php script
"""
jm = JobsModel(self.project)
try:
buildapi_artifact = jm.get_job_artifact_list(0, 1, {
'job_id': set([("=", self.job_id)]),
'name': set([("=", "buildapi")])
})[0]
job_data = jm.get_job(self.job_id)[0]
option_collection = jm.refdata_model.get_all_option_collections()
revision_list = jm.get_resultset_revisions_list(job_data["result_set_id"])
finally:
jm.disconnect()
self.body = {
"buildname": buildapi_artifact["blob"]["buildername"],
"machinename": job_data["machine_name"],
"os": job_data["platform"],
# I'm using the request time date here, as start time is not
# available for pending jobs
"date": datetime.fromtimestamp(
int(job_data["submit_timestamp"])).strftime("%Y-%m-%d"),
"type": job_data["job_type_name"],
"buildtype": option_collection[
job_data["option_collection_hash"]
]["opt"],
"starttime": int(job_data["start_timestamp"]),
# "logfile": "",
"tree": self.project,
"rev": revision_list[0]["revision"],
"comment": "Bug {0}".format(self.bug_id),
"who": self.who,
"timestamp": self.submit_timestamp,
"logfile": "00000000"
}
def handle(self, *args, **options):
self.is_debug = options['debug']
if options['cycle_interval']:
cycle_interval = datetime.timedelta(days=options['cycle_interval'])
else:
cycle_interval = settings.DATA_CYCLE_INTERVAL
self.debug("cycle interval: {0}".format(cycle_interval))
projects = Datasource.objects\
.filter(contenttype='jobs')\
.values_list('project', flat=True)
for project in projects:
self.debug("Cycling Database: {0}".format(project))
jm = JobsModel(project)
try:
num_deleted = jm.cycle_data(cycle_interval,
options['chunk_size'],
options['sleep_time'])
self.debug("Deleted {0} resultsets from {1}".format(
num_deleted, project))
finally:
jm.disconnect()
def handle(self, *args, **options):
self.is_debug = options['debug']
if options['cycle_interval']:
cycle_interval = datetime.timedelta(days=options['cycle_interval'])
else:
cycle_interval = settings.DATA_CYCLE_INTERVAL
self.debug("cycle interval: {0}".format(cycle_interval))
projects = Datasource.objects\
.filter(contenttype='jobs')\
.values_list('project', flat=True)
for project in projects:
self.debug("Cycling Database: {0}".format(project))
jm = JobsModel(project)
try:
num_deleted = jm.cycle_data(cycle_interval,
options['chunk_size'],
options['sleep_time'])
self.debug("Deleted {0} resultsets from {1}".format(
num_deleted, project))
finally:
jm.disconnect()
def parse_log(project, job_id, result_set_id, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
pattern_obj = re.compile('\d+:\d+:\d+\s+')
jm = JobsModel(project=project)
rdm = RefDataManager()
open_bugs_cache = {}
closed_bugs_cache = {}
status_publisher = JobStatusPublisher(settings.BROKER_URL)
failure_publisher = JobFailurePublisher(settings.BROKER_URL)
try:
# return the resultset with the job id to identify if the UI wants
# to fetch the whole thing.
resultset = jm.get_result_set_by_id(result_set_id=result_set_id)[0]
del(resultset["active_status"])
del(resultset["revision_hash"])
log_references = jm.get_log_references(job_id)
# we may have many log references per job
for log in log_references:
# parse a log given its url
artifact_bc = ArtifactBuilderCollection(
log['url'],
check_errors=check_errors,
)
artifact_bc.parse()
artifact_list = []
for name, artifact in artifact_bc.artifacts.items():
artifact_list.append((job_id, name, 'json', json.dumps(artifact)))
if check_errors:
# I'll try to begin with a full_text search on the entire row
all_errors = artifact_bc.artifacts['Structured Log']['step_data']['all_errors']
open_bugs_suggestions = {}
closed_bugs_suggestions = {}
for err in all_errors:
# remove timestamp
clean_line = pattern_obj.sub('', err['line'])
if clean_line not in open_bugs_cache:
open_bugs_cache[clean_line] = rdm.get_suggested_bugs(
clean_line)
if clean_line not in closed_bugs_cache:
closed_bugs_cache[clean_line] = rdm.get_suggested_bugs(
clean_line, open_bugs=False)
open_bugs_suggestions[ err['line'] ] = open_bugs_cache[clean_line]
closed_bugs_suggestions[ err['line'] ] = closed_bugs_cache[clean_line]
artifact_list.append((job_id, 'Open bugs', 'json', json.dumps(open_bugs_suggestions)))
artifact_list.append((job_id, 'Closed bugs', 'json', json.dumps(closed_bugs_suggestions)))
# store the artifacts generated
jm.store_job_artifact(artifact_list)
status_publisher.publish(job_id, resultset, project, 'processed')
if check_errors:
failure_publisher.publish(job_id, project)
finally:
rdm.disconnect()
jm.disconnect()
status_publisher.disconnect()
failure_publisher.disconnect()
def generate_request_body(self):
"""
Create the data structure required by tbpl's submitBugzillaComment.php script
This is triggered by a new bug-job association.
"""
jm = JobsModel(self.project)
try:
job = jm.get_job(self.job_id)[0]
failures_artifacts = jm.get_job_artifact_list(0, 1, {
'job_id': set([('=', job['id'])]),
'name': set([('=', 'Bug suggestions')]),
})
error_lines = []
for artifact in failures_artifacts:
# a bug suggestion aritfact looks like this:
# [{ "search": "my-error-line", "bugs": ....}]
error_lines += [line["search"] for line in artifact["blob"]]
bug_job_map = jm.get_bug_job_map_detail(self.job_id, self.bug_id)
revision_list = jm.get_resultset_revisions_list(
job["result_set_id"]
)
buildapi_info = jm.get_job_artifact_list(0, 1, {
'job_id': set([("=", self.job_id)]),
'name': set([("=", "buildapi")])
})
finally:
jm.disconnect()
who = bug_job_map["who"]\
.replace("@", "[at]")\
.replace(".", "[dot]")
submit_date = datetime.fromtimestamp(bug_job_map["submit_timestamp"])\
.replace(microsecond=0)\
.isoformat()
job_description = {
'repository': self.project,
'who': who,
'submit_timestamp': submit_date,
'log': "{0}{1}/logviewer.html#?repo={2}&job_id={3}".format(
settings.SITE_URL,
settings.UI_PREFIX,
self.project,
self.job_id
),
'machine': job["machine_name"],
'revision': revision_list[0]["revision"],
}
if buildapi_info:
job_description['buildname'] = buildapi_info[0]["blob"]["buildername"]
body_comment = '\n'.join(
["{0}: {1}".format(k, v) for k, v in job_description.items()])
body_comment += '\n\n'
body_comment += '\n'.join(error_lines)
self.body = {
"id": self.bug_id,
"comment": body_comment
}
