def test_post_job_collection(self):
"""Can add a treeherder collections to a TreeherderRequest."""
tjc = TreeherderJobCollection()
for job in self.job_data:
tjc.add(tjc.get_job(job))
client = TreeherderClient(
server_url='http://host',
client_id='client-abc',
secret='secret123',
)
def request_callback(request):
# Check that the expected content was POSTed.
posted_json = json.loads(request.body)
self.assertEqual(posted_json, tjc.get_collection_data())
return (200, {}, '{"message": "Job successfully updated"}')
url = client._get_endpoint_url(tjc.endpoint_base, project='project')
responses.add_callback(responses.POST,
url,
match_querystring=True,
callback=request_callback,
content_type='application/json')
client.post_collection('project', tjc)
def test_job_collection(self):
"""Confirm the collection matches the sample data"""
tjc = TreeherderJobCollection()
for job in self.job_data:
tj = TreeherderJob(job)
tjc.add(tj)
self.assertTrue(len(self.job_data) == len(tjc.data))
def test_job_collection(self):
"""Confirm the collection matches the sample data"""
tjc = TreeherderJobCollection()
for job in self.job_data:
tj = TreeherderJob(job)
tjc.add(tj)
self.assertTrue(len(self.job_data) == len(tjc.data))
def completed_jobs_stored(test_repository, failure_classifications,
completed_jobs, result_set_stored, mock_post_json):
"""
stores a list of buildapi completed jobs
"""
completed_jobs['revision'] = result_set_stored[0]['revision']
completed_jobs.update({'project': test_repository.name})
tjc = TreeherderJobCollection()
tj = tjc.get_job(completed_jobs)
tjc.add(tj)
test_utils.post_collection(test_repository.name, tjc)
def running_jobs_stored(test_repository, failure_classifications, running_jobs,
result_set_stored, mock_post_json):
"""
stores a list of buildapi running jobs
"""
running_jobs.update(result_set_stored[0])
running_jobs.update({'project': test_repository.name})
tjc = TreeherderJobCollection()
tj = tjc.get_job(running_jobs)
tjc.add(tj)
test_utils.post_collection(test_repository.name, tjc)
def completed_jobs_stored(
test_repository, failure_classifications, completed_jobs,
push_stored, mock_post_json):
"""
stores a list of buildapi completed jobs
"""
completed_jobs['revision'] = push_stored[0]['revision']
completed_jobs.update({'project': test_repository.name})
tjc = TreeherderJobCollection()
tj = tjc.get_job(completed_jobs)
tjc.add(tj)
test_utils.post_collection(test_repository.name, tjc)
def running_jobs_stored(
test_repository, failure_classifications, running_jobs,
push_stored, mock_post_json):
"""
stores a list of buildapi running jobs
"""
running_jobs.update(push_stored[0])
running_jobs.update({'project': test_repository.name})
tjc = TreeherderJobCollection()
tj = tjc.get_job(running_jobs)
tjc.add(tj)
test_utils.post_collection(test_repository.name, tjc)
def pending_jobs_stored(test_repository, failure_classifications, pending_jobs,
result_set_stored, mock_post_json):
"""
stores a list of buildapi pending jobs into the jobs store
using BuildApiTreeHerderAdapter
"""
pending_jobs.update(result_set_stored[0])
pending_jobs.update({'project': test_repository.name})
tjc = TreeherderJobCollection()
tj = tjc.get_job(pending_jobs)
tjc.add(tj)
test_utils.post_collection(test_repository.name, tjc)
def pending_jobs_stored(
test_repository, failure_classifications, pending_jobs,
push_stored):
"""
stores a list of buildapi pending jobs into the jobs store
"""
pending_jobs.update(push_stored[0])
pending_jobs.update({'project': test_repository.name})
tjc = TreeherderJobCollection()
tj = tjc.get_job(pending_jobs)
tjc.add(tj)
store_job_data(test_repository, tjc.get_collection_data())
def pending_jobs_stored(
test_repository, failure_classifications, pending_jobs,
push_stored, mock_post_json):
"""
stores a list of buildapi pending jobs into the jobs store
using BuildApiTreeHerderAdapter
"""
pending_jobs.update(push_stored[0])
pending_jobs.update({'project': test_repository.name})
tjc = TreeherderJobCollection()
tj = tjc.get_job(pending_jobs)
tjc.add(tj)
test_utils.post_collection(test_repository.name, tjc)
def test_job_collection_job_type(self):
"""
Confirm that the job_type argument changes the endpoint_base property
"""
tjc = TreeherderJobCollection()
self.assertTrue(tjc.endpoint_base, 'jobs')
def test_post_job_collection(self):
"""Can add a treeherder collections to a TreeherderRequest."""
tjc = TreeherderJobCollection()
for job in self.job_data:
tjc.add(tjc.get_job(job))
client = TreeherderClient(
server_url='http://host',
client_id='client-abc',
secret='secret123',
)
def request_callback(request):
# Check that the expected content was POSTed.
posted_json = json.loads(request.body)
self.assertEqual(posted_json, tjc.get_collection_data())
return (200, {}, '{"message": "Job successfully updated"}')
url = client._get_endpoint_url(tjc.endpoint_base, project='project')
responses.add_callback(responses.POST, url, match_querystring=True,
callback=request_callback, content_type='application/json')
client.post_collection('project', tjc)
def completed_jobs_stored(test_repository, failure_classifications,
completed_jobs, push_stored):
"""
stores a list of buildapi completed jobs
"""
completed_jobs['revision'] = push_stored[0]['revision']
completed_jobs.update({'project': test_repository.name})
tjc = TreeherderJobCollection()
tj = tjc.get_job(completed_jobs)
tjc.add(tj)
store_job_data(test_repository, tjc.get_collection_data())
def running_jobs_stored(test_repository, failure_classifications, running_jobs,
push_stored):
"""
stores a list of buildapi running jobs
"""
running_jobs.update(push_stored[0])
running_jobs.update({'project': test_repository.name})
tjc = TreeherderJobCollection()
tj = tjc.get_job(running_jobs)
tjc.add(tj)
store_job_data(test_repository, tjc.get_collection_data())
def pending_jobs_stored(test_repository, failure_classifications, pending_jobs,
push_stored):
"""
stores a list of buildapi pending jobs into the jobs store
"""
pending_jobs.update(push_stored[0])
pending_jobs.update({'project': test_repository.name})
tjc = TreeherderJobCollection()
tj = tjc.get_job(pending_jobs)
tjc.add(tj)
store_job_data(test_repository, tjc.get_collection_data())
def transform(self,
data,
source,
revision_filter=None,
project_filter=None,
job_group_filter=None):
"""
transform the buildapi structure into something we can ingest via
our restful api
"""
valid_projects = set(Repository.objects.values_list('name', flat=True))
revision_dict = defaultdict(list)
# loop to catch all the revisions
for project, revisions in iteritems(data[source]):
if common.should_skip_project(project, valid_projects,
project_filter):
continue
for rev in revisions:
if common.should_skip_revision(rev, revision_filter):
continue
revision_dict[project].append(rev)
job_ids_seen_last_time = cache.get(CACHE_KEYS[source], set())
job_ids_seen_now = set()
th_collections = {}
for project, revisions in iteritems(data[source]):
if common.should_skip_project(project, valid_projects,
project_filter):
continue
revisions_seen_now_for_project = set()
for revision, jobs in revisions.items():
if common.should_skip_revision(revision, revision_filter):
continue
# it should be quite rare for a job to be ingested before a
# revision, but it could happen
if revision not in revisions_seen_now_for_project and \
not Push.objects.filter(repository__name=project,
revision__startswith=revision).exists():
logger.warning(
"skipping jobs since %s revision %s "
"not yet ingested", project, revision)
continue
revisions_seen_now_for_project.add(revision)
# using project and revision form the revision lookups
# to filter those jobs with unmatched revision
for job in jobs:
job_ids_seen_now.add(job['id'])
# Don't process jobs that we saw the last time this task
# completed successfully.
if job['id'] in job_ids_seen_last_time:
continue
treeherder_data = {
'revision': revision,
'project': project,
}
buildername = job['buildername']
platform_info = buildbot.extract_platform_info(buildername)
job_name_info = buildbot.extract_name_info(buildername)
if (job_group_filter
and job_name_info.get('group_symbol', '').lower()
!= job_group_filter.lower()):
continue
if source == 'pending':
request_id = job['id']
elif source == 'running':
# The last element in request_ids corresponds to the request id of this job,
# the others are for the requests that were superseded by this one.
request_id = job['request_ids'][-1]
new_job = {
'job_guid':
common.generate_job_guid(request_id, buildername),
'name':
job_name_info.get('name', ''),
'job_symbol':
job_name_info.get('job_symbol', ''),
'group_name':
job_name_info.get('group_name', ''),
'group_symbol':
job_name_info.get('group_symbol', ''),
'reference_data_name':
buildername,
'state':
source,
'submit_timestamp':
job['submitted_at'],
'build_platform': {
'os_name': platform_info['os'],
'platform': platform_info['os_platform'],
'architecture': platform_info['arch'],
},
# where are we going to get this data from?
'machine_platform': {
'os_name': platform_info['os'],
'platform': platform_info['os_platform'],
'architecture': platform_info['arch'],
},
'who':
'unknown',
'option_collection': {
# build_type contains an option name, eg. PGO
buildbot.extract_build_type(buildername):
True
},
'log_references': [],
'artifacts': [
{
'type': 'json',
'name': 'buildapi',
'log_urls': [],
'blob': {
'buildername': buildername,
'request_id': request_id
}
},
]
}
if source == 'running':
new_job['start_timestamp'] = job['start_time']
treeherder_data['job'] = new_job
if project not in th_collections:
th_collections[project] = TreeherderJobCollection()
# get treeherder job instance and add the job instance
# to the collection instance
th_job = th_collections[project].get_job(treeherder_data)
th_collections[project].add(th_job)
num_new_jobs = len(job_ids_seen_now.difference(job_ids_seen_last_time))
logger.info("Imported %d %s jobs, skipped %d previously seen",
num_new_jobs, source,
len(job_ids_seen_now) - num_new_jobs)
return th_collections, job_ids_seen_now
def transform(self,
data,
project_filter=None,
revision_filter=None,
job_group_filter=None):
"""
transform the builds4h structure into something we can ingest via
our restful api
"""
valid_projects = set(Repository.objects.values_list('name', flat=True))
for build in data['builds']:
try:
prop = build['properties']
project = prop['branch']
if common.should_skip_project(project, valid_projects,
project_filter):
continue
if common.should_skip_revision(prop['revision'],
revision_filter):
continue
except KeyError as e:
logger.warning(
"skipping builds-4hr job %s since missing property: %s",
build['id'], str(e))
continue
job_ids_seen_last_time = cache.get(CACHE_KEYS['complete'], set())
job_ids_seen_now = set()
revisions_seen_for_project = defaultdict(set)
# Holds one collection per unique branch/project
th_collections = {}
for build in data['builds']:
try:
prop = build['properties']
project = prop['branch']
buildername = prop['buildername']
if common.should_skip_project(project, valid_projects,
project_filter):
continue
if common.should_skip_revision(prop['revision'],
revision_filter):
continue
except KeyError:
continue
# it should be quite rare for a job to be ingested before a
# revision, but it could happen
revision = prop['revision']
if (revision not in revisions_seen_for_project[project]
and not Push.objects.filter(
repository__name=project,
revision__startswith=revision).exists()):
logger.warning(
"skipping jobs since %s revision %s "
"not yet ingested", project, revision)
continue
revisions_seen_for_project[project].add(revision)
# We record the id here rather than at the start of the loop, since we
# must not count jobs whose revisions were not yet imported as processed,
# or we'll never process them once we've ingested their associated revision.
job_ids_seen_now.add(build['id'])
# Don't process jobs that were already present in builds-4hr
# the last time this task completed successfully.
if build['id'] in job_ids_seen_last_time:
continue
platform_info = buildbot.extract_platform_info(buildername)
job_name_info = buildbot.extract_name_info(buildername)
if (job_group_filter and job_name_info.get(
'group_symbol', '').lower() != job_group_filter.lower()):
continue
treeherder_data = {
'revision': prop['revision'],
'project': project,
'superseded': []
}
log_reference = []
if 'log_url' in prop:
log_reference.append({
'url': prop['log_url'],
'name': 'buildbot_text'
})
# add structured logs to the list of log references
if 'blobber_files' in prop:
try:
blobber_files = json.loads(prop['blobber_files'])
for bf, url in blobber_files.items():
if bf and url and bf.endswith('_errorsummary.log'):
log_reference.append({
'url': url,
'name': 'errorsummary_json'
})
except Exception as e:
logger.warning(
"invalid blobber_files json for build id %s (%s): %s",
build['id'], buildername, e)
try:
job_guid_data = self.find_job_guid(build)
# request_ids is mandatory, but can be found in several places.
request_ids = prop.get('request_ids', build['request_ids'])
# The last element in request_ids corresponds to the request id of this job,
# the others are for the requests that were superseded by this one.
request_id = request_ids[-1]
except KeyError:
continue
treeherder_data['superseded'] = job_guid_data['superseded']
job = {
'job_guid':
job_guid_data['job_guid'],
'name':
job_name_info.get('name', ''),
'job_symbol':
job_name_info.get('job_symbol', ''),
'group_name':
job_name_info.get('group_name', ''),
'group_symbol':
job_name_info.get('group_symbol', ''),
'reference_data_name':
buildername,
'product_name':
prop.get('product', ''),
'state':
'completed',
'result':
buildbot.RESULT_DICT[build['result']],
'reason':
build['reason'],
# scheduler, if 'who' property is not present
'who':
prop.get('who', prop.get('scheduler', '')),
'submit_timestamp':
build['requesttime'],
'start_timestamp':
build['starttime'],
'end_timestamp':
build['endtime'],
'machine':
prop.get('slavename', 'unknown'),
# build_platform same as machine_platform
'build_platform': {
# platform attributes sometimes parse without results
'os_name': platform_info.get('os', ''),
'platform': platform_info.get('os_platform', ''),
'architecture': platform_info.get('arch', '')
},
'machine_platform': {
'os_name': platform_info.get('os', ''),
'platform': platform_info.get('os_platform', ''),
'architecture': platform_info.get('arch', '')
},
# pgo or non-pgo dependent on buildername parsing
'option_collection': {
buildbot.extract_build_type(buildername): True
},
'log_references':
log_reference,
'artifacts': [
{
'type': 'json',
'name': 'buildapi',
'log_urls': [],
'blob': {
'buildername': buildername,
'request_id': request_id
}
},
]
}
treeherder_data['job'] = job
if project not in th_collections:
th_collections[project] = TreeherderJobCollection()
# get treeherder job instance and add the job instance
# to the collection instance
th_job = th_collections[project].get_job(treeherder_data)
th_collections[project].add(th_job)
num_new_jobs = len(job_ids_seen_now.difference(job_ids_seen_last_time))
logger.info("Imported %d completed jobs, skipped %d previously seen",
num_new_jobs,
len(job_ids_seen_now) - num_new_jobs)
return th_collections, job_ids_seen_now
