def post_request(self, machine, project, job_collection, attempts, last_attempt):
logger.debug('AutophoneTreeherder.post_request: %s, attempt=%d, last=%s' %
(job_collection.__dict__, attempts, last_attempt))
client = TreeherderClient(protocol=self.protocol,
host=self.server,
client_id=self.client_id,
secret=self.secret)
try:
client.post_collection(project, job_collection)
return True
except Exception, e:
logger.exception('Error submitting request to Treeherder, attempt=%d, last=%s' %
(attempts, last_attempt))
if self.mailer:
if hasattr(e, 'response') and e.response:
response_json = json.dumps(e.response.json(),
indent=2, sort_keys=True)
else:
response_json = None
self.mailer.send(
'%s attempt %d Error submitting request to Treeherder' %
(utils.host(), attempts),
'Phone: %s\n'
'Exception: %s\n'
'Last attempt: %s\n'
'Response: %s\n' % (
machine,
e,
last_attempt,
response_json))
def submit(self,
revision,
browser,
timestamp,
perf_data,
link='',
version='',
repo_link='',
video_links='',
extra_info_obj={}):
j_dataset = self.create_job_dataset(revision=revision,
browser=browser,
timestamp=timestamp,
perf_data=perf_data,
link=link,
version=version,
repo_link=repo_link,
video_links=video_links,
extra_info_obj=extra_info_obj)
tjc = self.create_job_collection(j_dataset)
if self.server_url:
client = TreeherderClient(server_url=self.server_url,
client_id=self.client_id,
secret=self.secret)
else:
client = TreeherderClient(client_id=self.client_id,
secret=self.secret)
client.post_collection(self.repo, tjc)
def post_request(self, project, job_collection, guid=None):
self.logger.debug(type(self).__name__ + '.post_request - '
'job_collection =\n%s' %
pretty(job_collection.get_collection_data()))
client = TreeherderClient(protocol=self.protocol,
host=self.server,
client_id=self.credentials['client_id'],
secret=self.credentials['secret'])
for attempt in range(1, self.retries + 1):
try:
client.post_collection(project, job_collection)
self.logger.info(type(self).__name__ +
'.post_request - collection posted')
if guid:
job_url = self.request_job_url(project, guid)
self.logger.info(type(self).__name__ +
'.post_request - url is %s' % job_url)
return
except requests.exceptions.Timeout:
message = ('Attempt %d to post result to '
'Treeherder timed out.' % attempt)
self.logger.error(message)
time.sleep(self.retry_wait)
except Exception as e:
message = ('Error submitting request to Treeherder\n\n'
'Exception: %s\n'
'TreeherderJobCollection %s\n' %
(e, pretty(job_collection.get_collection_data())))
self.logger.exception(message)
return
self.logger.error('Error submitting request to Treeherder.')
def post_request(self, project, job_collection, guid=None):
self.logger.debug(type(self).__name__ + '.post_request - '
'job_collection =\n%s' %
pretty(job_collection.get_collection_data()))
client = TreeherderClient(protocol=self.protocol,
host=self.server,
client_id=self.credentials['client_id'],
secret=self.credentials['secret'])
for attempt in range(1, self.retries + 1):
try:
client.post_collection(project, job_collection)
self.logger.info(type(self).__name__ +
'.post_request - collection posted')
if guid:
job_url = self.request_job_url(project, guid)
self.logger.info(type(self).__name__ +
'.post_request - url is %s' % job_url)
return
except requests.exceptions.Timeout:
message = ('Attempt %d to post result to '
'Treeherder timed out.' % attempt)
self.logger.error(message)
time.sleep(self.retry_wait)
except Exception as e:
message = ('Error submitting request to Treeherder\n\n'
'Exception: %s\n'
'TreeherderJobCollection %s\n' %
(e, pretty(job_collection.get_collection_data())))
self.logger.exception(message)
return
self.logger.error('Error submitting request to Treeherder.')
def submit(self, job, logs=None):
logs = logs or []
# We can only submit job info once, so it has to be done in completed
if self._job_details:
job.add_artifact('Job Info', 'json',
{'job_details': self._job_details})
job_collection = TreeherderJobCollection()
job_collection.add(job)
print('Sending results to Treeherder: {}'.format(
job_collection.to_json()))
url = urlparse(self.url)
client = TreeherderClient(protocol=url.scheme,
host=url.hostname,
client_id=self.client_id,
secret=self.secret)
client.post_collection(self.repository, job_collection)
print('Results are available to view at: {}'.format(
urljoin(
self.url,
JOB_FRAGMENT.format(repository=self.repository,
revision=self.revision))))
def submit(self, job):
"""Submit the job to treeherder.
:param job: Treeherder job instance to use for submission.
"""
job.add_submit_timestamp(int(time.time()))
# We can only submit job info once, so it has to be done in completed
if self._job_details:
job.add_artifact('Job Info', 'json', {'job_details': self._job_details})
job_collection = TreeherderJobCollection()
job_collection.add(job)
logger.info('Sending results to Treeherder: {}'.format(job_collection.to_json()))
url = urlparse(self.url)
client = TreeherderClient(protocol=url.scheme, host=url.hostname,
client_id=self.client_id, secret=self.secret)
client.post_collection(self.repository, job_collection)
logger.info('Results are available to view at: {}'.format(
urljoin(self.url,
JOB_FRAGMENT.format(repository=self.repository,
revision=self.revision))))
def submit(self,
revision,
browser,
timestamp,
perf_data,
version='',
repo_link='',
video_links='',
extra_info_obj={}):
j_dataset = self.create_job_dataset(revision=revision,
browser=browser,
timestamp=timestamp,
perf_data=perf_data,
version=version,
repo_link=repo_link,
video_links=video_links,
extra_info_obj=extra_info_obj)
tjc = self.create_job_collection(j_dataset)
if self.server_url:
client = TreeherderClient(server_url=self.server_url,
client_id=self.client_id,
secret=self.secret)
else:
client = TreeherderClient(client_id=self.client_id,
secret=self.secret)
try:
return_result = client.post_collection(self.repo, tjc)
except Exception as e:
print e.message
print traceback.print_exc()
return None
return return_result
def submit(self, revision, browser, timestamp, perf_data, version='', repo_link='', video_links='', extra_info_obj={}):
j_dataset = self.create_job_dataset(revision=revision,
browser=browser,
timestamp=timestamp,
perf_data=perf_data,
version=version,
repo_link=repo_link,
video_links=video_links,
extra_info_obj=extra_info_obj)
tjc = self.create_job_collection(j_dataset)
if self.server_url:
client = TreeherderClient(server_url=self.server_url,
client_id=self.client_id,
secret=self.secret)
else:
client = TreeherderClient(client_id=self.client_id,
secret=self.secret)
try:
return_result = client.post_collection(self.repo, tjc)
except Exception as e:
print e.message
print traceback.print_exc()
return None
return return_result
def post_request(self, machine, project, job_collection):
logger.debug('AutophoneTreeherder.post_request: %s' % job_collection.__dict__)
logger.debug('AutophoneTreeherder shared_lock.acquire')
self.shared_lock.acquire()
try:
client = TreeherderClient(protocol=self.protocol, host=self.server)
for attempt in range(1, self.retries+1):
try:
client.post_collection(
project,
self.credentials[project]['consumer_key'],
self.credentials[project]['consumer_secret'],
job_collection)
return
except requests.exceptions.Timeout:
msg = ('Attempt %d to post result to '
'Treeherder timed out.\n\n\n' % attempt)
logger.error(msg)
if self.mailer:
self.mailer.send('Attempt %d for Phone %s failed to post to Treeherder' %
(attempt, machine), msg)
time.sleep(self.retry_wait)
except Exception, e:
logger.exception('Error submitting request to Treeherder')
if self.mailer:
self.mailer.send('Error submitting request to Treeherder',
'Phone: %s\n'
'TreeherderClientError: %s\n'
'TreeherderJobCollection %s\n' % (
machine,
e,
job_collection.to_json()))
return
logger.error('Error submitting request to Treeherder')
if self.mailer:
self.mailer.send('Error submitting request to Treeherder',
'Phone: %s\n'
'TreeherderClientError: %s\n'
'TreeherderJobCollection %s\n' % (
machine,
e,
job_collection.to_json()))
def post_request(self, machine, project, job_collection):
logger.debug('AutophoneTreeherder.post_request: %s' % job_collection.__dict__)
logger.debug('AutophoneTreeherder shared_lock.acquire')
self.shared_lock.acquire()
try:
auth = TreeherderAuth(self.credentials[project]['consumer_key'],
self.credentials[project]['consumer_secret'],
project)
client = TreeherderClient(protocol=self.protocol, host=self.server, auth=auth)
for attempt in range(1, self.retries+1):
try:
client.post_collection(project, job_collection)
return
except Exception, e:
logger.exception('Error submitting request to Treeherder')
if self.mailer:
if e.response:
response_json = json.dumps(e.response.json(),
indent=2, sort_keys=True)
else:
response_json = None
self.mailer.send(
'Attempt %d Error submitting request to Treeherder' %
attempt,
'Phone: %s\n'
'TreeherderClientError: %s\n'
'Response: %s\n' % (
machine,
e,
response_json))
time.sleep(self.retry_wait)
logger.error('Error submitting request to Treeherder')
if self.mailer:
self.mailer.send('Error submitting request to Treeherder',
'Phone: %s\n'
'TreeherderClientError: %s\n'
'Response: %s\n'
'TreeherderJobCollection %s\n' % (
machine,
e,
response_json,
job_collection.to_json()))
def submit_results(self, job):
job.add_project(self.project)
job.add_revision_hash(self.retrieve_revision_hash())
job.add_submit_timestamp(int(time.time()))
job_collection = TreeherderJobCollection()
job_collection.add(job)
# self.logger.info
print('Sending results to Treeherder: %s' % job_collection.to_json())
url = urlparse(self.url)
client = TreeherderClient(protocol=url.scheme, host=url.hostname,
client_id=self.client_id, secret=self.secret)
client.post_collection(self.project, job_collection)
# self.logger.info
print('Results are available to view at: %s' % (
urljoin(self.url,
REVISON_FRAGMENT % (self.project, self.revision))))
def submit(self, job, logs=None):
logs = logs or []
# We can only submit job info once, so it has to be done in completed
if self._job_details:
job.add_artifact('Job Info', 'json', {'job_details': self._job_details})
job_collection = TreeherderJobCollection()
job_collection.add(job)
print('Sending results to Treeherder: {}'.format(job_collection.to_json()))
url = urlparse(self.url)
client = TreeherderClient(protocol=url.scheme, host=url.hostname,
client_id=self.client_id, secret=self.secret)
client.post_collection(self.repository, job_collection)
print('Results are available to view at: {}'.format(
urljoin(self.url,
JOB_FRAGMENT.format(repository=self.repository, revision=self.revision))))
def submit(self, revision, browser, timestamp, perf_data, link='', version='', repo_link='', video_links='', extra_info_obj={}):
j_dataset = self.create_job_dataset(revision=revision,
browser=browser,
timestamp=timestamp,
perf_data=perf_data,
link=link,
version=version,
repo_link=repo_link,
video_links=video_links,
extra_info_obj=extra_info_obj)
tjc = self.create_job_collection(j_dataset)
if self.server_url:
client = TreeherderClient(server_url=self.server_url,
client_id=self.client_id,
secret=self.secret)
else:
client = TreeherderClient(client_id=self.client_id,
secret=self.secret)
client.post_collection(self.repo, tjc)
def submit(self, revision, browser, timestamp, perf_data, link='', version='', repo_link='', video_links='', extra_info_obj={}):
# rs_dataset = self.create_resultset_dataset(revision=revision,
# timestamp=timestamp)
# trsc = self.create_resultset_collection(rs_dataset)
j_dataset = self.create_job_dataset(revision=revision,
browser=browser,
timestamp=timestamp,
perf_data=perf_data,
link=link,
version=version,
repo_link=repo_link,
video_links=video_links,
extra_info_obj=extra_info_obj)
tjc = self.create_job_collection(j_dataset)
client = TreeherderClient(protocol=self.potocol,
host=self.host,
client_id=self.client_id,
secret=self.secret)
# don't post resultset, that overwrites existing data. see: https://bugzilla.mozilla.org/show_bug.cgi?id=1320694
# client.post_collection(self.repo, trsc)
client.post_collection(self.repo, tjc)
def submit(perf_data, failures, revision, summary, engine):
print("[DEBUG] failures:")
print(list(map(lambda x: x['testcase'], failures)))
author = "{} <{}>".format(revision['author']['name'],
revision['author']['email'])
dataset = [{
# The top-most revision in the list of commits for a push.
'revision':
revision['commit'],
'author':
author,
'push_timestamp':
int(revision['author']['timestamp']),
'type':
'push',
# a list of revisions associated with the resultset. There should
# be at least one.
'revisions': [{
'comment': revision['subject'],
'revision': revision['commit'],
'repository': 'servo',
'author': author
}]
}]
trsc = create_resultset_collection(dataset)
result = "success"
# TODO: verify a failed test won't affect Perfherder visualization
# if len(failures) > 0:
# result = "testfailed"
hashlen = len(revision['commit'])
job_guid = ''.join(
random.choice(string.ascii_letters + string.digits)
for i in range(hashlen))
if (engine == "gecko"):
project = "servo"
job_symbol = 'PLG'
group_symbol = 'SPG'
group_name = 'Servo Perf on Gecko'
else:
project = "servo"
job_symbol = 'PL'
group_symbol = 'SP'
group_name = 'Servo Perf'
dataset = [{
'project': project,
'revision': revision['commit'],
'job': {
'job_guid':
job_guid,
'product_name':
project,
'reason':
'scheduler',
# TODO: What is `who` for?
'who':
'Servo',
'desc':
'Servo Page Load Time Tests',
'name':
'Servo Page Load Time',
# The symbol representing the job displayed in
# treeherder.allizom.org
'job_symbol':
job_symbol,
# The symbol representing the job group in
# treeherder.allizom.org
'group_symbol':
group_symbol,
'group_name':
group_name,
# TODO: get the real timing from the test runner
'submit_timestamp':
str(int(time.time())),
'start_timestamp':
str(int(time.time())),
'end_timestamp':
str(int(time.time())),
'state':
'completed',
'result':
result, # "success" or "testfailed"
'machine':
'local-machine',
# TODO: read platform from test result
'build_platform': {
'platform': 'linux64',
'os_name': 'linux',
'architecture': 'x86_64'
},
'machine_platform': {
'platform': 'linux64',
'os_name': 'linux',
'architecture': 'x86_64'
},
'option_collection': {
'opt': True
},
# jobs can belong to different tiers
# setting the tier here will determine which tier the job
# belongs to. However, if a job is set as Tier of 1, but
# belongs to the Tier 2 profile on the server, it will still
# be saved as Tier 2.
'tier':
1,
# the ``name`` of the log can be the default of "buildbot_text"
# however, you can use a custom name. See below.
# TODO: point this to the log when we have them uploaded to S3
'log_references': [{
'url': 'TBD',
'name': 'test log'
}],
# The artifact can contain any kind of structured data
# associated with a test.
'artifacts': [
{
'type': 'json',
'name': 'performance_data',
# TODO: include the job_guid when the runner actually
# generates one
# 'job_guid': job_guid,
'blob': perf_data
},
{
'type': 'json',
'name': 'Job Info',
# 'job_guid': job_guid,
"blob": {
"job_details": [{
"content_type": "link",
"url": "https://www.github.com/servo/servo",
"value": "GitHub",
"title": "Source code"
}, {
"content_type": "raw_html",
"title": "Result Summary",
"value": summary
}]
}
}
],
# List of job guids that were coalesced to this job
'coalesced': []
}
}]
tjc = create_job_collection(dataset)
# TODO: extract this read credential code out of this function.
cred = {
'client_id': os.environ['TREEHERDER_CLIENT_ID'],
'secret': os.environ['TREEHERDER_CLIENT_SECRET']
}
client = TreeherderClient(server_url='https://treeherder.mozilla.org',
client_id=cred['client_id'],
secret=cred['secret'])
# data structure validation is automatically performed here, if validation
# fails a TreeherderClientError is raised
client.post_collection('servo', trsc)
client.post_collection('servo', tjc)
class AutophoneTreeherder(object):
def __init__(self, worker_subprocess, options, jobs, s3_bucket=None,
mailer=None):
assert options, "options is required."
logger = utils.getLogger()
self.options = options
self.jobs = jobs
self.s3_bucket = s3_bucket
self.mailer = mailer
self.worker = worker_subprocess
self.shutdown_requested = False
logger.debug('AutophoneTreeherder')
self.url = self.options.treeherder_url
if not self.url:
logger.debug('AutophoneTreeherder: no treeherder url')
return
self.client_id = self.options.treeherder_client_id
self.secret = self.options.treeherder_secret
self.retry_wait = self.options.treeherder_retry_wait
self.client = TreeherderClient(server_url=self.url,
client_id=self.client_id,
secret=self.secret)
logger.debug('AutophoneTreeherder: %s', self)
def __str__(self):
# Do not publish sensitive information
whitelist = ('url',
'retry_wait')
d = {}
for attr in whitelist:
d[attr] = getattr(self, attr)
return '%s' % d
def post_request(self, machine, project, job_collection, attempts, last_attempt):
logger = utils.getLogger()
logger.debug('AutophoneTreeherder.post_request: %s, attempt=%d, last=%s',
job_collection.__dict__, attempts, last_attempt)
try:
self.client.post_collection(project, job_collection)
return True
except Exception, e:
logger.exception('Error submitting request to Treeherder, attempt=%d, last=%s',
attempts, last_attempt)
if attempts > 1 and self.mailer:
if hasattr(e, 'response') and e.response:
response_json = json.dumps(e.response.json(),
indent=2, sort_keys=True)
else:
response_json = None
request_len = len(job_collection.to_json())
self.mailer.send(
'%s attempt %d Error submitting request to Treeherder' %
(utils.host(), attempts),
'Phone: %s\n'
'Exception: %s\n'
'Last attempt: %s\n'
'Request length: %d\n'
'Response: %s\n' % (
machine,
e,
last_attempt,
request_len,
response_json))
return False
def submit(perf_data, failures, revision, summary, engine):
print("[DEBUG] failures:")
print(list(map(lambda x: x['testcase'], failures)))
# TODO: read the correct guid from test result
author = "{} <{}>".format(revision['author']['name'],
revision['author']['email'])
# XXX: Hack! make gecko data points shown on the same date as it submitted
# if (engine == "gecko"):
# revision['author']['timestamp'] = str(int(time.time()))
dataset = [
{
# The top-most revision in the list of commits for a push.
'revision': revision['commit'],
'author': author,
'push_timestamp': int(revision['author']['timestamp']),
'type': 'push',
# a list of revisions associated with the resultset. There should
# be at least one.
'revisions': [
{
'comment': revision['subject'],
'revision': revision['commit'],
'repository': 'servo',
'author': author
}
]
}
]
trsc = create_resultset_collection(dataset)
result = "success"
# FIXME: Always passing until https://bugzil.la/1276178 is fixed
# if len(failures) > 0:
# result = "testfailed"
hashlen = len(revision['commit'])
# job_guid = "x" * hashlen
job_guid = ''.join(
random.choice(string.ascii_letters + string.digits) for i in range(hashlen)
)
if (engine == "gecko"):
# project = "mozilla-release"
# TODO: figure out how to use mozilla-release as project
project = "servo"
job_symbol = 'PLG'
group_symbol = 'SPG'
group_name = 'Servo Perf on Gecko'
else:
project = "servo"
job_symbol = 'PL'
group_symbol = 'SP'
group_name = 'Servo Perf'
dataset = [
{
'project': project,
'revision': revision['commit'],
'job': {
'job_guid': job_guid,
'product_name': project,
'reason': 'scheduler',
# TODO:What is `who` for?
'who': 'Servo',
'desc': 'Servo Page Load Time Tests',
'name': 'Servo Page Load Time',
# The symbol representing the job displayed in
# treeherder.allizom.org
'job_symbol': job_symbol,
# The symbol representing the job group in
# treeherder.allizom.org
'group_symbol': group_symbol,
'group_name': group_name,
# TODO: get the real timing from the test runner
'submit_timestamp': str(int(time.time())),
'start_timestamp': str(int(time.time())),
'end_timestamp': str(int(time.time())),
'state': 'completed',
'result': result, # "success" or "testfailed"
'machine': 'local-machine',
# TODO: read platform test result
'build_platform': {
'platform': 'linux64',
'os_name': 'linux',
'architecture': 'x86_64'
},
'machine_platform': {
'platform': 'linux64',
'os_name': 'linux',
'architecture': 'x86_64'
},
'option_collection': {'opt': True},
# jobs can belong to different tiers
# setting the tier here will determine which tier the job
# belongs to. However, if a job is set as Tier of 1, but
# belongs to the Tier 2 profile on the server, it will still
# be saved as Tier 2.
'tier': 1,
# the ``name`` of the log can be the default of "buildbot_text"
# however, you can use a custom name. See below.
# TODO: point this to the log when we have them uploaded
'log_references': [
{
'url': 'TBD',
'name': 'test log'
}
],
# The artifact can contain any kind of structured data
# associated with a test.
'artifacts': [
{
'type': 'json',
'name': 'performance_data',
# 'job_guid': job_guid,
'blob': perf_data
# {
# "performance_data": {
# # that is not `talos`?
# "framework": {"name": "talos"},
# "suites": [{
# "name": "performance.timing.domComplete",
# "value": random.choice(range(15,25)),
# "subtests": [
# {"name": "responseEnd", "value": 123},
# {"name": "loadEventEnd", "value": 223}
# ]
# }]
# }
# }
},
{
'type': 'json',
'name': 'Job Info',
# 'job_guid': job_guid,
"blob": {
"job_details": [
{
"content_type": "link",
"url": "https://www.github.com/servo/servo",
"value": "GitHub",
"title": "Source code"
},
{
"content_type": "raw_html",
"title": "Result Summary",
"value": summary
}
]
}
}
],
# List of job guids that were coalesced to this job
'coalesced': []
}
}
]
tjc = create_job_collection(dataset)
# TODO: extract this read credential code out of this function.
with open('credential.json', 'r') as f:
cred = json.load(f)
client = TreeherderClient(protocol='https',
host='treeherder.allizom.org',
# protocol='http',
# host='local.treeherder.mozilla.org',
client_id=cred['client_id'],
secret=cred['secret'])
# data structure validation is automatically performed here, if validation
# fails a TreeherderClientError is raised
client.post_collection('servo', trsc)
client.post_collection('servo', tjc)
def post_to_treeherder(self, tests):
version = mozversion.get_version(
binary=self.bin, sources=self.sources,
dm_type='adb', device_serial=self.device_serial)
job_collection = TreeherderJobCollection()
job = job_collection.get_job()
device = version.get('device_id')
device_firmware_version_release = \
version.get('device_firmware_version_release')
if not device:
self.logger.error('Submitting to Treeherder is currently limited '
'to devices.')
return
try:
group = DEVICE_GROUP_MAP[device][device_firmware_version_release]
job.add_group_name(group['name'])
job.add_group_symbol(group['symbol'])
job.add_job_name('Gaia Python Integration Test (%s)' % group['symbol'])
job.add_job_symbol('Gip')
except KeyError:
self.logger.error('Unknown device id: %s or device firmware '
'version: %s. Unable to determine Treeherder '
'group. Supported devices: %s'
% (device, device_firmware_version_release,
['%s: %s' % (k, [fw for fw in v.keys()])
for k, v in DEVICE_GROUP_MAP.iteritems()]))
return
# Determine revision hash from application revision
revision = version['application_changeset']
project = version['application_repository'].split('/')[-1]
lookup_url = urljoin(
self.treeherder_url,
'api/project/%s/revision-lookup/?revision=%s' % (
project, revision))
self.logger.debug('Getting revision hash from: %s' % lookup_url)
response = requests.get(lookup_url)
response.raise_for_status()
assert response.json(), 'Unable to determine revision hash for %s. ' \
'Perhaps it has not been ingested by ' \
'Treeherder?' % revision
revision_hash = response.json()[revision]['revision_hash']
job.add_revision_hash(revision_hash)
job.add_project(project)
job.add_job_guid(str(uuid.uuid4()))
job.add_product_name('b2g')
job.add_state('completed')
# Determine test result
if self.failed or self.unexpected_successes:
job.add_result('testfailed')
else:
job.add_result('success')
job.add_submit_timestamp(int(self.start_time))
job.add_start_timestamp(int(self.start_time))
job.add_end_timestamp(int(self.end_time))
job.add_machine(socket.gethostname())
job.add_build_info('b2g', 'b2g-device-image', 'x86')
job.add_machine_info('b2g', 'b2g-device-image', 'x86')
# All B2G device builds are currently opt builds
job.add_option_collection({'opt': True})
date_format = '%d %b %Y %H:%M:%S'
job_details = [{
'content_type': 'link',
'title': 'Gaia revision:',
'url': 'https://github.com/mozilla-b2g/gaia/commit/%s' %
version.get('gaia_changeset'),
'value': version.get('gaia_changeset'),
}, {
'content_type': 'text',
'title': 'Gaia date:',
'value': version.get('gaia_date') and time.strftime(
date_format, time.localtime(int(version.get('gaia_date')))),
}, {
'content_type': 'text',
'title': 'Device identifier:',
'value': version.get('device_id')
}, {
'content_type': 'text',
'title': 'Device firmware (date):',
'value': version.get('device_firmware_date') and time.strftime(
date_format, time.localtime(int(
version.get('device_firmware_date')))),
}, {
'content_type': 'text',
'title': 'Device firmware (incremental):',
'value': version.get('device_firmware_version_incremental')
}, {
'content_type': 'text',
'title': 'Device firmware (release):',
'value': version.get('device_firmware_version_release')
}]
ci_url = os.environ.get('BUILD_URL')
if ci_url:
job_details.append({
'url': ci_url,
'value': ci_url,
'content_type': 'link',
'title': 'CI build:'})
# Attach logcat
adb_device = ADBDevice(self.device_serial)
with tempfile.NamedTemporaryFile(suffix='logcat.txt') as f:
f.writelines(adb_device.get_logcat())
self.logger.debug('Logcat stored in: %s' % f.name)
try:
url = self.upload_to_s3(f.name)
job_details.append({
'url': url,
'value': 'logcat.txt',
'content_type': 'link',
'title': 'Log:'})
except S3UploadError:
job_details.append({
'value': 'Failed to upload logcat.txt',
'content_type': 'text',
'title': 'Error:'})
# Attach log files
handlers = [handler for handler in self.logger.handlers
if isinstance(handler, StreamHandler) and
os.path.exists(handler.stream.name)]
for handler in handlers:
path = handler.stream.name
filename = os.path.split(path)[-1]
try:
url = self.upload_to_s3(path)
job_details.append({
'url': url,
'value': filename,
'content_type': 'link',
'title': 'Log:'})
# Add log reference
if type(handler.formatter) is TbplFormatter or \
type(handler.formatter) is LogLevelFilter and \
type(handler.formatter.inner) is TbplFormatter:
job.add_log_reference(filename, url)
except S3UploadError:
job_details.append({
'value': 'Failed to upload %s' % filename,
'content_type': 'text',
'title': 'Error:'})
# Attach reports
for report in [self.html_output]:
if report is not None:
filename = os.path.split(report)[-1]
try:
url = self.upload_to_s3(report)
job_details.append({
'url': url,
'value': filename,
'content_type': 'link',
'title': 'Report:'})
except S3UploadError:
job_details.append({
'value': 'Failed to upload %s' % filename,
'content_type': 'text',
'title': 'Error:'})
if job_details:
job.add_artifact('Job Info', 'json', {'job_details': job_details})
job_collection.add(job)
# Send the collection to Treeherder
url = urlparse(self.treeherder_url)
client = TreeherderClient(protocol=url.scheme, host=url.hostname)
self.logger.debug('Sending results to Treeherder: %s' %
job_collection.to_json())
client.post_collection(project, os.environ.get('TREEHERDER_KEY'),
os.environ.get('TREEHERDER_SECRET'), job_collection)
self.logger.info('Results are available to view at: %s' % (
urljoin(self.treeherder_url, '/ui/#/jobs?repo=%s&revision=%s' % (
project, revision))))
def submit(perf_data, revision):
print("[DEBUG] performance data:")
print(perf_data)
# TODO: read the correct guid from test result
hashlen = len(revision['commit'])
job_guid = ''.join(
random.choice(string.letters + string.digits) for i in xrange(hashlen)
)
trsc = TreeherderResultSetCollection()
author = "{} <{}>".format(revision['author']['name'],
revision['author']['email'])
dataset = [
{
# The top-most revision in the list of commits for a push.
'revision': revision['commit'],
'author': author,
'push_timestamp': int(revision['author']['timestamp']),
'type': 'push',
# a list of revisions associated with the resultset. There should
# be at least one.
'revisions': [
{
'comment': revision['subject'],
'revision': revision['commit'],
'repository': 'servo',
'author': author
}
]
}
]
for data in dataset:
trs = trsc.get_resultset()
trs.add_push_timestamp(data['push_timestamp'])
trs.add_revision(data['revision'])
trs.add_author(data['author'])
# trs.add_type(data['type'])
revisions = []
for rev in data['revisions']:
tr = trs.get_revision()
tr.add_revision(rev['revision'])
tr.add_author(rev['author'])
tr.add_comment(rev['comment'])
tr.add_repository(rev['repository'])
revisions.append(tr)
trs.add_revisions(revisions)
trsc.add(trs)
dataset = [
{
'project': 'servo',
'revision': revision['commit'],
'job': {
'job_guid': job_guid,
'product_name': 'servo',
'reason': 'scheduler',
# TODO:What is `who` for?
'who': 'Servo',
'desc': 'Servo Page Load Time Tests',
'name': 'Servo Page Load Time',
# The symbol representing the job displayed in
# treeherder.allizom.org
'job_symbol': 'PL',
# The symbol representing the job group in
# treeherder.allizom.org
'group_symbol': 'SP',
'group_name': 'Servo Perf',
# TODO: get the real timing from the test runner
'submit_timestamp': revision['author']['timestamp'],
'start_timestamp': revision['author']['timestamp'],
'end_timestamp': revision['author']['timestamp'],
'state': 'completed',
'result': 'success',
'machine': 'local-machine',
# TODO: read platform test result
'build_platform': {
'platform': 'linux64',
'os_name': 'linux',
'architecture': 'x86_64'
},
'machine_platform': {
'platform': 'linux64',
'os_name': 'linux',
'architecture': 'x86_64'
},
'option_collection': {'opt': True},
# jobs can belong to different tiers
# setting the tier here will determine which tier the job
# belongs to. However, if a job is set as Tier of 1, but
# belongs to the Tier 2 profile on the server, it will still
# be saved as Tier 2.
'tier': 1,
# the ``name`` of the log can be the default of "buildbot_text"
# however, you can use a custom name. See below.
# TODO: point this to the log when we have them uploaded
'log_references': [
{
'url': 'TBD',
'name': 'test log'
}
],
# The artifact can contain any kind of structured data
# associated with a test.
'artifacts': [
{
'type': 'json',
'name': 'performance_data',
# 'job_guid': job_guid,
'blob': perf_data
# {
# "performance_data": {
# # that is not `talos`?
# "framework": {"name": "talos"},
# "suites": [{
# "name": "performance.timing.domComplete",
# "value": random.choice(range(15,25)),
# "subtests": [
# {"name": "responseEnd", "value": 123},
# {"name": "loadEventEnd", "value": 223}
# ]
# }]
# }
# }
},
{
'type': 'json',
'name': 'Job Info',
# 'job_guid': job_guid,
"blob": {
"job_details": [
{
"url": "https://www.github.com/servo/servo",
"value": "website",
"content_type": "link",
"title": "Source code"
}
]
}
}
],
# List of job guids that were coalesced to this job
'coalesced': []
}
}
]
tjc = TreeherderJobCollection()
for data in dataset:
tj = tjc.get_job()
tj.add_revision(data['revision'])
tj.add_project(data['project'])
tj.add_coalesced_guid(data['job']['coalesced'])
tj.add_job_guid(data['job']['job_guid'])
tj.add_job_name(data['job']['name'])
tj.add_job_symbol(data['job']['job_symbol'])
tj.add_group_name(data['job']['group_name'])
tj.add_group_symbol(data['job']['group_symbol'])
tj.add_description(data['job']['desc'])
tj.add_product_name(data['job']['product_name'])
tj.add_state(data['job']['state'])
tj.add_result(data['job']['result'])
tj.add_reason(data['job']['reason'])
tj.add_who(data['job']['who'])
tj.add_tier(data['job']['tier'])
tj.add_submit_timestamp(data['job']['submit_timestamp'])
tj.add_start_timestamp(data['job']['start_timestamp'])
tj.add_end_timestamp(data['job']['end_timestamp'])
tj.add_machine(data['job']['machine'])
tj.add_build_info(
data['job']['build_platform']['os_name'],
data['job']['build_platform']['platform'],
data['job']['build_platform']['architecture']
)
tj.add_machine_info(
data['job']['machine_platform']['os_name'],
data['job']['machine_platform']['platform'],
data['job']['machine_platform']['architecture']
)
tj.add_option_collection(data['job']['option_collection'])
# for log_reference in data['job']['log_references']:
# tj.add_log_reference( 'buildbot_text', log_reference['url'])
# data['artifact'] is a list of artifacts
for artifact_data in data['job']['artifacts']:
tj.add_artifact(
artifact_data['name'],
artifact_data['type'],
artifact_data['blob']
)
tjc.add(tj)
# TODO: extract this read credential code out of this function.
with open('credential.json', 'rb') as f:
cred = json.load(f)
client = TreeherderClient(protocol='https',
# host='local.treeherder.mozilla.org',
host='treeherder.allizom.org',
client_id=cred['client_id'],
secret=cred['secret'])
# data structure validation is automatically performed here, if validation
# fails a TreeherderClientError is raised
client.post_collection('servo', trsc)
client.post_collection('servo', tjc)
class Submission(object):
"""Class for submitting reports to Treeherder."""
def __init__(self, repository, revision, settings,
treeherder_url, treeherder_client_id, treeherder_secret):
"""Creates new instance of the submission class.
:param repository: Name of the repository the build has been built from.
:param revision: Changeset of the repository the build has been built from.
:param settings: Settings for the Treeherder job as retrieved from the config file.
:param treeherder_url: URL of the Treeherder instance.
:param treeherder_client_id: The client ID necessary for the Hawk authentication.
:param treeherder_secret: The secret key necessary for the Hawk authentication.
"""
self.repository = repository
self.revision = revision
self.settings = settings
self._job_details = []
self.client = TreeherderClient(server_url=treeherder_url,
client_id=treeherder_client_id,
secret=treeherder_secret)
def _get_treeherder_platform(self):
"""Returns the Treeherder equivalent platform identifier of the current platform."""
platform = None
info = mozinfo.info
if info['os'] == 'linux':
platform = ('linux', '%s%s' % (info['os'], info['bits']), '%s' % info['processor'])
elif info['os'] == 'mac':
platform = ('mac', 'osx-%s' % info['os_version'].replace('.', '-'), info['processor'])
elif info['os'] == 'win':
versions = {'5.1': 'xp', '6.1': '7', '6.2': '8'}
bits = ('-%s' % info['bits']) if info['os_version'] != '5.1' else ''
platform = ('win', 'windows%s%s' % (versions[info['os_version']], '%s' % bits),
info['processor'],
)
return platform
def create_job(self, data=None, **kwargs):
"""Creates a new instance of a Treeherder job for submission.
:param data: Job data to use for initilization, e.g. from a previous submission, optional
:param kwargs: Dictionary of necessary values to build the job details. The
properties correlate to the placeholders in config.py.
"""
data = data or {}
job = TreeherderJob(data=data)
# If no data is available we have to set all properties
if not data:
job.add_job_guid(str(uuid.uuid4()))
job.add_tier(self.settings['treeherder']['tier'])
job.add_product_name('firefox')
job.add_project(self.repository)
job.add_revision(self.revision)
# Add platform and build information
job.add_machine(socket.getfqdn())
platform = self._get_treeherder_platform()
job.add_machine_info(*platform)
job.add_build_info(*platform)
# TODO debug or others?
job.add_option_collection({'opt': True})
# TODO: Add e10s group once we run those tests
job.add_group_name(self.settings['treeherder']['group_name'].format(**kwargs))
job.add_group_symbol(self.settings['treeherder']['group_symbol'].format(**kwargs))
# Bug 1174973 - for now we need unique job names even in different groups
job.add_job_name(self.settings['treeherder']['job_name'].format(**kwargs))
job.add_job_symbol(self.settings['treeherder']['job_symbol'].format(**kwargs))
job.add_start_timestamp(int(time.time()))
# Bug 1175559 - Workaround for HTTP Error
job.add_end_timestamp(0)
return job
@retriable(sleeptime=30, jitter=0)
def submit(self, job):
"""Submit the job to treeherder.
:param job: Treeherder job instance to use for submission.
"""
job.add_submit_timestamp(int(time.time()))
if self._job_details:
job.add_artifact('Job Info', 'json',
{'job_details': copy.deepcopy(self._job_details)})
self._job_details = []
job_collection = TreeherderJobCollection()
job_collection.add(job)
logger.info('Sending results to Treeherder: {}'.format(job_collection.to_json()))
self.client.post_collection(self.repository, job_collection)
logger.info('Results are available to view at: {}'.format(
urljoin('{0}://{1}'.format(self.client.protocol, self.client.host),
JOB_FRAGMENT.format(repository=self.repository,
revision=self.revision))))
def submit_running_job(self, job):
"""Submit job as state running.
:param job: Treeherder job instance to use for submission.
"""
job.add_state('running')
if os.environ.get('BUILD_URL'):
self._job_details.append({
'title': 'Inspect Jenkins Build (VPN required)',
'value': os.environ['BUILD_URL'],
'content_type': 'link',
'url': os.environ['BUILD_URL']
})
self.submit(job)
def submit_completed_job(self, job, retval, uploaded_logs):
"""Submit job as state completed.
:param job: Treeherder job instance to use for submission.
:param retval: Return value of the build process to determine build state.
:param uploaded_logs: List of uploaded logs to reference in the job.
"""
job.add_state('completed')
job.add_result(BuildExitCode[retval])
job.add_end_timestamp(int(time.time()))
# Add reference to the log which will be parsed by Treeherder
log_reference = uploaded_logs.get(self.settings['treeherder']['log_reference'])
if log_reference:
job.add_log_reference(name='buildbot_text', url=log_reference.get('url'))
# Add all uploaded logs as artifacts
for log in uploaded_logs:
self._job_details.append({
'title': log,
'value': uploaded_logs[log]['url'],
'content_type': 'link',
'url': uploaded_logs[log]['url'],
})
self.submit(job)
def submit(perf_data, failures, revision, summary, engine):
print("[DEBUG] failures:")
print(list(map(lambda x: x['testcase'], failures)))
author = "{} <{}>".format(revision['author']['name'],
revision['author']['email'])
dataset = [
{
# The top-most revision in the list of commits for a push.
'revision': revision['commit'],
'author': author,
'push_timestamp': int(revision['author']['timestamp']),
'type': 'push',
# a list of revisions associated with the resultset. There should
# be at least one.
'revisions': [
{
'comment': revision['subject'],
'revision': revision['commit'],
'repository': 'servo',
'author': author
}
]
}
]
trsc = create_resultset_collection(dataset)
result = "success"
# TODO: verify a failed test won't affect Perfherder visualization
# if len(failures) > 0:
# result = "testfailed"
hashlen = len(revision['commit'])
job_guid = ''.join(
random.choice(string.ascii_letters + string.digits) for i in range(hashlen)
)
if (engine == "gecko"):
project = "servo"
job_symbol = 'PLG'
group_symbol = 'SPG'
group_name = 'Servo Perf on Gecko'
else:
project = "servo"
job_symbol = 'PL'
group_symbol = 'SP'
group_name = 'Servo Perf'
dataset = [
{
'project': project,
'revision': revision['commit'],
'job': {
'job_guid': job_guid,
'product_name': project,
'reason': 'scheduler',
# TODO: What is `who` for?
'who': 'Servo',
'desc': 'Servo Page Load Time Tests',
'name': 'Servo Page Load Time',
# The symbol representing the job displayed in
# treeherder.allizom.org
'job_symbol': job_symbol,
# The symbol representing the job group in
# treeherder.allizom.org
'group_symbol': group_symbol,
'group_name': group_name,
# TODO: get the real timing from the test runner
'submit_timestamp': str(int(time.time())),
'start_timestamp': str(int(time.time())),
'end_timestamp': str(int(time.time())),
'state': 'completed',
'result': result, # "success" or "testfailed"
'machine': 'local-machine',
# TODO: read platform from test result
'build_platform': {
'platform': 'linux64',
'os_name': 'linux',
'architecture': 'x86_64'
},
'machine_platform': {
'platform': 'linux64',
'os_name': 'linux',
'architecture': 'x86_64'
},
'option_collection': {'opt': True},
# jobs can belong to different tiers
# setting the tier here will determine which tier the job
# belongs to. However, if a job is set as Tier of 1, but
# belongs to the Tier 2 profile on the server, it will still
# be saved as Tier 2.
'tier': 1,
# the ``name`` of the log can be the default of "buildbot_text"
# however, you can use a custom name. See below.
# TODO: point this to the log when we have them uploaded to S3
'log_references': [
{
'url': 'TBD',
'name': 'test log'
}
],
# The artifact can contain any kind of structured data
# associated with a test.
'artifacts': [
{
'type': 'json',
'name': 'performance_data',
# TODO: include the job_guid when the runner actually
# generates one
# 'job_guid': job_guid,
'blob': perf_data
},
{
'type': 'json',
'name': 'Job Info',
# 'job_guid': job_guid,
"blob": {
"job_details": [
{
"content_type": "raw_html",
"title": "Result Summary",
"value": summary
}
]
}
}
],
# List of job guids that were coalesced to this job
'coalesced': []
}
}
]
tjc = create_job_collection(dataset)
# TODO: extract this read credential code out of this function.
cred = {
'client_id': os.environ['TREEHERDER_CLIENT_ID'],
'secret': os.environ['TREEHERDER_CLIENT_SECRET']
}
client = TreeherderClient(server_url='https://treeherder.mozilla.org',
client_id=cred['client_id'],
secret=cred['secret'])
# data structure validation is automatically performed here, if validation
# fails a TreeherderClientError is raised
client.post_collection('servo', trsc)
client.post_collection('servo', tjc)
def post_to_treeherder(self, tests):
version = mozversion.get_version(
binary=self.bin, sources=self.sources,
dm_type='adb', device_serial=self.device_serial)
job_collection = TreeherderJobCollection()
job = job_collection.get_job()
device = version.get('device_id')
device_firmware_version_release = \
version.get('device_firmware_version_release')
if not device:
self.logger.error('Submitting to Treeherder is currently limited '
'to devices.')
return
try:
group = DEVICE_GROUP_MAP[device][device_firmware_version_release]
job.add_group_name(group['name'])
job.add_group_symbol(group['symbol'])
job.add_job_name('Gaia Python Integration Test (%s)' % group['symbol'])
job.add_job_symbol('Gip')
except KeyError:
self.logger.error('Unknown device id: %s or device firmware '
'version: %s. Unable to determine Treeherder '
'group. Supported devices: %s'
% (device, device_firmware_version_release,
['%s: %s' % (k, [fw for fw in v.keys()])
for k, v in DEVICE_GROUP_MAP.iteritems()]))
return
# Determine revision hash from application revision
revision = version['application_changeset']
project = version['application_repository'].split('/')[-1]
lookup_url = urljoin(
self.treeherder_url,
'api/project/%s/revision-lookup/?revision=%s' % (
project, revision))
self.logger.debug('Getting revision hash from: %s' % lookup_url)
response = requests.get(lookup_url)
response.raise_for_status()
assert response.json(), 'Unable to determine revision hash for %s. ' \
'Perhaps it has not been ingested by ' \
'Treeherder?' % revision
revision_hash = response.json()[revision]['revision_hash']
job.add_revision_hash(revision_hash)
job.add_project(project)
job.add_job_guid(str(uuid.uuid4()))
job.add_product_name('b2g')
job.add_state('completed')
# Determine test result
if self.failed or self.unexpected_successes:
job.add_result('testfailed')
else:
job.add_result('success')
job.add_submit_timestamp(int(self.start_time))
job.add_start_timestamp(int(self.start_time))
job.add_end_timestamp(int(self.end_time))
job.add_machine(socket.gethostname())
job.add_build_info('b2g', 'b2g-device-image', 'x86')
job.add_machine_info('b2g', 'b2g-device-image', 'x86')
# All B2G device builds are currently opt builds
job.add_option_collection({'opt': True})
date_format = '%d %b %Y %H:%M:%S'
job_details = [{
'content_type': 'link',
'title': 'Gaia revision:',
'url': 'https://github.com/mozilla-b2g/gaia/commit/%s' %
version.get('gaia_changeset'),
'value': version.get('gaia_changeset'),
}, {
'content_type': 'text',
'title': 'Gaia date:',
'value': version.get('gaia_date') and time.strftime(
date_format, time.localtime(int(version.get('gaia_date')))),
}, {
'content_type': 'text',
'title': 'Device identifier:',
'value': version.get('device_id')
}, {
'content_type': 'text',
'title': 'Device firmware (date):',
'value': version.get('device_firmware_date') and time.strftime(
date_format, time.localtime(int(
version.get('device_firmware_date')))),
}, {
'content_type': 'text',
'title': 'Device firmware (incremental):',
'value': version.get('device_firmware_version_incremental')
}, {
'content_type': 'text',
'title': 'Device firmware (release):',
'value': version.get('device_firmware_version_release')
}]
ci_url = os.environ.get('BUILD_URL')
if ci_url:
job_details.append({
'url': ci_url,
'value': ci_url,
'content_type': 'link',
'title': 'CI build:'})
# Attach logcat
adb_device = ADBDevice(self.device_serial)
with tempfile.NamedTemporaryFile(suffix='logcat.txt') as f:
f.writelines(adb_device.get_logcat())
self.logger.debug('Logcat stored in: %s' % f.name)
try:
url = self.upload_to_s3(f.name)
job_details.append({
'url': url,
'value': 'logcat.txt',
'content_type': 'link',
'title': 'Log:'})
except S3UploadError:
job_details.append({
'value': 'Failed to upload logcat.txt',
'content_type': 'text',
'title': 'Error:'})
# Attach log files
handlers = [handler for handler in self.logger.handlers
if isinstance(handler, StreamHandler) and
os.path.exists(handler.stream.name)]
for handler in handlers:
path = handler.stream.name
filename = os.path.split(path)[-1]
try:
url = self.upload_to_s3(path)
job_details.append({
'url': url,
'value': filename,
'content_type': 'link',
'title': 'Log:'})
# Add log reference
if type(handler.formatter) is TbplFormatter or \
type(handler.formatter) is LogLevelFilter and \
type(handler.formatter.inner) is TbplFormatter:
job.add_log_reference(filename, url)
except S3UploadError:
job_details.append({
'value': 'Failed to upload %s' % filename,
'content_type': 'text',
'title': 'Error:'})
# Attach reports
for report in [self.html_output]:
if report is not None:
filename = os.path.split(report)[-1]
try:
url = self.upload_to_s3(report)
job_details.append({
'url': url,
'value': filename,
'content_type': 'link',
'title': 'Report:'})
except S3UploadError:
job_details.append({
'value': 'Failed to upload %s' % filename,
'content_type': 'text',
'title': 'Error:'})
if job_details:
job.add_artifact('Job Info', 'json', {'job_details': job_details})
job_collection.add(job)
# Send the collection to Treeherder
url = urlparse(self.treeherder_url)
client = TreeherderClient(protocol=url.scheme, host=url.hostname)
self.logger.debug('Sending results to Treeherder: %s' %
job_collection.to_json())
client.post_collection(project, os.environ.get('TREEHERDER_KEY'),
os.environ.get('TREEHERDER_SECRET'), job_collection)
self.logger.info('Results are available to view at: %s' % (
urljoin(self.treeherder_url, '/ui/#/jobs?repo=%s&revision=%s' % (
project, revision))))
class AutophoneTreeherder(object):
def __init__(self,
worker_subprocess,
options,
jobs,
s3_bucket=None,
mailer=None):
assert options, "options is required."
logger = utils.getLogger()
self.options = options
self.jobs = jobs
self.s3_bucket = s3_bucket
self.mailer = mailer
self.worker = worker_subprocess
self.shutdown_requested = False
logger.debug('AutophoneTreeherder')
self.url = self.options.treeherder_url
if not self.url:
logger.debug('AutophoneTreeherder: no treeherder url')
return
self.client_id = self.options.treeherder_client_id
self.secret = self.options.treeherder_secret
self.retry_wait = self.options.treeherder_retry_wait
self.client = TreeherderClient(server_url=self.url,
client_id=self.client_id,
secret=self.secret)
logger.debug('AutophoneTreeherder: %s', self)
def __str__(self):
# Do not publish sensitive information
whitelist = ('url', 'retry_wait')
d = {}
for attr in whitelist:
d[attr] = getattr(self, attr)
return '%s' % d
def post_request(self, machine, project, job_collection, attempts,
last_attempt):
logger = utils.getLogger()
logger.debug(
'AutophoneTreeherder.post_request: %s, attempt=%d, last=%s',
job_collection.__dict__, attempts, last_attempt)
try:
self.client.post_collection(project, job_collection)
return True
except Exception, e:
logger.exception(
'Error submitting request to Treeherder, attempt=%d, last=%s',
attempts, last_attempt)
if attempts > 1 and self.mailer:
if hasattr(e, 'response') and e.response:
response_json = json.dumps(e.response.json(),
indent=2,
sort_keys=True)
else:
response_json = None
request_len = len(job_collection.to_json())
self.mailer.send(
'%s attempt %d Error submitting request to Treeherder' %
(utils.host(), attempts), 'Phone: %s\n'
'Exception: %s\n'
'Last attempt: %s\n'
'Request length: %d\n'
'Response: %s\n' %
(machine, e, last_attempt, request_len, response_json))
return False
def submit(perf_data, failures, revision, summary, engine):
print("[DEBUG] failures:")
print(list(map(lambda x: x["testcase"], failures)))
author = "{} <{}>".format(revision["author"]["name"], revision["author"]["email"])
dataset = [
{
# The top-most revision in the list of commits for a push.
"revision": revision["commit"],
"author": author,
"push_timestamp": int(revision["author"]["timestamp"]),
"type": "push",
# a list of revisions associated with the resultset. There should
# be at least one.
"revisions": [
{
"comment": revision["subject"],
"revision": revision["commit"],
"repository": "servo",
"author": author,
}
],
}
]
trsc = create_resultset_collection(dataset)
result = "success"
# TODO: verify a failed test won't affect Perfherder visualization
# if len(failures) > 0:
# result = "testfailed"
hashlen = len(revision["commit"])
job_guid = "".join(random.choice(string.ascii_letters + string.digits) for i in range(hashlen))
if engine == "gecko":
project = "servo"
job_symbol = "PLG"
group_symbol = "SPG"
group_name = "Servo Perf on Gecko"
else:
project = "servo"
job_symbol = "PL"
group_symbol = "SP"
group_name = "Servo Perf"
dataset = [
{
"project": project,
"revision": revision["commit"],
"job": {
"job_guid": job_guid,
"product_name": project,
"reason": "scheduler",
# TODO: What is `who` for?
"who": "Servo",
"desc": "Servo Page Load Time Tests",
"name": "Servo Page Load Time",
# The symbol representing the job displayed in
# treeherder.allizom.org
"job_symbol": job_symbol,
# The symbol representing the job group in
# treeherder.allizom.org
"group_symbol": group_symbol,
"group_name": group_name,
# TODO: get the real timing from the test runner
"submit_timestamp": str(int(time.time())),
"start_timestamp": str(int(time.time())),
"end_timestamp": str(int(time.time())),
"state": "completed",
"result": result, # "success" or "testfailed"
"machine": "local-machine",
# TODO: read platform from test result
"build_platform": {"platform": "linux64", "os_name": "linux", "architecture": "x86_64"},
"machine_platform": {"platform": "linux64", "os_name": "linux", "architecture": "x86_64"},
"option_collection": {"opt": True},
# jobs can belong to different tiers
# setting the tier here will determine which tier the job
# belongs to. However, if a job is set as Tier of 1, but
# belongs to the Tier 2 profile on the server, it will still
# be saved as Tier 2.
"tier": 1,
# the ``name`` of the log can be the default of "buildbot_text"
# however, you can use a custom name. See below.
# TODO: point this to the log when we have them uploaded to S3
"log_references": [{"url": "TBD", "name": "test log"}],
# The artifact can contain any kind of structured data
# associated with a test.
"artifacts": [
{
"type": "json",
"name": "performance_data",
# TODO: include the job_guid when the runner actually
# generates one
# 'job_guid': job_guid,
"blob": perf_data,
},
{
"type": "json",
"name": "Job Info",
# 'job_guid': job_guid,
"blob": {
"job_details": [
{
"content_type": "link",
"url": "https://www.github.com/servo/servo",
"value": "GitHub",
"title": "Source code",
},
{"content_type": "raw_html", "title": "Result Summary", "value": summary},
]
},
},
],
# List of job guids that were coalesced to this job
"coalesced": [],
},
}
]
tjc = create_job_collection(dataset)
# TODO: extract this read credential code out of this function.
cred = {"client_id": os.environ["TREEHERDER_CLIENT_ID"], "secret": os.environ["TREEHERDER_CLIENT_SECRET"]}
client = TreeherderClient(
server_url="https://treeherder.mozilla.org", client_id=cred["client_id"], secret=cred["secret"]
)
# data structure validation is automatically performed here, if validation
# fails a TreeherderClientError is raised
client.post_collection("servo", trsc)
client.post_collection("servo", tjc)
def submit(perf_data, failures, revision, summary, engine):
print("[DEBUG] failures:")
print(list(map(lambda x: x['testcase'], failures)))
# TODO: read the correct guid from test result
author = "{} <{}>".format(revision['author']['name'],
revision['author']['email'])
# XXX: Hack! make gecko data points shown on the same date as it submitted
# if (engine == "gecko"):
# revision['author']['timestamp'] = str(int(time.time()))
dataset = [{
# The top-most revision in the list of commits for a push.
'revision':
revision['commit'],
'author':
author,
'push_timestamp':
int(revision['author']['timestamp']),
'type':
'push',
# a list of revisions associated with the resultset. There should
# be at least one.
'revisions': [{
'comment': revision['subject'],
'revision': revision['commit'],
'repository': 'servo',
'author': author
}]
}]
trsc = create_resultset_collection(dataset)
result = "success"
# FIXME: Always passing until https://bugzil.la/1276178 is fixed
# if len(failures) > 0:
# result = "testfailed"
hashlen = len(revision['commit'])
# job_guid = "x" * hashlen
job_guid = ''.join(
random.choice(string.ascii_letters + string.digits)
for i in range(hashlen))
if (engine == "gecko"):
# project = "mozilla-release"
# TODO: figure out how to use mozilla-release as project
project = "servo"
job_symbol = 'PLG'
group_symbol = 'SPG'
group_name = 'Servo Perf on Gecko'
else:
project = "servo"
job_symbol = 'PL'
group_symbol = 'SP'
group_name = 'Servo Perf'
dataset = [{
'project': project,
'revision': revision['commit'],
'job': {
'job_guid':
job_guid,
'product_name':
project,
'reason':
'scheduler',
# TODO:What is `who` for?
'who':
'Servo',
'desc':
'Servo Page Load Time Tests',
'name':
'Servo Page Load Time',
# The symbol representing the job displayed in
# treeherder.allizom.org
'job_symbol':
job_symbol,
# The symbol representing the job group in
# treeherder.allizom.org
'group_symbol':
group_symbol,
'group_name':
group_name,
# TODO: get the real timing from the test runner
'submit_timestamp':
str(int(time.time())),
'start_timestamp':
str(int(time.time())),
'end_timestamp':
str(int(time.time())),
'state':
'completed',
'result':
result, # "success" or "testfailed"
'machine':
'local-machine',
# TODO: read platform test result
'build_platform': {
'platform': 'linux64',
'os_name': 'linux',
'architecture': 'x86_64'
},
'machine_platform': {
'platform': 'linux64',
'os_name': 'linux',
'architecture': 'x86_64'
},
'option_collection': {
'opt': True
},
# jobs can belong to different tiers
# setting the tier here will determine which tier the job
# belongs to. However, if a job is set as Tier of 1, but
# belongs to the Tier 2 profile on the server, it will still
# be saved as Tier 2.
'tier':
1,
# the ``name`` of the log can be the default of "buildbot_text"
# however, you can use a custom name. See below.
# TODO: point this to the log when we have them uploaded
'log_references': [{
'url': 'TBD',
'name': 'test log'
}],
# The artifact can contain any kind of structured data
# associated with a test.
'artifacts': [
{
'type': 'json',
'name': 'performance_data',
# 'job_guid': job_guid,
'blob': perf_data
# {
# "performance_data": {
# # that is not `talos`?
# "framework": {"name": "talos"},
# "suites": [{
# "name": "performance.timing.domComplete",
# "value": random.choice(range(15,25)),
# "subtests": [
# {"name": "responseEnd", "value": 123},
# {"name": "loadEventEnd", "value": 223}
# ]
# }]
# }
# }
},
{
'type': 'json',
'name': 'Job Info',
# 'job_guid': job_guid,
"blob": {
"job_details": [{
"content_type": "link",
"url": "https://www.github.com/servo/servo",
"value": "GitHub",
"title": "Source code"
}, {
"content_type": "raw_html",
"title": "Result Summary",
"value": summary
}]
}
}
],
# List of job guids that were coalesced to this job
'coalesced': []
}
}]
tjc = create_job_collection(dataset)
# TODO: extract this read credential code out of this function.
with open('credential.json', 'r') as f:
cred = json.load(f)
client = TreeherderClient(
protocol='https',
host='treeherder.allizom.org',
# protocol='http',
# host='local.treeherder.mozilla.org',
client_id=cred['client_id'],
secret=cred['secret'])
# data structure validation is automatically performed here, if validation
# fails a TreeherderClientError is raised
client.post_collection('servo', trsc)
client.post_collection('servo', tjc)
class Submission(object):
"""Class for submitting reports to Treeherder."""
def __init__(self, repository, revision, settings, treeherder_url,
treeherder_client_id, treeherder_secret):
"""Creates new instance of the submission class.
:param repository: Name of the repository the build has been built from.
:param revision: Changeset of the repository the build has been built from.
:param settings: Settings for the Treeherder job as retrieved from the config file.
:param treeherder_url: URL of the Treeherder instance.
:param treeherder_client_id: The client ID necessary for the Hawk authentication.
:param treeherder_secret: The secret key necessary for the Hawk authentication.
"""
self.repository = repository
self.revision = revision
self.settings = settings
self._job_details = []
self.client = TreeherderClient(server_url=treeherder_url,
client_id=treeherder_client_id,
secret=treeherder_secret)
def _get_treeherder_platform(self):
"""Returns the Treeherder equivalent platform identifier of the current platform."""
platform = None
info = mozinfo.info
if info['os'] == 'linux':
platform = ('linux', '%s%s' % (info['os'], info['bits']),
'%s' % info['processor'])
elif info['os'] == 'mac':
platform = ('mac', 'osx-%s' % info['os_version'].replace('.', '-'),
info['processor'])
elif info['os'] == 'win':
versions = {'5.1': 'xp', '6.1': '7', '6.2': '8'}
bits = ('-%s' %
info['bits']) if info['os_version'] != '5.1' else ''
platform = (
'win',
'windows%s%s' % (versions[info['os_version']], '%s' % bits),
info['processor'],
)
return platform
def create_job(self, data=None, **kwargs):
"""Creates a new instance of a Treeherder job for submission.
:param data: Job data to use for initilization, e.g. from a previous submission, optional
:param kwargs: Dictionary of necessary values to build the job details. The
properties correlate to the placeholders in config.py.
"""
data = data or {}
job = TreeherderJob(data=data)
# If no data is available we have to set all properties
if not data:
job.add_job_guid(str(uuid.uuid4()))
job.add_tier(self.settings['treeherder']['tier'])
job.add_product_name('firefox')
job.add_project(self.repository)
job.add_revision(self.revision)
# Add platform and build information
job.add_machine(socket.getfqdn())
platform = self._get_treeherder_platform()
job.add_machine_info(*platform)
job.add_build_info(*platform)
# TODO debug or others?
job.add_option_collection({'opt': True})
# TODO: Add e10s group once we run those tests
job.add_group_name(
self.settings['treeherder']['group_name'].format(**kwargs))
job.add_group_symbol(
self.settings['treeherder']['group_symbol'].format(**kwargs))
# Bug 1174973 - for now we need unique job names even in different groups
job.add_job_name(
self.settings['treeherder']['job_name'].format(**kwargs))
job.add_job_symbol(
self.settings['treeherder']['job_symbol'].format(**kwargs))
job.add_start_timestamp(int(time.time()))
# Bug 1175559 - Workaround for HTTP Error
job.add_end_timestamp(0)
return job
@retriable(sleeptime=30, jitter=0)
def submit(self, job):
"""Submit the job to treeherder.
:param job: Treeherder job instance to use for submission.
"""
job.add_submit_timestamp(int(time.time()))
if self._job_details:
job.add_artifact('Job Info', 'json',
{'job_details': copy.deepcopy(self._job_details)})
self._job_details = []
job_collection = TreeherderJobCollection()
job_collection.add(job)
logger.info('Sending results to Treeherder: {}'.format(
job_collection.to_json()))
self.client.post_collection(self.repository, job_collection)
logger.info('Results are available to view at: {}'.format(
urljoin(
self.client.server_url,
JOB_FRAGMENT.format(repository=self.repository,
revision=self.revision))))
def submit_running_job(self, job):
"""Submit job as state running.
:param job: Treeherder job instance to use for submission.
"""
job.add_state('running')
if os.environ.get('BUILD_URL'):
self._job_details.append({
'title': 'Inspect Jenkins Build (VPN required)',
'value': os.environ['BUILD_URL'],
'content_type': 'link',
'url': os.environ['BUILD_URL']
})
self.submit(job)
def submit_completed_job(self, job, retval, uploaded_logs):
"""Submit job as state completed.
:param job: Treeherder job instance to use for submission.
:param retval: Return value of the build process to determine build state.
:param uploaded_logs: List of uploaded logs to reference in the job.
"""
job.add_state('completed')
job.add_result(BuildExitCode[retval])
job.add_end_timestamp(int(time.time()))
# Add reference to the log which will be parsed by Treeherder
log_reference = uploaded_logs.get(
self.settings['treeherder']['log_reference'])
if log_reference:
job.add_log_reference(name='buildbot_text',
url=log_reference.get('url'))
# Add all uploaded logs as artifacts
for log in uploaded_logs:
self._job_details.append({
'title': log,
'value': uploaded_logs[log]['url'],
'content_type': 'link',
'url': uploaded_logs[log]['url'],
})
self.submit(job)
