def send_request(self):
"""
Send request to Elasticsearch.
"""
es_url = "%s/bugs/bug_info/" % settings.ES_HOST
logger.info("Submitting %s job %s's classification of bug %s to Elasticsearch", self.project, self.job_id, self.bug_id)
try:
make_request(es_url, method='POST', json=self.body)
except requests.exceptions.HTTPError as e:
r = e.response
logger.error("HTTPError %s submitting to %s: %s", r.status_code, es_url, r.text)
raise
def send_request(self):
"""
Submit classification report to Elasticsearch, via OrangeFactor's API.
"""
url = settings.ORANGEFACTOR_SUBMISSION_URL
auth = HawkAuth(id=settings.ORANGEFACTOR_HAWK_ID, key=settings.ORANGEFACTOR_HAWK_KEY)
logger.info("Submitting %s job %s's classification of bug %s to OrangeFactor", self.project, self.job_id, self.bug_id)
try:
make_request(url, method='POST', json=self.body, auth=auth)
except requests.exceptions.HTTPError as e:
r = e.response
logger.error("HTTPError %s submitting to %s: %s", r.status_code, url, r.text)
raise
def send_request(self):
"""
Submit classification report to Elasticsearch, via OrangeFactor's API.
"""
url = settings.ORANGEFACTOR_SUBMISSION_URL
auth = HawkAuth(id=settings.ORANGEFACTOR_HAWK_ID, key=settings.ORANGEFACTOR_HAWK_KEY)
logger.info("Submitting %s job %s's classification of bug %s to OrangeFactor", self.project, self.job_id, self.bug_id)
try:
make_request(url, method='POST', json=self.body, auth=auth)
except requests.exceptions.HTTPError as e:
r = e.response
logger.error("HTTPError %s submitting to %s: %s", r.status_code, url, r.text)
raise
def send_request(self):
"""
Send request to Elasticsearch.
"""
es_url = "%s/bugs/bug_info/" % settings.ES_HOST
logger.info(
"Submitting %s job %s's classification of bug %s to Elasticsearch",
self.project, self.job_id, self.bug_id)
try:
make_request(es_url, method='POST', json=self.body)
except requests.exceptions.HTTPError as e:
r = e.response
logger.error("HTTPError %s submitting to %s: %s", r.status_code,
es_url, r.text)
raise
def create_bug(self, request):
"""
Create a bugzilla bug with passed params
"""
if settings.BZ_API_KEY is None:
return Response({"failure": "Bugzilla API key not defined. This shouldn't happen."}, status=status.HTTP_400_BAD_REQUEST)
else:
params = request.data
url = settings.BZ_API_URL + "/rest/bug"
headers = {
'x-bugzilla-api-key': settings.BZ_API_KEY
}
data = {
'product': params["product"],
'component': params["component"],
'summary': params["summary"],
'keywords': params["keywords"],
'version': params["version"],
'description': "Filed by: " + request.user.username + "\n\n" + params["description"],
'comment_tags': "treeherder",
}
try:
response = make_request(url, method='POST', headers=headers, json=data)
except requests.exceptions.HTTPError as e:
response = e.response
try:
rsperror = response.json()['message']
except:
rsperror = response
return Response({"failure": rsperror}, status=status.HTTP_400_BAD_REQUEST)
return Response({"success": response.json()["id"]})
def parse(self):
"""
Iterate over each line of the log, running each parser against it.
Stream lines from the gzip file and run each parser against it,
building the ``artifact`` as we go.
"""
with closing(make_request(self.url, stream=True)) as response:
# Temporary annotation of log size to help set thresholds in bug 1295997.
newrelic.agent.add_custom_parameter(
'unstructured_log_size',
int(response.headers.get('Content-Length', -1))
)
newrelic.agent.add_custom_parameter(
'unstructured_log_encoding',
response.headers.get('Content-Encoding', 'None')
)
for line in response.iter_lines():
for builder in self.builders:
builder.parse_line(line)
# gather the artifacts from all builders
for builder in self.builders:
# Run end-of-parsing actions for this parser,
# in case the artifact needs clean-up/summarising.
builder.finish_parse()
name = builder.name
artifact = builder.get_artifact()
if name == 'performance_data' and not artifact[name]:
continue
self.artifacts[name] = artifact
def create_bug(self, request):
"""
Create a bugzilla bug with passed params
"""
if settings.BUGFILER_API_KEY is None:
return Response({"failure": "Bugzilla API key not set!"},
status=HTTP_400_BAD_REQUEST)
params = request.data
# Arbitrarily cap crash signatures at 2048 characters to prevent perf issues on bmo
crash_signature = params.get("crash_signature")
if crash_signature and len(crash_signature) > 2048:
return Response(
{
"failure":
"Crash signature can't be more than 2048 characters."
},
status=HTTP_400_BAD_REQUEST)
description = u"**Filed by:** {}\n{}".format(
request.user.email.replace('@', " [at] "),
params.get("comment", "")).encode("utf-8")
summary = params.get("summary").encode("utf-8").strip()
url = settings.BUGFILER_API_URL + "/rest/bug"
headers = {
'x-bugzilla-api-key': settings.BUGFILER_API_KEY,
'Accept': 'application/json'
}
data = {
'type': "defect",
'product': params.get("product"),
'component': params.get("component"),
'summary': summary,
'keywords': params.get("keywords"),
'blocks': params.get("blocks"),
'depends_on': params.get("depends_on"),
'see_also': params.get("see_also"),
'version': params.get("version"),
'cf_crash_signature': params.get("crash_signature"),
'severity': params.get("severity"),
'priority': params.get("priority"),
'description': description,
'comment_tags': "treeherder",
}
try:
response = make_request(url,
method='POST',
headers=headers,
json=data)
except requests.exceptions.HTTPError as e:
try:
message = e.response.json()['message']
except (ValueError, KeyError):
message = e.response.text
return Response({"failure": message}, status=HTTP_400_BAD_REQUEST)
return Response({"success": response.json()["id"]})
def create_bug(self, request):
"""
Create a bugzilla bug with passed params
"""
if settings.BZ_API_KEY is None:
return Response(
{
"failure":
"Bugzilla API key not defined. This shouldn't happen."
},
status=status.HTTP_400_BAD_REQUEST)
else:
params = request.data
url = settings.BZ_API_URL + "/rest/bug"
headers = {
'x-bugzilla-api-key': settings.BZ_API_KEY,
'Accept': 'application/json'
}
data = {
'product':
params["product"],
'component':
params["component"],
'summary':
params["summary"],
'keywords':
params["keywords"],
'version':
params["version"],
'description':
"Filed by: " + request.user.username + "\n\n" +
params["description"],
'comment_tags':
"treeherder",
}
try:
response = make_request(url,
method='POST',
headers=headers,
json=data)
except requests.exceptions.HTTPError as e:
response = e.response
try:
rsperror = response.json()['message']
except:
rsperror = response
return Response({"failure": rsperror},
status=status.HTTP_400_BAD_REQUEST)
return Response({"success": response.json()["id"]})
def create_bug(self, request):
"""
Create a bugzilla bug with passed params
"""
if settings.BUGFILER_API_KEY is None:
return Response({"failure": "Bugzilla API key not set!"},
status=HTTP_400_BAD_REQUEST)
params = request.data
# Arbitrarily cap crash signatures at 2048 characters to prevent perf issues on bmo
crash_signature = params.get("crash_signature")
if crash_signature and len(crash_signature) > 2048:
return Response({"failure": "Crash signature can't be more than 2048 characters."},
status=HTTP_400_BAD_REQUEST)
description = u"**Filed by:** {}\n{}".format(
request.user.email.replace('@', " [at] "),
params.get("comment", "")
).encode("utf-8")
summary = params.get("summary").encode("utf-8").strip()
url = settings.BUGFILER_API_URL + "/rest/bug"
headers = {
'x-bugzilla-api-key': settings.BUGFILER_API_KEY,
'Accept': 'application/json'
}
data = {
'product': params.get("product"),
'component': params.get("component"),
'summary': summary,
'keywords': params.get("keywords"),
'blocks': params.get("blocks"),
'depends_on': params.get("depends_on"),
'see_also': params.get("see_also"),
'version': params.get("version"),
'cf_crash_signature': params.get("crash_signature"),
'severity': params.get("severity"),
'priority': params.get("priority"),
'description': description,
'comment_tags': "treeherder",
}
try:
response = make_request(url, method='POST', headers=headers, json=data)
except requests.exceptions.HTTPError as e:
try:
message = e.response.json()['message']
except (ValueError, KeyError):
message = e.response.text
return Response({"failure": message}, status=HTTP_400_BAD_REQUEST)
return Response({"success": response.json()["id"]})
def parse(self):
"""
Iterate over each line of the log, running each parser against it.
Stream lines from the gzip file and run each parser against it,
building the ``artifact`` as we go.
"""
with make_request(self.url, stream=True) as response:
download_size_in_bytes = int(
response.headers.get('Content-Length', -1))
# Temporary annotation of log size to help set thresholds in bug 1295997.
newrelic.agent.add_custom_parameter('unstructured_log_size',
download_size_in_bytes)
newrelic.agent.add_custom_parameter(
'unstructured_log_encoding',
response.headers.get('Content-Encoding', 'None'))
if download_size_in_bytes > MAX_DOWNLOAD_SIZE_IN_BYTES:
raise LogSizeException(
'Download size of %i bytes exceeds limit' %
download_size_in_bytes)
# Lines must be explicitly decoded since `iter_lines()`` returns bytes by default
# and we cannot use its `decode_unicode=True` mode, since otherwise Unicode newline
# characters such as `\u0085` (which can appear in test output) are treated the same
# as `\n` or `\r`, and so split into unwanted additional lines by `iter_lines()`.
for line in response.iter_lines():
for builder in self.builders:
try:
# Using `replace` to prevent malformed unicode (which might possibly exist
# in test message output) from breaking parsing of the rest of the log.
builder.parse_line(line.decode('utf-8', 'replace'))
except EmptyPerformanceData:
logger.warning(
"We have parsed an empty PERFHERDER_DATA for %s",
self.url)
# gather the artifacts from all builders
for builder in self.builders:
# Run end-of-parsing actions for this parser,
# in case the artifact needs clean-up/summarising.
builder.finish_parse()
name = builder.name
artifact = builder.get_artifact()
if name == 'performance_data' and not artifact[name]:
continue
self.artifacts[name] = artifact
def create_bug(self, request):
"""
Create a bugzilla bug with passed params
"""
if settings.BZ_API_KEY is None:
return Response(
{
"failure":
"Bugzilla API key not defined. This shouldn't happen."
},
status=HTTP_400_BAD_REQUEST)
params = request.data
description = "Filed by: {}\n\n{}".format(
request.user.email.replace('@', " [at] "),
params.get("comment", ""))
url = settings.BZ_API_URL + "/rest/bug"
headers = {
'x-bugzilla-api-key': settings.BZ_API_KEY,
'Accept': 'application/json'
}
data = {
'product': params.get("product"),
'component': params.get("component"),
'summary': params.get("summary"),
'keywords': params.get("keywords"),
'blocks': params.get("blocks"),
'depends_on': params.get("depends_on"),
'see_also': params.get("see_also"),
'version': params.get("version"),
'description': description,
'comment_tags': "treeherder",
}
try:
response = make_request(url,
method='POST',
headers=headers,
json=data)
except requests.exceptions.HTTPError as e:
try:
message = e.response.json()['message']
except (ValueError, KeyError):
message = e.response.text
return Response({"failure": message}, status=HTTP_400_BAD_REQUEST)
return Response({"success": response.json()["id"]})
def parse(self):
"""
Iterate over each line of the log, running each parser against it.
Stream lines from the gzip file and run each parser against it,
building the ``artifact`` as we go.
"""
with make_request(self.url, stream=True) as response:
download_size_in_bytes = int(response.headers.get('Content-Length', -1))
# Temporary annotation of log size to help set thresholds in bug 1295997.
newrelic.agent.add_custom_parameter(
'unstructured_log_size',
download_size_in_bytes
)
newrelic.agent.add_custom_parameter(
'unstructured_log_encoding',
response.headers.get('Content-Encoding', 'None')
)
if download_size_in_bytes > MAX_DOWNLOAD_SIZE_IN_BYTES:
raise LogSizeException('Download size of %i bytes exceeds limit' % download_size_in_bytes)
# Lines must be explicitly decoded since `iter_lines()`` returns bytes by default
# and we cannot use its `decode_unicode=True` mode, since otherwise Unicode newline
# characters such as `\u0085` (which can appear in test output) are treated the same
# as `\n` or `\r`, and so split into unwanted additional lines by `iter_lines()`.
for line in response.iter_lines():
for builder in self.builders:
# Using `replace` to prevent malformed unicode (which might possibly exist
# in test message output) from breaking parsing of the rest of the log.
builder.parse_line(line.decode('utf-8', 'replace'))
# gather the artifacts from all builders
for builder in self.builders:
# Run end-of-parsing actions for this parser,
# in case the artifact needs clean-up/summarising.
builder.finish_parse()
name = builder.name
artifact = builder.get_artifact()
if name == 'performance_data' and not artifact[name]:
continue
self.artifacts[name] = artifact
def create_bug(self, request):
"""
Create a bugzilla bug with passed params
"""
if settings.BZ_API_KEY is None:
return Response({"failure": "Bugzilla API key not defined. This shouldn't happen."},
status=HTTP_400_BAD_REQUEST)
params = request.data
description = "Filed by: {}\n\n{}".format(
request.user.email.replace('@', " [at] "),
params.get("comment", "")
)
url = settings.BZ_API_URL + "/rest/bug"
headers = {
'x-bugzilla-api-key': settings.BZ_API_KEY,
'Accept': 'application/json'
}
data = {
'product': params.get("product"),
'component': params.get("component"),
'summary': params.get("summary"),
'keywords': params.get("keywords"),
'blocks': params.get("blocks"),
'depends_on': params.get("depends_on"),
'see_also': params.get("see_also"),
'version': params.get("version"),
'description': description,
'comment_tags': "treeherder",
}
try:
response = make_request(url, method='POST', headers=headers, json=data)
except requests.exceptions.HTTPError as e:
try:
message = e.response.json()['message']
except (ValueError, KeyError):
message = e.response.text
return Response({"failure": message}, status=HTTP_400_BAD_REQUEST)
return Response({"success": response.json()["id"]})
def parse(self):
"""
Iterate over each line of the log, running each parser against it.
Stream lines from the gzip file and run each parser against it,
building the ``artifact`` as we go.
"""
with closing(make_request(self.url, stream=True)) as response:
for line in response.iter_lines():
for builder in self.builders:
builder.parse_line(line)
# gather the artifacts from all builders
for builder in self.builders:
# Run end-of-parsing actions for this parser,
# in case the artifact needs clean-up/summarising.
builder.finish_parse()
name = builder.name
artifact = builder.get_artifact()
if name == 'performance_data' and not artifact[name]:
continue
self.artifacts[name] = artifact
def parse(self):
"""
Iterate over each line of the log, running each parser against it.
Stream lines from the gzip file and run each parser against it,
building the ``artifact`` as we go.
"""
with closing(make_request(self.url, stream=True)) as response:
for line in response.iter_lines():
for builder in self.builders:
builder.parse_line(line)
# gather the artifacts from all builders
for builder in self.builders:
# Run end-of-parsing actions for this parser,
# in case the artifact needs clean-up/summarising.
builder.finish_parse()
name = builder.name
artifact = builder.get_artifact()
if name == 'performance_data' and not artifact[name]:
continue
self.artifacts[name] = artifact
def get_log_handle(self, url):
"""Hook to get a handle to the log with this url"""
response = make_request(url)
return closing(BytesIO(response).content)
def list(self, request, project):
"""
GET method implementation for log slicer
Receives a line range and job_id and returns those lines
"""
job_id = request.query_params.get("job_id")
log_name = request.query_params.get("name")
if log_name:
log_names = [log_name]
else:
log_names = ["buildbot_text", "builds-4h"]
format = 'json' if log_name == 'mozlog_json' else 'text'
file = None
start_line = request.query_params.get("start_line")
end_line = request.query_params.get("end_line")
if not start_line or not end_line:
return Response(
"``start_line`` and ``end_line`` parameters are both required",
400)
try:
start_line = abs(int(start_line))
end_line = abs(int(end_line))
except ValueError:
return Response("parameters could not be converted to integers",
400)
if start_line >= end_line:
return Response("``end_line`` must be larger than ``start_line``",
400)
try:
job = Job.objects.get(repository__name=project,
project_specific_id=job_id)
except Job.DoesNotExist:
return Response("Job does not exist", 404)
try:
url = JobLog.objects.filter(
job=job, name__in=log_names)[0:1].values_list('url',
flat=True)[0]
except JobLog.DoesNotExist:
return Response("Job log does not exist", 404)
try:
file = filesystem.get(url)
if not file:
r = make_request(url)
try:
file = gzip.GzipFile(fileobj=BytesIO(r.content))
# read 16 bytes, just to make sure the file is gzipped
file.read(16)
file.seek(0)
filesystem.set(url, file.fileobj)
except IOError:
# file is not gzipped, but we should still store / read
# it as such, to save space
file = BytesIO(r.content)
gz_file_content = BytesIO()
with gzip.GzipFile('none', 'w',
fileobj=gz_file_content) as gz:
gz.write(r.content)
filesystem.set(url, gz_file_content)
else:
file = gzip.GzipFile(fileobj=file)
lines = []
for i, line in enumerate(file):
if i < start_line:
continue
elif i >= end_line:
break
if format == 'json':
lines.append({"data": json.loads(line), "index": i})
else:
lines.append({"text": line, "index": i})
return Response(lines)
finally:
if file:
file.close()
def list(self, request, project):
"""
GET method implementation for log slicer
Receives a line range and job_id and returns those lines
"""
job_id = request.query_params.get("job_id")
log_name = request.query_params.get("name")
if log_name:
log_names = [log_name]
else:
log_names = ["buildbot_text", "builds-4h"]
format = 'json' if log_name == 'mozlog_json' else 'text'
file = None
start_line = request.query_params.get("start_line")
end_line = request.query_params.get("end_line")
if not start_line or not end_line:
return Response("``start_line`` and ``end_line`` parameters are both required", 400)
try:
start_line = abs(int(start_line))
end_line = abs(int(end_line))
except ValueError:
return Response("parameters could not be converted to integers", 400)
if start_line >= end_line:
return Response("``end_line`` must be larger than ``start_line``", 400)
try:
job = Job.objects.get(repository__name=project,
project_specific_id=job_id)
except Job.DoesNotExist:
return Response("Job does not exist", 404)
try:
url = JobLog.objects.filter(
job=job, name__in=log_names)[0:1].values_list('url',
flat=True)[0]
except JobLog.DoesNotExist:
return Response("Job log does not exist", 404)
try:
file = filesystem.get(url)
if not file:
r = make_request(url)
try:
file = gzip.GzipFile(fileobj=BytesIO(r.content))
# read 16 bytes, just to make sure the file is gzipped
file.read(16)
file.seek(0)
filesystem.set(url, file.fileobj)
except IOError:
# file is not gzipped, but we should still store / read
# it as such, to save space
file = BytesIO(r.content)
gz_file_content = BytesIO()
with gzip.GzipFile('none', 'w', fileobj=gz_file_content) as gz:
gz.write(r.content)
filesystem.set(url, gz_file_content)
else:
file = gzip.GzipFile(fileobj=file)
lines = []
for i, line in enumerate(file):
if i < start_line:
continue
elif i >= end_line:
break
if format == 'json':
lines.append({"data": json.loads(line), "index": i})
else:
lines.append({"text": line, "index": i})
return Response(lines)
finally:
if file:
file.close()
def get_log_handle(self, url):
"""Hook to get a handle to the log with this url"""
response = make_request(url)
return closing(BytesIO(response).content)
