def test_block_unmocked_requests():
"""Ensure the `block_unmocked_requests` fixture prevents requests from hitting the network."""
url = 'https://example.com'
with pytest.raises(RuntimeError, message='Tests must mock all HTTP requests!'):
fetch_text(url)
with responses.RequestsMock() as rsps:
rsps.add(responses.GET, url, body='Mocked requests still work')
text = fetch_text(url)
assert text == 'Mocked requests still work'
def test_block_unmocked_requests():
"""Ensure the `block_unmocked_requests` fixture prevents requests from hitting the network."""
url = 'https://example.com'
with pytest.raises(RuntimeError, message='Tests must mock all HTTP requests!'):
fetch_text(url)
with responses.RequestsMock() as rsps:
rsps.add(responses.GET, url, body='Mocked requests still work')
text = fetch_text(url)
assert text == 'Mocked requests still work'
def handle(self, *args, **options):
try:
repository_name, job_guid, log_url_or_obj = args
except ValueError:
raise CommandError('3 arguments required, %s given' % len(args))
try:
log_obj = json.loads(log_url_or_obj)
except ValueError:
try:
log_obj = expand_log_url(repository_name, job_guid, log_url_or_obj)
except ValueError:
# This log_url either isn't in the database, or there are multiple possible
# urls in the database, so we will be unable to update the pending state
log_obj = None
if log_obj:
log_url = log_obj["url"]
else:
log_url = log_url_or_obj
log_text = fetch_text(log_url)
if not log_text:
return
log_content = StringIO(log_text)
try:
repository = Repository.objects.get(name=repository_name, active_status='active')
except Repository.DoesNotExist:
raise CommandError('Unknown repository %s' % repository_name)
log_iter = reader.read(log_content)
failure_lines_cutoff = settings.FAILURE_LINES_CUTOFF
log_iter = list(islice(log_iter, failure_lines_cutoff+1))
if len(log_iter) > failure_lines_cutoff:
# Alter the N+1th log line to indicate the list was truncated.
log_iter[-1].update(action='truncated')
with transaction.atomic():
FailureLine.objects.bulk_create(
[FailureLine(repository=repository, job_guid=job_guid, **failure_line)
for failure_line in log_iter]
)
if log_obj is not None:
with JobsModel(repository_name) as jm:
jm.update_job_log_url_status(log_obj["id"], "parsed")
else:
logger.warning("Unable to set parsed state of job log")
def fetch_log(job_log):
try:
log_text = fetch_text(job_log.url)
except HTTPError as e:
job_log.update_status(JobLog.FAILED)
if e.response is not None and e.response.status_code in (403, 404):
logger.warning("Unable to retrieve log for %s: %s", job_log.url, e)
return
raise
if not log_text:
return
return (json.loads(item) for item in log_text.splitlines())
def fetch_log(job_log):
try:
log_text = fetch_text(job_log.url)
except HTTPError as e:
job_log.update_status(JobLog.FAILED)
if e.response is not None and e.response.status_code in (403, 404):
logger.warning("Unable to retrieve log for %s: %s",
job_log.url, e)
return
raise
if not log_text:
return
return (json.loads(item) for item in log_text.splitlines())
def store_failure_lines(repository_name, job_guid, job_log):
try:
log_text = fetch_text(job_log.url)
except HTTPError as e:
job_log.status = JobLog.FAILED
job_log.save()
if e.response is not None and e.response.status_code in (403, 404):
logger.warning("Unable to retrieve log for %s: %s", job_log.url, e)
return
raise
if not log_text:
return
log_iter = (json.loads(item) for item in log_text.splitlines())
try:
repository = Repository.objects.get(name=repository_name,
active_status='active')
except Repository.DoesNotExist:
logger.error("Unknown repository %s" % repository_name)
raise
failure_lines_cutoff = settings.FAILURE_LINES_CUTOFF
log_iter = list(islice(log_iter, failure_lines_cutoff + 1))
if len(log_iter) > failure_lines_cutoff:
# Alter the N+1th log line to indicate the list was truncated.
log_iter[-1].update(action='truncated')
retry = False
with transaction.atomic():
try:
create(repository, job_guid, job_log, log_iter)
except OperationalError as e:
logger.warning("Got OperationalError inserting failure_line")
# Retry iff this error is the "incorrect String Value" error
retry = e.args[0] == 1366
logger.info("store failure lines 3")
if retry:
with transaction.atomic():
logger.info("Retrying insert with astral character replacement")
# Sometimes get an error if we can't save a string as MySQL pseudo-UTF8
log_iter = list(replace_astral(log_iter))
create(repository, job_guid, job_log, log_iter)
def store_failure_lines(repository_name, job_guid, job_log):
try:
log_text = fetch_text(job_log.url)
except HTTPError as e:
job_log.status = JobLog.FAILED
job_log.save()
if e.response is not None and e.response.status_code in (403, 404):
logger.warning("Unable to retrieve log for %s: %s",
job_log.url, e)
return
raise
if not log_text:
return
log_iter = (json.loads(item) for item in log_text.splitlines())
try:
repository = Repository.objects.get(name=repository_name, active_status='active')
except Repository.DoesNotExist:
logger.error("Unknown repository %s" % repository_name)
raise
failure_lines_cutoff = settings.FAILURE_LINES_CUTOFF
log_iter = list(islice(log_iter, failure_lines_cutoff+1))
if len(log_iter) > failure_lines_cutoff:
# Alter the N+1th log line to indicate the list was truncated.
log_iter[-1].update(action='truncated')
retry = False
with transaction.atomic():
try:
create(repository, job_guid, job_log, log_iter)
except OperationalError as e:
logger.warning("Got OperationalError inserting failure_line")
# Retry iff this error is the "incorrect String Value" error
retry = e.args[0] == 1366
logger.info("store failure lines 3")
if retry:
with transaction.atomic():
logger.info("Retrying insert with astral character replacement")
# Sometimes get an error if we can't save a string as MySQL pseudo-UTF8
log_iter = list(replace_astral(log_iter))
create(repository, job_guid, job_log, log_iter)
def handle(self, *args, **options):
try:
log_url, job_guid, repository_name = args
except ValueError:
raise CommandError('3 arguments required, %s given' % len(args))
log_text = fetch_text(log_url)
if not log_text:
return
log_content = StringIO(log_text)
try:
repository = Repository.objects.get(name=repository_name,
active_status='active')
except Repository.DoesNotExist:
raise CommandError('Unknown repository %s' % repository_name)
log_iter = reader.read(log_content)
failure_lines_cutoff = settings.FAILURE_LINES_CUTOFF
log_iter = list(islice(log_iter, failure_lines_cutoff + 1))
if len(log_iter) > failure_lines_cutoff:
# Alter the N+1th log line to indicate the list was truncated.
log_iter[-1].update(action='truncated')
with JobsModel(repository_name) as jobs_model:
job_id = jobs_model.get_job_ids_by_guid([job_guid])
if not job_id:
raise CommandError(
'No job found with guid %s in the %s repository' %
(job_guid, repository_name))
with transaction.atomic():
FailureLine.objects.bulk_create([
FailureLine(repository=repository,
job_guid=job_guid,
**failure_line) for failure_line in log_iter
])
def handle(self, *args, **options):
try:
log_url, job_guid, repository_name = args
except ValueError:
raise CommandError('3 arguments required, %s given' % len(args))
log_text = fetch_text(log_url)
if not log_text:
return
log_content = StringIO(log_text)
try:
repository = Repository.objects.get(name=repository_name, active_status='active')
except Repository.DoesNotExist:
raise CommandError('Unknown repository %s' % repository_name)
log_iter = reader.read(log_content)
failure_lines_cutoff = settings.FAILURE_LINES_CUTOFF
log_iter = list(islice(log_iter, failure_lines_cutoff+1))
if len(log_iter) > failure_lines_cutoff:
# Alter the N+1th log line to indicate the list was truncated.
log_iter[-1].update(action='truncated')
with JobsModel(repository_name) as jobs_model:
job_id = jobs_model.get_job_ids_by_guid([job_guid])
if not job_id:
raise CommandError('No job found with guid %s in the %s repository' % (job_guid, repository_name))
with transaction.atomic():
FailureLine.objects.bulk_create(
[FailureLine(repository=repository, job_guid=job_guid, **failure_line)
for failure_line in log_iter]
)
def handle(self, *args, **options):
if options["min_id"] is None:
options["min_id"] = (FailureLine.objects.filter(
action="test_result").exclude(message=None).exclude(
message="").order_by("-id").values_list(
"id", flat=True)[options["num_lines"]])
failure_lines = (FailureLine.objects.filter(
id__gt=options["min_id"],
action="test_result").exclude(message=None).exclude(
message="").order_by("id")[:options["num_lines"]])
self.stderr.write("Using min id %d" % options["min_id"])
self.stderr.write("Got %d lines" % len(failure_lines))
t0 = time.time()
fetch_text(settings.ELASTIC_SEARCH["url"])
self.stderr.write("Simple GET took %dms" % ((time.time() - t0) * 1000))
failure_lines_by_job = defaultdict(list)
for line in failure_lines:
failure_lines_by_job[line.job_guid].append(line)
matcher = matchers.ElasticSearchTestMatcher(None)
all_matches = {}
if options["profile"]:
import cProfile
prof = cProfile.Profile()
prof.enable()
total_lines = 0
t0 = time.time()
for job_guid, failure_lines in iteritems(failure_lines_by_job):
total_lines += len(failure_lines)
matches = matcher(failure_lines)
all_matches[job_guid] = matches
duration = 1000 * (time.time() - t0)
self.stderr.write("Total lines %d" % total_lines)
self.stderr.write("Called ElasticSearch %i times" % matcher.calls)
self.stderr.write("Took %dms" % duration)
if options["profile"]:
prof.disable()
prof.dump_stats(options["profile"])
json_data = {}
for key, values in iteritems(all_matches):
json_values = [[item[0].id, item[1].id, item[2]]
for item in values]
json_data[key] = json_values
json_string = json.dumps(json_data)
if options["ref_data"]:
with open(options["ref_data"]) as f:
ref_data = json.load(f)
this_data = json.loads(json_string)
if this_data == ref_data:
self.stderr.write("Output matches refdata")
else:
self.stderr.write("Output does not match refdata")
self.stdout.write(json_string)
def handle(self, *args, **options):
if options["min_id"] is None:
options["min_id"] = (FailureLine.objects
.filter(action="test_result")
.exclude(message=None)
.exclude(message="")
.order_by("-id")
.values_list("id", flat=True)[options["num_lines"]])
failure_lines = (FailureLine.objects
.filter(id__gt=options["min_id"],
action="test_result")
.exclude(message=None)
.exclude(message="")
.order_by("id")[:options["num_lines"]])
self.stderr.write("Using min id %d" % options["min_id"])
self.stderr.write("Got %d lines" % len(failure_lines))
t0 = time.time()
fetch_text(settings.ELASTIC_SEARCH["url"])
self.stderr.write("Simple GET took %dms" % ((time.time() - t0) * 1000))
failure_lines_by_job = defaultdict(list)
for line in failure_lines:
failure_lines_by_job[line.job_guid].append(line)
matcher = matchers.ElasticSearchTestMatcher(None)
all_matches = {}
if options["profile"]:
import cProfile
prof = cProfile.Profile()
prof.enable()
total_lines = 0
t0 = time.time()
for job_guid, failure_lines in failure_lines_by_job.iteritems():
total_lines += len(failure_lines)
matches = matcher(failure_lines)
all_matches[job_guid] = matches
duration = 1000 * (time.time() - t0)
self.stderr.write("Total lines %d" % total_lines)
self.stderr.write("Total lines in matcher %d" % matcher.lines)
self.stderr.write("Called ElasticSearch %i times" % matcher.calls)
self.stderr.write("Took %dms" % duration)
if options["profile"]:
prof.disable()
prof.dump_stats(options["profile"])
json_data = {}
for key, values in all_matches.iteritems():
json_values = [[item[0].id, item[1].id, item[2]] for item in values]
json_data[key] = json_values
json_string = json.dumps(json_data)
if options["ref_data"]:
with open(options["ref_data"]) as f:
ref_data = json.load(f)
this_data = json.loads(json_string)
if this_data == ref_data:
self.stderr.write("Output matches refdata")
else:
self.stderr.write("Output does not match refdata")
self.stdout.write(json_string)
