def fetch_layout_test_results(self, results_url):
# FIXME: This should cache that the result was a 404 and stop hitting the network.
results_file = NetworkTransaction(convert_404_to_None=True).run(lambda: self._fetch_file_from_results(results_url, "full_results.json"))
if not results_file:
results_file = NetworkTransaction(convert_404_to_None=True).run(lambda: self._fetch_file_from_results(results_url, "results.html"))
# results_from_string accepts either ORWT html or NRWT json.
return LayoutTestResults.results_from_string(results_file)
def test_retry(self):
transaction = NetworkTransaction(initial_backoff_seconds=0)
self.assertEqual(transaction.run(self._raise_500_error), 42)
self.assertEqual(self._run_count, 3)
self.assertLog(['WARNING: Received HTTP status 500 loading "http://example.com/". '
'Retrying in 0 seconds...\n',
'WARNING: Received HTTP status 500 loading "http://example.com/". '
'Retrying in 0.0 seconds...\n'])
def fetch_layout_test_results(self, results_url):
"""Returns a LayoutTestResults object for results fetched from a given URL."""
results_file = NetworkTransaction(convert_404_to_None=True).run(
lambda: self._fetch_file(results_url, "failing_results.json"))
revision = NetworkTransaction(convert_404_to_None=True).run(
lambda: self._fetch_file(results_url, "LAST_CHANGE"))
if not revision:
results_file = None
return LayoutTestResults.results_from_string(results_file, revision)
def test_retry(self):
self._run_count = 0
transaction = NetworkTransaction(initial_backoff_seconds=0)
self.assertEqual(transaction.run(lambda: self._raise_500_error()), 42)
self.assertEqual(self._run_count, 3)
self.assertLog(['WARNING: Received HTTP status 500 from server. '
'Retrying in 0 seconds...\n',
'WARNING: Received HTTP status 500 from server. '
'Retrying in 0.0 seconds...\n'])
def test_exception(self):
transaction = NetworkTransaction()
did_process_exception = False
did_throw_exception = True
try:
transaction.run(lambda: self._raise_exception())
did_throw_exception = False
except Exception, e:
did_process_exception = True
self.assertEqual(e, self.exception)
def test_exception(self):
transaction = NetworkTransaction()
did_process_exception = False
did_throw_exception = True
try:
transaction.run(lambda: self._raise_exception())
did_throw_exception = False
except Exception, e:
did_process_exception = True
self.assertEqual(e, self.exception)
def test_timeout(self):
self._run_count = 0
transaction = NetworkTransaction(initial_backoff_seconds=60*60, timeout_seconds=60)
did_process_exception = False
did_throw_exception = True
try:
transaction.run(lambda: self._raise_500_error())
did_throw_exception = False
except NetworkTimeout, e:
did_process_exception = True
def test_timeout(self):
self._run_count = 0
transaction = NetworkTransaction(initial_backoff_seconds=60*60, timeout_seconds=60)
did_process_exception = False
did_throw_exception = True
try:
transaction.run(lambda: self._raise_500_error())
did_throw_exception = False
except NetworkTimeout, e:
did_process_exception = True
def fetch_layout_test_results(self, results_url):
# FIXME: This should cache that the result was a 404 and stop hitting the network.
results_file = NetworkTransaction(convert_404_to_None=True).run(
lambda: self._fetch_file_from_results(results_url,
"failing_results.json"))
revision = NetworkTransaction(convert_404_to_None=True).run(
lambda: self._fetch_file_from_results(results_url, "LAST_CHANGE"))
if not revision:
results_file = None
return LayoutTestResults.results_from_string(results_file, revision)
def test_timeout(self):
transaction = NetworkTransaction(initial_backoff_seconds=60 * 60, timeout_seconds=60)
did_process_exception = False
did_throw_exception = True
try:
transaction.run(self._raise_500_error)
did_throw_exception = False
except NetworkTimeout:
did_process_exception = True
self.assertTrue(did_throw_exception)
self.assertTrue(did_process_exception)
def test_retry(self):
self._run_count = 0
transaction = NetworkTransaction(initial_backoff_seconds=0)
self.assertEqual(transaction.run(lambda: self._raise_http_error()), 42)
self.assertEqual(self._run_count, 3)
self.assertLog([
'WARNING: Received HTTP status 500 from server. '
'Retrying in 0 seconds...\n',
'WARNING: Received HTTP status 500 from server. '
'Retrying in 0.0 seconds...\n'
])
def test_retry_on_URLError(self):
self._run_count = 0
url = "http://example.com/"
transaction = NetworkTransaction(initial_backoff_seconds=0)
self.assertEqual(transaction.run(lambda: self._raise_URLError(), url),
43)
self.assertEqual(self._run_count, 3)
self.assertLog([
'WARNING: Received URLError: "[Errno 60] Operation timed out" while loading http://example.com/. '
'Retrying in 0 seconds...\n',
'WARNING: Received URLError: "[Errno 60] Operation timed out" while loading http://example.com/. '
'Retrying in 0.0 seconds...\n'
])
def upload_attachment(self, attachment):
_log.info('Uploading attachment {} to status server'.format(
attachment.id()))
# FIXME: Remove argument convert_404_to_None once we update AppEngine to support uploading attachments.
return NetworkTransaction(convert_404_to_None=True).run(
lambda: self._upload_attachment_to_server(attachment.id(
), attachment.to_json(), attachment.contents()))
def update_work_items(self, queue_name, high_priority_work_items,
work_items):
_log.info("Recording work items: %s for %s" %
(high_priority_work_items + work_items, queue_name))
return NetworkTransaction().run(
lambda: self._post_work_items_to_server(
queue_name, high_priority_work_items, work_items))
def _upload_data(self, content_type, data):
def callback():
# FIXME: Setting a timeout, either globally using socket.setdefaulttimeout()
# or in urlopen(), doesn't appear to work on Mac 10.5 with Python 2.7.
# For now we will ignore the timeout value and hope for the best.
request = urllib2.Request(self._url, data, {"Content-Type": content_type})
return urllib2.urlopen(request)
return NetworkTransaction(timeout_seconds=self._timeout_seconds).run(callback)
def fetch_retry_summary_json(self, build):
"""Fetches and returns the text of the archived retry_summary file.
This file is expected to contain the results of retrying layout tests
with and without a patch in a try job. It includes lists of tests
that failed only with the patch ("failures"), and tests that failed
both with and without ("ignored").
"""
url_base = "%s/%s" % (self.builder_results_url_base(
build.builder_name), build.build_number)
return NetworkTransaction(convert_404_to_None=True).run(
lambda: self._fetch_file(url_base, "retry_summary.json"))
def upload(self, params, files, timeout_seconds):
file_objs = []
for filename, path in files:
with codecs.open(path, "rb") as file:
file_objs.append(('file', filename, file.read()))
orig_timeout = socket.getdefaulttimeout()
try:
socket.setdefaulttimeout(timeout_seconds)
NetworkTransaction(timeout_seconds=timeout_seconds).run(
lambda: self._upload_files(params, file_objs))
finally:
socket.setdefaulttimeout(orig_timeout)
def upload(self, params, files, timeout_seconds):
orig_timeout = socket.getdefaulttimeout()
file_objs = []
try:
file_objs = [(filename, open(path, "rb"))
for (filename, path) in files]
socket.setdefaulttimeout(timeout_seconds)
NetworkTransaction(timeout_seconds=timeout_seconds).run(
lambda: self._upload_files(params, file_objs))
finally:
socket.setdefaulttimeout(orig_timeout)
for (filename, handle) in file_objs:
handle.close()
def upload(self, params, files, timeout_seconds):
file_objs = []
for filename, path in files:
# FIXME: We should talk to the filesytem via a Host object.
with codecs.open(path, "rb") as file:
file_objs.append(('file', filename, file.read()))
orig_timeout = socket.getdefaulttimeout()
try:
# FIXME: We shouldn't mutate global static state.
socket.setdefaulttimeout(timeout_seconds)
NetworkTransaction(timeout_seconds=timeout_seconds).run(
lambda: self._upload_files(params, file_objs))
finally:
socket.setdefaulttimeout(orig_timeout)
def _upload_data(self, content_type, data):
def callback():
request = urllib2.Request(self._url, data,
{"Content-Type": content_type})
return urllib2.urlopen(request)
orig_timeout = socket.getdefaulttimeout()
response = None
try:
# FIXME: We shouldn't mutate global static state.
socket.setdefaulttimeout(self._timeout_seconds)
response = NetworkTransaction(
timeout_seconds=self._timeout_seconds).run(callback)
finally:
socket.setdefaulttimeout(orig_timeout)
return response
def test_convert_404_to_None(self):
transaction = NetworkTransaction(convert_404_to_None=True)
self.assertEqual(transaction.run(lambda: self._raise_404_error()), None)
def test_convert_404_to_None(self):
transaction = NetworkTransaction(convert_404_to_None=True)
self.assertIsNone(transaction.run(self._raise_404_error))
def test_success(self):
transaction = NetworkTransaction()
self.assertEqual(transaction.run(lambda: 42), 42)
def update_svn_revision(self, svn_revision_number, broken_bot):
_log.info("SVN revision: %s broke %s" %
(svn_revision_number, broken_bot))
return NetworkTransaction().run(
lambda: self._post_svn_revision_to_server(svn_revision_number,
broken_bot))
def test_success(self):
transaction = NetworkTransaction()
self.assertEqual(transaction.run(lambda: 42), 42)
def open_url(self, url):
return NetworkTransaction().run(lambda: self.browser.open(url), url)
def release_lock(self, queue_name, patch):
_log.info("Releasing lock for work item %s from %s" %
(patch.id(), queue_name))
return NetworkTransaction(convert_404_to_None=True).run(
lambda: self._post_release_lock(queue_name, patch))
def submit_to_ews(self, attachment_id):
_log.info("Submitting attachment %s to EWS queues" % attachment_id)
return NetworkTransaction().run(
lambda: self._post_work_item_to_ews(attachment_id))
def submit_to_ews(self, attachment_id):
_log.info(
'Submitting attachment {} to EWS queues'.format(attachment_id))
return NetworkTransaction().run(
lambda: self._post_patch_to_ews(attachment_id))
def get_binary(self, url, convert_404_to_None=False):
return NetworkTransaction(convert_404_to_None=convert_404_to_None).run(
lambda: urllib2.urlopen(url).read())
def bug_id_for_attachment_id(self, attachment_id):
return NetworkTransaction().run(
lambda: self.get_bug_id_for_attachment_id(attachment_id))
def update_work_items(self, queue_name, work_items):
_log.debug("Recording work items: %s for %s" %
(work_items, queue_name))
return NetworkTransaction().run(
lambda: self._post_work_items_to_server(queue_name, work_items))
def _retrieve_zip_file(self, zip_url):
temp_file = NetworkTransaction().run(
lambda: urllib.urlretrieve(zip_url)[0])
return (temp_file, zipfile.ZipFile(temp_file))
def update_status(self, queue_name, status, patch=None, results_file=None):
_log.info(status)
return NetworkTransaction().run(lambda: self._post_status_to_server(
queue_name, status, patch, results_file))
def bug_id_for_attachment_id(self,
attachment_id,
throw_on_access_error=False):
return NetworkTransaction().run(
lambda: self.get_bug_id_for_attachment_id(attachment_id,
throw_on_access_error))
def test_convert_404_to_None(self):
transaction = NetworkTransaction(convert_404_to_None=True)
self.assertEqual(transaction.run(lambda: self._raise_404_error()), None)
