def test_fake_socket_passes_through_fileno():
import socket
with httpretty.enabled():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.truesock = MagicMock()
expect(s.fileno).called_with().should_not.throw(AttributeError)
s.truesock.fileno.assert_called_with()
def test_fake_socket_passes_through_fileno():
import socket
with httpretty.enabled():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.truesock = MagicMock()
expect(s.fileno).called_with().should_not.throw(AttributeError)
s.truesock.fileno.assert_called_with()
def test_retry_task_with_http_errors(self):
# We create two clients on purpose because of a bug in httpretty
# https://github.com/gabrielfalcao/HTTPretty/issues/381
# Also this allow us to test a simple requests with no retryer
client = get_client(http_retry=None)
spec = client.RecognitionSpec.retrieve('imagenet-inception-v3')
task = spec.inference(inputs=[ImageInput(DEMO_URL)],
return_task=True,
wait_task=False)
client = self.get_client_with_retry()
with httpretty.enabled():
task = client.Task.retrieve(task.pk)
httpretty.register_uri(
httpretty.GET,
re.compile(r'https?://.*?/tasks/\d+/?'),
status=502,
)
with pytest.raises(TaskTimeout) as task_timeout:
task.wait(timeout=5)
# Test nested retry errors
# TaskRetryError has been raised because of too many HTTPRetryError
# (couldn't refresh the status once)
retry_error = task_timeout.value.retry_error
assert isinstance(retry_error, TaskRetryError)
last_exception = retry_error.last_attempt.exception(timeout=0)
assert isinstance(last_exception, HTTPRetryError)
assert 502 == last_exception.last_attempt.result().status_code
def test_send_batch(error_service_thread):
decompressed_body = {
"notifier": "scout_apm_python",
"root": WORKING_DIRECTORY,
"environment": None,
"problems": [{
"foo": 0
}, {
"foo": 1
}, {
"foo": 2
}, {
"foo": 3
}, {
"foo": 4
}],
}
try:
with httpretty.enabled(allow_net_connect=False):
httpretty.register_uri(
httpretty.POST,
"https://errors.scoutapm.com/apps/error.scout",
body="Hello world!",
)
for i in range(5):
ErrorServiceThread.send(error={"foo": i})
ErrorServiceThread.wait_until_drained()
request = httpretty.last_request()
assert (json.loads(gzip_decompress(
request.body).decode("utf-8")) == decompressed_body)
assert request.headers.get("X-Error-Count") == "5"
finally:
scout_config.reset_all()
def set_write_mode_merge():
get_unique_connection_write_mode.cache_clear()
with httpretty.enabled():
httpretty.register_uri(
httpretty.GET,
"https://connect-eu.heroku.com/api/v3/connections",
body=json.dumps(fixtures.connections),
status=200,
content_type="application/json",
)
yield
def test_request_ignore_errors_host(tracked_request):
ensure_installed()
with httpretty.enabled(allow_net_connect=False):
httpretty.register_uri(httpretty.POST,
"https://errors.scoutapm.com",
body="Hello World!")
http = urllib3_cert_pool_manager()
response = http.request("POST", "https://errors.scoutapm.com")
assert response.status == 200
assert response.data == b"Hello World!"
assert len(tracked_request.complete_spans) == 0
def set_write_mode_ordered():
get_unique_connection_write_mode.cache_clear()
connections = copy.deepcopy(fixtures.connections)
connections["results"][0]["features"] = dict(poll_db_no_merge=True)
with httpretty.enabled():
httpretty.register_uri(
httpretty.GET,
"https://connect-eu.heroku.com/api/v3/connections",
body=json.dumps(connections),
status=200,
content_type="application/json",
)
yield
def test_send_unserializable_data(error_service_thread, caplog):
with httpretty.enabled(allow_net_connect=False):
ErrorServiceThread.send(error={"value": datetime.now()})
ErrorServiceThread.wait_until_drained()
if ErrorServiceThread._queue.empty() and not caplog.record_tuples:
# py38-django20 and py36-django11 tend to fail
# here by indicating the log never occurred despite
# the message being pushed down.
sleep(2)
assert caplog.record_tuples[-1][0] == "scout_apm.core.error_service"
assert caplog.record_tuples[-1][1] == logging.DEBUG
assert caplog.record_tuples[-1][2].startswith(
"Exception when serializing error message:")
def runTest(self):
url = "http://myown.link:80/luke/playlist.m3u"
with open("tests/resources/m3u8.m3u", encoding="utf-8") as content:
body = "".join(content.readlines())
with httpretty.enabled():
httpretty.register_uri(
httpretty.GET,
url,
adding_headers={"Content-Type": "application/octet-stream"},
body=body,
status=200
)
pl = playlist.loadu(url)
httpretty.disable()
httpretty.reset()
self.assertEqual(test_data.expected_m3u8, pl, "The two playlists are not equal")
def test_request(tracked_request):
ensure_installed()
with httpretty.enabled(allow_net_connect=False):
httpretty.register_uri(httpretty.GET,
"https://example.com/",
body="Hello World!")
http = urllib3.PoolManager()
response = http.request("GET", "https://example.com")
assert response.status == 200
assert response.data == b"Hello World!"
assert len(tracked_request.complete_spans) == 1
span = tracked_request.complete_spans[0]
assert span.operation == "HTTP/GET"
assert span.tags["url"] == "https://example.com:443/"
def test_download_geoip(self):
""" Test downloading geolite database """
geoip_license_key = "license_key"
url = (
"https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key="
f"{geoip_license_key}&suffix=tar.gz"
)
download_path = "geolite_city.tar.gz"
extract_path = "geolite_city.mmdb"
with CliRunner().isolated_filesystem():
with httpretty.enabled():
httpretty.register_uri(httpretty.GET, uri=url, body="success", content_type="application/gzip")
# empty files are used, so will raise error
with self.assertRaises(RuntimeError):
download_geoip(geoip_license_key, download_path, extract_path)
self.assertTrue(os.path.isfile(download_path))
self.assertTrue(os.path.isfile(extract_path))
def test_send_api_error(error_service_thread, caplog):
try:
with httpretty.enabled(allow_net_connect=False):
httpretty.register_uri(
httpretty.POST,
"https://errors.scoutapm.com/apps/error.scout",
body="Unexpected Error",
status=500,
)
ErrorServiceThread.send(error={"foo": "BØØM!"})
ErrorServiceThread.wait_until_drained()
finally:
scout_config.reset_all()
assert caplog.record_tuples[-1][0] == "scout_apm.core.error_service"
assert caplog.record_tuples[-1][1] == logging.DEBUG
assert caplog.record_tuples[-1][2].startswith(
"ErrorServiceThread 500 response error on _send:")
def runTest(self):
# For some reason the error 421 is not recognized by httpretty so it has been removed from the list
error_codes = ["400", "401", "402", "403", "404", "405", "406", "407", "408", "409",
"410", "411", "412", "413", "414", "415", "416", "417", "418",
"422", "423", "424", "425", "426", "428", "429", "431", "451", "500",
"501", "502", "503", "504", "505", "506", "507", "508", "510", "511"]
url = "http://myown.link:80/luke/playlist.m3u"
with httpretty.enabled():
for code in error_codes:
httpretty.register_uri(
httpretty.GET,
url,
status=code
)
self.assertRaises(Exception, playlist.loadu, url)
httpretty.disable()
httpretty.reset()
def test_check_api_status(self):
# mock http://localhost:8080/api/
api_url = 'http://localhost:8080/api/'
httpretty.register_uri(
httpretty.GET,
api_url,
body=json.dumps({'success': True}),
content_type='application/json',
)
with httpretty.enabled(allow_net_connect=False):
view, request = self.get_view(
CheckAppApiStatus, 'server_control_panel:app_api_status')
view.app = self.app
result = view.check_api_status()
def test_request_no_absolute_url(caplog, tracked_request):
ensure_installed()
delete_absolute_url = delete_attributes(urllib3.HTTPConnectionPool,
"_absolute_url")
with httpretty.enabled(allow_net_connect=False), delete_absolute_url:
httpretty.register_uri(httpretty.GET,
"https://example.com/",
body="Hello World!")
http = urllib3.PoolManager()
response = http.request("GET", "https://example.com")
assert response.status == 200
assert response.data == b"Hello World!"
assert len(tracked_request.complete_spans) == 1
span = tracked_request.complete_spans[0]
assert span.operation == "HTTP/GET"
assert span.tags["url"] == "Unknown"
def test_send(config, decoded_body, expected_headers, expected_uri,
error_service_thread):
scout_config.set(**config)
try:
with httpretty.enabled(allow_net_connect=False):
httpretty.register_uri(
httpretty.POST,
expected_uri,
body="Hello World!",
)
ErrorServiceThread.send(error={"foo": "BØØM!"})
ErrorServiceThread.wait_until_drained()
request = httpretty.last_request()
assert (json.loads(gzip_decompress(
request.body).decode("utf-8")) == decoded_body)
assert request.headers.get("X-Error-Count") == "1"
finally:
scout_config.reset_all()
def mock_requests_fixture():
with httpretty.enabled(allow_net_connect=False):
httpretty.register_uri(httpretty.GET,
'https://example.test/foo/bar',
body='hello')
yield
def whispir(request):
with httpretty.enabled():
yield whispyr.Whispir(TEST_USERNAME, TEST_PASSWORD, TEST_API_KEY)
def mock_send_request(mock_status):
with httpretty.enabled():
httpretty.register_uri(httpretty.POST, TEST_URL, status=mock_status)
yield
def test_download_access_stats_new(self, mock_replace_ip):
""" Test downloading access stats since April 2020 """
mock_replace_ip.return_value = ("23.1194", "-82.392", "Suva", "Peru", "PE")
# Test with and without publisher_uuid
for publisher_uuid in ["", "publisher_uuid"]:
with CliRunner().isolated_filesystem():
file_path = "oapen_access_stats.jsonl.gz"
release_date = "2020-04"
requestor_id = "requestor_id"
api_key = "api_key"
base_url = (
f"https://irus.jisc.ac.uk/api/oapen/reports/oapen_ir/?platform=215&requestor_id"
f"={requestor_id}&api_key={api_key}&begin_date={release_date}&end_date={release_date}"
)
if publisher_uuid:
base_url += f"&publisher={publisher_uuid}"
# Test response with header displaying error & without
for base_url_path in [self.download_path_v5_base, self.download_path_v5_base_error]:
with httpretty.enabled():
# register base url
with open(base_url_path, "rb") as f:
body = f.read()
httpretty.register_uri(httpretty.GET, uri=base_url, body=body, match_querystring=True)
# register country url
with open(self.download_path_v5_country, "rb") as f:
body = f.read()
url_country = base_url + "&attributes_to_show=Country"
httpretty.register_uri(httpretty.GET, uri=url_country, body=body, match_querystring=True)
# register ip url
with open(self.download_path_v5_ip, "rb") as f:
body = f.read()
url_ip = base_url + "&attributes_to_show=Client_IP"
httpretty.register_uri(httpretty.GET, uri=url_ip, body=body, match_querystring=True)
# test response without error in header
if base_url_path == self.download_path_v5_base:
no_entries = download_access_stats_new(
file_path,
release_date,
requestor_id,
api_key,
publisher_uuid,
Mock(spec=geoip2.database.Reader),
)
self.assertEqual(2, no_entries)
actual_hash = gzip_file_crc(file_path)
self.assertEqual(self.download_hash_v5, actual_hash)
# test response with error in header
else:
with self.assertRaises(RuntimeError):
download_access_stats_new(
file_path,
release_date,
requestor_id,
api_key,
publisher_uuid,
Mock(spec=geoip2.database.Reader),
)
# Test response status that is not 200
with httpretty.enabled():
httpretty.register_uri(httpretty.GET, uri=base_url, status=400)
with self.assertRaises(RuntimeError):
download_access_stats_new(
file_path,
release_date,
requestor_id,
api_key,
publisher_uuid,
Mock(spec=geoip2.database.Reader),
)
def test_download_access_stats_old(self, mock_get_results, mock_get_publishers, mock_replace_ip):
""" Test downloading access stats before April 2020 """
mock_replace_ip.return_value = ("23.1194", "-82.392", "Suva", "Peru", "PE")
mock_get_publishers.return_value = ["Publisher1", "Publisher2"]
mock_get_results.return_value = [
{
"proprietary_id": "1495483",
"URI": None,
"DOI": None,
"ISBN": "8061163562660",
"book_title": "Treat reason send message.",
"grant": None,
"grant_number": None,
"publisher": "Publisher Name",
"begin_date": "2020-03-01",
"end_date": "2020-03-31",
"title_requests": 5,
"total_item_investigations": None,
"total_item_requests": None,
"unique_item_investigations": None,
"unique_item_requests": None,
"country": [
{
"name": "France",
"code": "",
"title_requests": "4",
"total_item_investigations": None,
"total_item_requests": None,
"unique_item_investigations": None,
"unique_item_requests": None,
},
{
"name": "Norway",
"code": "",
"title_requests": "1",
"total_item_investigations": None,
"total_item_requests": None,
"unique_item_investigations": None,
"unique_item_requests": None,
},
],
"locations": [
{
"latitude": "23.1194",
"longitude": "-82.392",
"city": "Suva",
"country_name": "Peru",
"country_code": "PE",
"title_requests": "4",
"total_item_investigations": None,
"total_item_requests": None,
"unique_item_investigations": None,
"unique_item_requests": None,
},
{
"latitude": "23.1194",
"longitude": "-82.392",
"city": "Suva",
"country_name": "Peru",
"country_code": "PE",
"title_requests": "1",
"total_item_investigations": None,
"total_item_requests": None,
"unique_item_investigations": None,
"unique_item_requests": None,
},
],
"version": "4",
},
{
"proprietary_id": "2962182",
"URI": None,
"DOI": None,
"ISBN": "4880476387609",
"book_title": "Hope continue view call.",
"grant": None,
"grant_number": None,
"publisher": "Publisher Name",
"begin_date": "2020-03-01",
"end_date": "2020-03-31",
"title_requests": 1,
"total_item_investigations": None,
"total_item_requests": None,
"unique_item_investigations": None,
"unique_item_requests": None,
"country": [
{
"name": "Denmark",
"code": "",
"title_requests": "1",
"total_item_investigations": None,
"total_item_requests": None,
"unique_item_investigations": None,
"unique_item_requests": None,
}
],
"locations": [
{
"latitude": "23.1194",
"longitude": "-82.392",
"city": "Suva",
"country_name": "Peru",
"country_code": "PE",
"title_requests": "1",
"total_item_investigations": None,
"total_item_requests": None,
"unique_item_investigations": None,
"unique_item_requests": None,
}
],
"version": "4",
},
]
with CliRunner().isolated_filesystem():
file_path = "oapen_access_stats.jsonl.gz"
release_date = "2020-03"
# Test with a given publisher name
with vcr.use_cassette(self.download_path_v4_publisher):
entries, publishers = download_access_stats_old(
file_path,
release_date,
"username",
"password",
"Publisher Name",
Mock(spec=geoip2.database.Reader),
"bucket",
"blob",
)
self.assertEqual(2, entries)
self.assertEqual([], publishers)
actual_hash = gzip_file_crc(file_path)
self.assertEqual(self.download_hash_v4_publisher, actual_hash)
# Test with multiple given publisher names, delimited by "|"
with vcr.use_cassette(self.download_path_v4_publishers):
entries, publishers = download_access_stats_old(
file_path,
release_date,
"username",
"password",
"Publisher1|Publisher2",
Mock(spec=geoip2.database.Reader),
"bucket",
"blob",
)
self.assertEqual(4, entries)
self.assertEqual([], publishers)
actual_hash = gzip_file_crc(file_path)
self.assertEqual(self.download_hash_v4_publishers, actual_hash)
# Test with a given publisher name, but no entries for that publisher
with vcr.use_cassette(self.download_path_v4_empty_publisher):
entries, publishers = download_access_stats_old(
file_path,
release_date,
"username",
"password",
"Empty Publisher",
Mock(spec=geoip2.database.Reader),
"bucket",
"blob",
)
self.assertEqual(0, entries)
self.assertEqual([], publishers)
actual_hash = gzip_file_crc(file_path)
self.assertEqual("00000000", actual_hash)
# Test when no publisher name is given and no unprocessed_publishers (all publishers are downloaded)
with vcr.use_cassette(self.download_path_v4_all_publishers):
entries, publishers = download_access_stats_old(
file_path,
release_date,
"username",
"password",
"",
Mock(spec=geoip2.database.Reader),
"bucket",
"blob",
)
self.assertEqual(4, entries)
self.assertEqual([], publishers)
actual_hash = gzip_file_crc(file_path)
self.assertEqual(self.download_hash_v4_all_publishers, actual_hash)
# Test when no publisher name is given with a list of unprocessed_publishers
with vcr.use_cassette(self.download_path_v4_all_publishers):
entries, publishers = download_access_stats_old(
file_path,
release_date,
"username",
"password",
"",
Mock(spec=geoip2.database.Reader),
"bucket",
"blob",
["Publisher1", "Publisher2"],
)
self.assertEqual(6, entries)
self.assertEqual([], publishers)
actual_hash = gzip_file_crc(file_path)
self.assertEqual(self.download_hash_v4_unprocessed_publishers, actual_hash)
with httpretty.enabled():
# register login page
httpretty.register_uri(
httpretty.POST,
uri="https://irus.jisc.ac.uk/IRUSConsult/irus-oapen/v2/?action=login",
body="After you have finished your session please remember to",
)
# Test response status that is not 200
start_date = release_date + "-01"
end_date = release_date + "-31"
ip_url = (
f"https://irus.jisc.ac.uk/IRUSConsult/irus-oapen/v2/br1b/?frmRepository=1%7COAPEN+Library"
f"&frmFrom={start_date}&frmTo={end_date}&frmFormat=TSV&Go=Generate+Report&frmPublisher=error"
)
httpretty.register_uri(httpretty.GET, uri=ip_url, status=400)
with self.assertRaises(RuntimeError):
download_access_stats_old(
file_path,
release_date,
"username",
"password",
"error",
Mock(spec=geoip2.database.Reader),
"bucket",
"blob",
)
with httpretty.enabled():
# Test response status that is not 200 for login
httpretty.register_uri(
httpretty.POST, uri="https://irus.jisc.ac.uk/IRUSConsult/irus-oapen/v2/?action=login", status=400
)
with self.assertRaises(RuntimeError):
download_access_stats_old(
file_path,
release_date,
"username",
"password",
"",
Mock(spec=geoip2.database.Reader),
"bucket",
"blob",
)
def chat():
with httpretty.enabled():
yield RocketChat(ROCKET_SERVER, ROCKET_USERNAME, ROCKET_PASSWORD)
