def test_scrape_ad(ad):
finnkode = "257069951"
responses.get(
f"https://www.finn.no/realestate/homes/ad.html?finnkode={finnkode}",
status=200,
body=ad,
)
res = scrape_ad(finnkode)
assert res
assert res == {
"Postadresse": "Thereses gate 35B, 0354 Oslo",
"url":
"https://www.finn.no/realestate/homes/ad.html?finnkode=257069951",
"Visning 1": "2022-05-08T13:00:00",
"Visning 2": "2022-05-09T14:30:00",
"Visninger": ["2022-05-08T13:00:00", "2022-05-09T14:30:00"],
"Omkostninger": 12600,
"Totalpris": 3802600,
"Felleskost/mnd.": 3077,
"Boligtype": "Leilighet",
"Eieform bolig": "Aksje",
"Soverom": 1,
"Primærrom": 30,
"Bruksareal": 30,
"Etasje": 4,
"Byggeår": 1923,
"Energimerking": "F - mørkegrønn",
"Rom": 2,
"Tomteareal": "1391 (eiet)",
"Boligselgerforsikring": "Ja",
"Fellesformue": 127032,
"Formuesverdi": 796595,
"Prisantydning": 3790000,
}
def test_pagination_not_handled(self):
"""At present, we don't handle any pagination of results."""
responses.get(
url="https://mitoc-gear.mit.edu/api-auth/v1/credentials",
json={
"count": 10,
"next":
'https://mitoc-gear.mit.edu/api-auth/v1/credentials?page=2',
"previous": None,
"results": [{
'user': '******',
'password': '******'
}],
},
status=200,
)
with mock.patch.object(geardb.logger, 'error') as log_error:
results = geardb.query_api('/api-auth/v1/credentials')
self.assertEqual(results, [{
'user': '******',
'password': '******'
}])
log_error.assert_called_once_with(
"Results are paginated; this is not expected or handled. (%s / %s results), URL: %s, Next: %s",
1,
10,
'https://mitoc-gear.mit.edu/api-auth/v1/credentials',
'https://mitoc-gear.mit.edu/api-auth/v1/credentials?page=2',
)
def test_bad_status_code(self):
responses.get(
url="https://mitoc-gear.mit.edu/credentials",
status=404,
)
with self.assertRaises(requests.exceptions.HTTPError):
geardb.query_api('/credentials', user='******')
def test_stats_grafana_xapi_fun_backend_RequestException(self, logger_mock):
"""RequestException from call to the backend should return stats with 0."""
video = VideoFactory()
settings = {
"api_key": "grafana_api_key",
"api_endpoint": "https://grafana.tld/api",
"api_datasource_id": "1",
"api_datastream": "statements-ds-marsha",
}
exception = RequestException("An error occurred")
responses.get(
url=(
f"{settings.get('api_endpoint')}/datasources/proxy/"
f"{settings.get('api_datasource_id')}/{settings.get('api_datastream')}/_count"
),
body=exception,
)
with mock.patch.object(stats, "capture_exception") as mock_capture_exception:
self.assertEqual(
{"nb_views": 0}, grafana_xapi_fun_backend(video, **settings)
)
logger_mock.error.assert_called_with(
"Request to grafana error: %s", exception
)
mock_capture_exception.assert_called_once_with(exception)
def test_no_membership_found(self):
user = factories.UserFactory(email='*****@*****.**')
responses.get(
url=
'https://mitoc-gear.mit.edu/api-auth/v1/membership_waiver/[email protected]',
json={
"count":
0,
"next":
None,
"previous":
None,
"results": [{
"email": "",
"alternate_emails": [],
"membership": {},
"waiver": {},
}],
},
)
membership = geardb.query_geardb_for_membership(user)
self.assertEqual(
membership,
geardb.MembershipWaiver(
email=None,
membership_expires=None,
waiver_expires=None,
),
)
def test_no_results(self):
"""Test our handling of an empty `results` list.
(I don't think this presently happens, but it's good to check)
"""
user = factories.UserFactory(email='*****@*****.**')
responses.get(
url=
'https://mitoc-gear.mit.edu/api-auth/v1/membership_waiver/[email protected]',
json={
"count": 0,
"next": None,
"previous": None,
"results": [],
},
)
membership = geardb.query_geardb_for_membership(user)
self.assertEqual(
membership,
geardb.MembershipWaiver(
email=None,
membership_expires=None,
waiver_expires=None,
),
)
def test_server_response_500(self):
responses.get('https://ipapi.co/192.0.2.0/json', status=500)
with self.assertLogs(logger, level='ERROR') as logged:
tz = ip_to_timezone('192.0.2.0')
self.assertEqual(logged.output, [
f"ERROR:{logger.name}:External query returned error code 500",
])
self.assertEqual(tz, None)
def test_fails_to_update_cache(self):
"""If the gear database is down, we just allow participants to attend."""
participant = factories.ParticipantFactory.create(membership=None)
responses.get(
'https://mitoc-gear.mit.edu/api-auth/v1/membership_waiver/',
status=500)
self.assertTrue(self._can_attend(participant.user))
def test_cache_updated(self):
"""If our local cache is outdated, we must hit the gear database for updates."""
with freeze_time(
"2018-10-23 14:55:45 EST"): # (Will have old `last_cached`)
dated_membership = factories.MembershipFactory.create(
membership_expires=date(2018, 11,
18), # Active, expires before trip
waiver_expires=date(2018, 12,
18), # Active, expires after trip
)
participant = factories.ParticipantFactory.create(
email='*****@*****.**', membership=dated_membership)
before_refreshing_ts = dated_membership.last_cached
# Right now, with our last-cached copy of their membership, they cannot attend
self.assertCountEqual(
participant.reasons_cannot_attend(self.trip),
[enums.TripIneligibilityReason.MEMBERSHIP_NEEDS_RENEWAL],
)
responses.get(
url=
'https://mitoc-gear.mit.edu/api-auth/v1/membership_waiver/[email protected]',
json={
"count":
1,
"next":
None,
"previous":
None,
"results": [{
"email": "*****@*****.**",
"alternate_emails": ["*****@*****.**"],
"membership": {
"membership_type": "NA",
"expires": "2019-11-19",
},
"waiver": {
"expires": "2019-11-19"
},
}],
},
)
# Tick forward in time so we can show that `last_cached` ts advances!
with freeze_time("2018-11-19 12:00:00 EST", tick=True):
can_attend = self._can_attend(participant.user)
self.assertTrue(can_attend)
# The participant now has an updated membership cache too!
dated_membership.refresh_from_db()
self.assertGreater(dated_membership.last_cached, before_refreshing_ts)
self.assertEqual(dated_membership.membership_expires,
date(2019, 11, 19))
self.assertEqual(dated_membership.waiver_expires, date(2019, 11, 19))
self.assertFalse(any(participant.reasons_cannot_attend(self.trip)))
def test_invalid_json_response(self):
responses.get('https://ipapi.co/192.0.2.0/json', body='not valid json')
with self.assertLogs(logger, level='ERROR') as logged:
tz = ip_to_timezone('192.0.2.0')
self.assertEqual(len(logged.output), 1)
self.assertRegex(
logged.output[0],
f'^ERROR:{logger.name}:Response doesn\'t appear to be valid JSON')
self.assertEqual(tz, None)
def test_timeout(self):
responses.get('https://ipapi.co/192.0.2.0/json',
body=requests.Timeout("too slow"))
with self.assertLogs(logger, level='ERROR') as logged:
tz = ip_to_timezone('192.0.2.0')
self.assertEqual(len(logged.output), 1)
self.assertRegex(logged.output[0],
f'^ERROR:{logger.name}:External query timed out')
self.assertEqual(tz, None)
def test_unexpected_json_structure(self):
responses.get('https://ipapi.co/192.0.2.0/json',
json={'TIMEZONE': 'UTC'})
with self.assertLogs(logger, level='ERROR') as logged:
tz = ip_to_timezone('192.0.2.0')
self.assertEqual(len(logged.output), 1)
self.assertRegex(
logged.output[0],
f'^ERROR:{logger.name}:Response JSON content has an unexpected structure'
)
self.assertEqual(tz, None)
def get_fasjson_client_override():
responses.add_passthru("https://json-schema.org")
fasjson_spec_path = os.path.join(os.path.dirname(__file__), "fixtures",
"fasjson-v1.json")
with open(fasjson_spec_path) as fasjson_spec_file:
fasjson_spec = json.load(fasjson_spec_file)
responses.get("http://fasjson.example.test/specs/v1.json",
json=fasjson_spec)
base_url = get_settings_override().fasjson_url
return main.FasjsonClient(base_url, auth=False)
def test_channels_single_page(self):
responses.get(
url="http://mythbackend/Channel/GetChannelInfoList?" +
"SourceId=1&Details=true&OnlyVisible=false" +
"&StartIndex=0&Count=100",
body=open("mock/channel_info_list.xml").read()
)
backend = MythTVBackend("http://mythbackend")
channels = backend.get_channel_info_list(1)
self.assertEqual(1, len(channels))
self.assertEqual("NBC5-US", channels[0].channel_name)
def test_proxy_get_capabilities_user2(self):
from c2cgeoportal_geoportal.views.tinyowsproxy import TinyOWSProxy
request = _create_dummy_request(username="******")
responses.get(
url="http://mapserver:8080/",
body=load_file(self.capabilities_response_file),
)
response = TinyOWSProxy(request).proxy()
self.assertEqual(
load_file(self.capabilities_response_filtered_file2).strip(),
response.body.decode().replace(" \n", "").strip(),
)
self.assertEqual("200 OK", response.status)
def test_proxy_get_capabilities_get(self):
from c2cgeoportal_geoportal.views.tinyowsproxy import TinyOWSProxy
request = _create_dummy_request(username="******")
request.params.update(
dict(service="wfs", version="1.1.0", request="GetCapabilities"))
responses.get(
url="http://mapserver:8080/",
body=load_file(self.capabilities_response_file),
)
response = TinyOWSProxy(request).proxy()
self.assertEqual(
load_file(self.capabilities_response_filtered_file1).strip(),
response.body.decode().replace(" \n", "").strip(),
)
self.assertEqual("200 OK", response.status)
def test_get_userinfo(client, mocker):
user_data = {
"username": "******",
"surname": "User",
"givenname": "Dummy",
"human_name": "Dummy User",
"emails": ["*****@*****.**"],
"ircnicks": ["irc://dummy", "matrix://dummy"],
"locale": "en-US",
"uri": "http://fasjson.example.test/v1/users/dummy/",
}
responses.get(
"http://fasjson.example.test/v1/users/dummy/",
json={"result": user_data},
)
response = client.get("/user/dummy/info")
assert response.status_code == 200
assert response.json() == user_data
def test_stats_grafana_xapi_fun_backend_HTTPError(self, logger_mock):
"""HttpError from call to the backend should return stats with 0."""
video = VideoFactory()
settings = {
"api_key": "grafana_api_key",
"api_endpoint": "https://grafana.tld/api",
"api_datasource_id": "1",
"api_datastream": "statements-ds-marsha",
}
exception = HTTPError("An error occurred")
responses.get(
url=(
f"{settings.get('api_endpoint')}/datasources/proxy/"
f"{settings.get('api_datasource_id')}/{settings.get('api_datastream')}/_count"
),
body=exception,
)
self.assertEqual({"nb_views": 0}, grafana_xapi_fun_backend(video, **settings))
logger_mock.warning.assert_called_with("Http error %s", exception)
def get_response(self, req_id):
for resp_id, resp in self.lost_and_found:
if resp_id == req_id:
self.lost_and_found.remove((resp_id, resp))
return resp
try:
resp_id, resp = responses.get(self._connection)
except responses.Dead, corpse:
# pgserver exited, or network error
import sys
print >> sys.stderr, corpse
raise SystemExit
def get_response(self, req_id):
for resp_id, resp in self.lost_and_found:
if resp_id == req_id:
self.lost_and_found.remove((resp_id, resp))
return resp
try:
resp_id, resp = responses.get(self._connection)
except responses.Dead, corpse:
# pgserver exited, or network error
import sys
print >> sys.stderr, corpse
raise SystemExit
def test_stats_grafana_xapi_fun_backend_success(self):
"""Successful call to the backend should return stats."""
video = VideoFactory()
settings = {
"api_key": "grafana_api_key",
"api_endpoint": "https://grafana.tld/api",
"api_datasource_id": "1",
"api_datastream": "statements-ds-marsha",
}
responses.get(
url=(
f"{settings.get('api_endpoint')}/datasources/proxy/"
f"{settings.get('api_datasource_id')}/{settings.get('api_datastream')}/_count"
),
match=[
responses.matchers.json_params_matcher(
{
"query": {
"query_string": {
"query": (
'verb.id:"https://w3id.org/xapi/video/verbs/played" '
f'AND object.id:"uuid://{video.id}" '
"AND result.extensions.https\\:\\/\\/w3id.org"
"\\/xapi\\/video\\/extensions\\/time:[0 TO 30]"
)
}
}
}
),
responses.matchers.header_matcher(
{
"Authorization": f"Bearer {settings.get('api_key')}",
"Content-Type": "application/json",
}
),
],
body='{"count":216,"_shards":{"total":24,"successful":24,"skipped":0,"failed":0}}',
)
self.assertEqual({"nb_views": 216}, grafana_xapi_fun_backend(video, **settings))
def test_success(self):
user = factories.UserFactory(email='*****@*****.**')
factories.EmailFactory(email='*****@*****.**',
verified=True,
user=user)
responses.get(
url=
'https://mitoc-gear.mit.edu/api-auth/v1/membership_waiver/[email protected]&[email protected]',
json={
"count":
0,
"next":
None,
"previous":
None,
"results": [{
"email": "*****@*****.**",
"alternate_emails": ["*****@*****.**"],
"membership": {
"membership_type": "NA",
"expires": "2023-05-05",
},
"waiver": {
"expires": "2023-05-04"
},
}],
},
)
membership = geardb.query_geardb_for_membership(user)
self.assertEqual(
membership,
geardb.MembershipWaiver(
email='*****@*****.**',
membership_expires=date(2023, 5, 5),
waiver_expires=date(2023, 5, 4),
),
)
def test_query_api(self, settings):
settings.GEARDB_SECRET_KEY = 'sooper.secret'
responses.get(
url="https://mitoc-gear.mit.edu/credentials",
json={
"count": 1,
"next": None,
"previous": None,
"results": [{
'user': '******',
'password': '******'
}],
},
status=200,
)
results = geardb.query_api('/credentials', user='******')
self.assertEqual(results, [{
'user': '******',
'password': '******'
}])
def test_proxy_describe_feature_type_get(self):
from c2cgeoportal_geoportal.views.tinyowsproxy import TinyOWSProxy
request = _create_dummy_request(username="******")
request.registry.settings["tinyowsproxy"].update({
"tinyows_host":
"demo.gmf.org",
"tinyows_url":
"http://example.com",
})
request.params.update(
dict(service="wfs",
version="1.1.0",
request="DescribeFeatureType",
typename="tows:layer_1"))
responses.get(
url="http://mapserver:8080/",
body="unfiltered response",
)
response = TinyOWSProxy(request).proxy()
self.assertEqual("200 OK", response.status)
def test_active_waiver_required(self):
"""Only active members are allowed on the trip."""
par = factories.ParticipantFactory.create(
membership=factories.MembershipFactory.create(
membership_expires=date(2020, 1, 5), waiver_expires=None))
mini_trip = self._upcoming_trip(membership_required=False)
# Membership is not required for the mini trip, but a waiver is!
self.assertFalse(par.should_renew_for(mini_trip))
check_waiver = responses.get(
url=
f'https://mitoc-gear.mit.edu/api-auth/v1/membership_waiver/?email={par.email}',
json={
"count":
0,
"next":
None,
"previous":
None,
"results": [{
"email": "",
"alternate_emails": [],
"membership": {
"membership_type": "NA",
"expires": "2023-05-05",
},
# No waiver!
"waiver": {},
}],
},
)
self.client.force_login(par.user)
resp = self._signup(mini_trip)
self.assertEqual(resp.status_code, 200) # (Failure, but errors shown)
# Because the user had an out-of-date waiver, we checked for the latest
self.assertEqual(check_waiver.call_count, 1)
form = resp.context['form']
self.assertEqual(form.errors,
{'__all__': ["A current waiver is required"]})
# Participant was not placed on the trip.
self.assertFalse(mini_trip.signup_set.filter(participant=par).exists())
def test_membership_required(self):
"""Only active members are allowed on the trip."""
par = factories.ParticipantFactory.create(membership=None,
email='*****@*****.**')
trip = self._upcoming_trip(membership_required=True)
self.assertTrue(par.should_renew_for(trip))
check_membership = responses.get(
url=
'https://mitoc-gear.mit.edu/api-auth/v1/membership_waiver/[email protected]',
json={
"count":
0,
"next":
None,
"previous":
None,
"results": [{
"email": "",
"alternate_emails": [],
"membership": {},
"waiver": {},
}],
},
)
self.client.force_login(par.user)
resp = self._signup(trip)
# Because the user had an out-of-date membership, we checked for the latest
self.assertEqual(check_membership.call_count, 1)
form = resp.context['form']
self.assertEqual(
form.errors,
{
'__all__': [
"An active membership is required",
"A current waiver is required",
]
},
)
# Participant was not placed on the trip.
self.assertFalse(trip.signup_set.filter(participant=par).exists())
async def search(*args, **kwargs):
logging.info(f'Recieved {kwargs}')
stat_funcs = {
'most_used_word': _get_most_used_word,
'word_frequency': _get_word_frequency,
}
func = stat_funcs[kwargs['stat']]
params = {}
query = WordCloud.query
words = await query.gino.all()
words_dump = [word.dump() for word in words]
params['words'] = words_dump
if words is None:
return responses.not_found()
else:
response = await func(params)
return responses.get(response)
def test_external_ip(self):
responses.get('https://ipapi.co/192.0.2.0/json',
json={'timezone': 'UTC'})
tz = ip_to_timezone('192.0.2.0')
self.assertEqual(tz, 'UTC')
