def test_query_deeds_page_json_from_api(self):
self.scraper = CoronaHelpersScraper()
with patch('requests.get',
return_value=self.deeds_test_response_mock()):
deeds_query_results = self.scraper.query_deeds_page_json_from_api(
1)
self.assertTrue(len(deeds_query_results) > 1)
def test_getCoordinatesFromDeedDetails(self):
self.scraper = CoronaHelpersScraper()
json_test_data = {"coordinates": None}
coordinates = self.scraper.getCoordinatesFromDeedDetails(json_test_data)
self.assertEqual(coordinates["lat"], None)
self.assertEqual(coordinates["lng"], None)
json_test_data = {"coordinates": {"lat": 123, "lng": 321}}
coordinates = self.scraper.getCoordinatesFromDeedDetails(json_test_data)
self.assertEqual(coordinates["lat"], 123)
self.assertEqual(coordinates["lng"], 321)
def test_get_deeds_from_page_json(self):
json_test_data = {
"status": 200,
"error": None,
"data": {
"results": "foobar"
}
}
self.scraper = CoronaHelpersScraper()
deed_details = self.scraper.get_deeds_from_page_json(json_test_data)
self.assertEqual(deed_details, "foobar")
def test_getHTTPParametersForPageQuery(self):
self.scraper = CoronaHelpersScraper()
http_parameters_result = self.scraper.getHTTPParametersForPageQuery(123, 321)
http_parameters_should_be = {
'query': '',
'page': '123',
'causes': '',
'activities': '',
'date': '',
'pageSize': '321',
'withOrganization': 'true'}
self.assertEqual(http_parameters_result, http_parameters_should_be)
def test_get_deed_details_from_json(self):
json_test_data = {
"status": 200,
"error": None,
"data": {
"deed": {
"title": "foobar"
}
}
}
self.scraper = CoronaHelpersScraper()
deed_details = self.scraper.get_deed_details_from_json(json_test_data)
self.assertEqual(deed_details["title"], "foobar")
def test_createInitiativeFromDeedDetails(self):
self.scraper = CoronaHelpersScraper()
test_feed_details = {
"id": 1337,
"fullType": "volunteering_long",
"summary": "test",
"address": "someplace",
"coordinates": {"lat": 52.2, "lng": 6.2}
}
initiative = self.scraper.createInitiativeFromDeedDetails(test_feed_details)
self.assertEqual(initiative.source, "https://www.coronahelpers.nl/api/deeds/1337")
self.assertEqual(initiative.category, "volunteering_long")
self.assertEqual(initiative.description, "test")
self.assertEqual(initiative.location, "someplace")
self.assertEqual(initiative.latitude, 52.2)
self.assertEqual(initiative.longitude, 6.2)
def test_get_page_count_from_page_json(self):
json_test_data = {
"status": 200,
"error": None,
"data": {
"results": [{}],
"pagination": {
"page": 1,
"pageSize": 2,
"rowCount": 826,
"pageCount": 413
}
}
}
self.scraper = CoronaHelpersScraper()
page_count = self.scraper.get_page_count_from_page_json(json_test_data)
self.assertEqual(page_count, 413)
class TestCoronaHelpersScraper(TestCase):
@staticmethod
def deeds_test_response_mock():
mock = Mock()
response = responses.read("coronahelpers_deeds.json")
mock.content = response
return mock
def setUp(self):
self.scraper = CoronaHelpersScraper()
def test_create_scraper_object(self):
self.assertEqual(type(self.scraper), CoronaHelpersScraper)
self.assertIsNotNone(self.scraper.domain)
def test_get_base_url(self):
url = self.scraper.get_base_url()
self.assertEqual(url, "https://www.coronahelpers.nl")
def test_check_connection_to_server(self):
with patch('requests.get',
return_value=self.deeds_test_response_mock()):
self.assertTrue(self.scraper.check_connection_to_server())
def test_get_page_count_from_api(self):
self.scraper = CoronaHelpersScraper()
with patch('requests.get',
return_value=self.deeds_test_response_mock()):
page_count = self.scraper.get_page_count_from_api()
self.assertTrue(int(page_count) > 1)
def test_query_deeds_page_json_from_api(self):
self.scraper = CoronaHelpersScraper()
with patch('requests.get',
return_value=self.deeds_test_response_mock()):
deeds_query_results = self.scraper.query_deeds_page_json_from_api(
1)
self.assertTrue(len(deeds_query_results) > 1)
def test_get_api_deeds_url(self):
self.scraper = CoronaHelpersScraper()
url = self.scraper.get_api_deeds_url()
self.assertEqual(url, "https://www.coronahelpers.nl/api/deeds")
def test_get_api_deed_details_url(self):
self.scraper = CoronaHelpersScraper()
url = self.scraper.get_api_deed_details_url(123)
self.assertEqual(url, "https://www.coronahelpers.nl/api/deeds/123")
def test_get_data_from_json(self):
self.scraper = CoronaHelpersScraper()
test_json = {"data": "foobar"}
data = self.scraper.get_data_from_json(test_json)
self.assertEqual(data, "foobar")
def test_get_status_from_json(self):
self.scraper = CoronaHelpersScraper()
test_json = {"status": "foobar"}
data = self.scraper.get_status_from_json(test_json)
self.assertEqual(data, "foobar")
def test_get_deed_details_from_json(self):
json_test_data = {
"status": 200,
"error": None,
"data": {
"deed": {
"title": "foobar"
}
}
}
self.scraper = CoronaHelpersScraper()
deed_details = self.scraper.get_deed_details_from_json(json_test_data)
self.assertEqual(deed_details["title"], "foobar")
def test_get_deeds_from_page_json(self):
json_test_data = {
"status": 200,
"error": None,
"data": {
"results": "foobar"
}
}
self.scraper = CoronaHelpersScraper()
deed_details = self.scraper.get_deeds_from_page_json(json_test_data)
self.assertEqual(deed_details, "foobar")
def test_get_deed_id_from_json(self):
self.scraper = CoronaHelpersScraper()
test_json = {"id": "foobar"}
data = self.scraper.get_deed_id_from_json(test_json)
self.assertEqual(data, "foobar")
def test_get_page_count_from_page_json(self):
json_test_data = {
"status": 200,
"error": None,
"data": {
"results": [{}],
"pagination": {
"page": 1,
"pageSize": 2,
"rowCount": 826,
"pageCount": 413
}
}
}
self.scraper = CoronaHelpersScraper()
page_count = self.scraper.get_page_count_from_page_json(json_test_data)
self.assertEqual(page_count, 413)
def test_get_coordinates_from_deed_details(self):
self.scraper = CoronaHelpersScraper()
json_test_data = {"coordinates": None}
coordinates = self.scraper.get_coordinates_from_deed_details(
json_test_data)
self.assertEqual(coordinates["lat"], None)
self.assertEqual(coordinates["lng"], None)
json_test_data = {"coordinates": {"lat": 123, "lng": 321}}
coordinates = self.scraper.get_coordinates_from_deed_details(
json_test_data)
self.assertEqual(coordinates["lat"], 123)
self.assertEqual(coordinates["lng"], 321)
def test_get_http_parameters_for_page_query(self):
self.scraper = CoronaHelpersScraper()
http_parameters_result = self.scraper.get_http_parameters_for_page_query(
123, 321)
http_parameters_should_be = {
'query': '',
'page': '123',
'causes': '',
'activities': '',
'date': '',
'pageSize': '321',
'withOrganization': 'true'
}
self.assertEqual(http_parameters_result, http_parameters_should_be)
def test_create_initiative_from_deed_details(self):
self.scraper = CoronaHelpersScraper()
test_feed_details = {
"id": 1337,
"fullType": "volunteering_long",
"summary": "test",
"address": "someplace",
"coordinates": {
"lat": 52.2,
"lng": 6.2
}
}
initiative = self.scraper.create_initiative_from_deed_details(
test_feed_details)
self.assertEqual(initiative.source,
"https://www.coronahelpers.nl/api/deeds/1337")
self.assertEqual(initiative.category, "volunteering_long")
self.assertEqual(initiative.description, "test")
self.assertEqual(initiative.location, "someplace")
self.assertEqual(initiative.latitude, 52.2)
self.assertEqual(initiative.longitude, 6.2)
def test_get_api_deed_details_url(self):
self.scraper = CoronaHelpersScraper()
url = self.scraper.get_api_deed_details_url(123)
self.assertEqual(url, "https://www.coronahelpers.nl/api/deeds/123")
def test_getPageCountFromAPI(self):
self.scraper = CoronaHelpersScraper()
page_count = self.scraper.getPageCountFromAPI()
self.assertTrue(int(page_count) > 1)
def test_get_page_count_from_api(self):
self.scraper = CoronaHelpersScraper()
with patch('requests.get',
return_value=self.deeds_test_response_mock()):
page_count = self.scraper.get_page_count_from_api()
self.assertTrue(int(page_count) > 1)
def setUp(self):
self.scraper = CoronaHelpersScraper()
def test_get_deed_id_from_json(self):
self.scraper = CoronaHelpersScraper()
test_json = {"id": "foobar"}
data = self.scraper.get_deed_id_from_json(test_json)
self.assertEqual(data, "foobar")
def test_queryDeedsPageJSONFromAPI(self):
self.scraper = CoronaHelpersScraper()
deeds_query_results = self.scraper.queryDeedsPageJSONFromAPI(1)
self.assertTrue(len(deeds_query_results) > 1)
def test_getDeedIDFromJSON(self):
self.scraper = CoronaHelpersScraper()
test_json = {"id": "foobar"}
data = self.scraper.getDeedIDFromJSON(test_json)
self.assertEqual(data, "foobar")
class TestCoronaHelpersScraper(TestCase):
def setUp(self):
self.scraper = CoronaHelpersScraper()
def test_createScraperObject(self):
self.assertEqual(type(self.scraper), CoronaHelpersScraper)
self.assertIsNotNone(self.scraper.domain)
def test_getBaseURL(self):
url = self.scraper.getBaseURL()
self.assertEqual(url, "https://www.coronahelpers.nl")
def test_checkConnectionToServer(self):
self.assertTrue(self.scraper.checkConnectionToServer())
def test_getPageCountFromAPI(self):
self.scraper = CoronaHelpersScraper()
page_count = self.scraper.getPageCountFromAPI()
self.assertTrue(int(page_count) > 1)
def test_queryDeedsPageJSONFromAPI(self):
self.scraper = CoronaHelpersScraper()
deeds_query_results = self.scraper.queryDeedsPageJSONFromAPI(1)
self.assertTrue(len(deeds_query_results) > 1)
def test_getAPIDeedsURL(self):
self.scraper = CoronaHelpersScraper()
url = self.scraper.getAPIDeedsURL()
self.assertEqual(url, "https://www.coronahelpers.nl/api/deeds")
def test_getAPIDeedDetailsURL(self):
self.scraper = CoronaHelpersScraper()
url = self.scraper.getAPIDeedDetailsURL(123)
self.assertEqual(url, "https://www.coronahelpers.nl/api/deeds/123")
def test_getDataFromJSON(self):
self.scraper = CoronaHelpersScraper()
test_json = {"data": "foobar"}
data = self.scraper.getDataFromJSON(test_json)
self.assertEqual(data, "foobar")
def test_getStatusFromJSON(self):
self.scraper = CoronaHelpersScraper()
test_json = {"status": "foobar"}
data = self.scraper.getStatusFromJSON(test_json)
self.assertEqual(data, "foobar")
def test_getDeedDetailsFromJSON(self):
json_test_data = {"status": 200, "error": None, "data": {"deed": {"title": "foobar"}}}
self.scraper = CoronaHelpersScraper()
deed_details = self.scraper.getDeedDetailsFromJSON(json_test_data)
self.assertEqual(deed_details["title"], "foobar")
def test_getDeedsFromPageJSON(self):
json_test_data = {"status": 200, "error": None, "data": {"results": "foobar"}}
self.scraper = CoronaHelpersScraper()
deed_details = self.scraper.getDeedsFromPageJSON(json_test_data)
self.assertEqual(deed_details, "foobar")
def test_getDeedIDFromJSON(self):
self.scraper = CoronaHelpersScraper()
test_json = {"id": "foobar"}
data = self.scraper.getDeedIDFromJSON(test_json)
self.assertEqual(data, "foobar")
def test_getPageCountFromPageJSON(self):
json_test_data = {"status": 200, "error": None, "data": {"results": [{}], "pagination": {"page": 1, "pageSize": 2, "rowCount": 826, "pageCount": 413}}}
self.scraper = CoronaHelpersScraper()
page_count = self.scraper.getPageCountFromPageJSON(json_test_data)
self.assertEqual(page_count, 413)
def test_getCoordinatesFromDeedDetails(self):
self.scraper = CoronaHelpersScraper()
json_test_data = {"coordinates": None}
coordinates = self.scraper.getCoordinatesFromDeedDetails(json_test_data)
self.assertEqual(coordinates["lat"], None)
self.assertEqual(coordinates["lng"], None)
json_test_data = {"coordinates": {"lat": 123, "lng": 321}}
coordinates = self.scraper.getCoordinatesFromDeedDetails(json_test_data)
self.assertEqual(coordinates["lat"], 123)
self.assertEqual(coordinates["lng"], 321)
def test_getHTTPParametersForPageQuery(self):
self.scraper = CoronaHelpersScraper()
http_parameters_result = self.scraper.getHTTPParametersForPageQuery(123, 321)
http_parameters_should_be = {
'query': '',
'page': '123',
'causes': '',
'activities': '',
'date': '',
'pageSize': '321',
'withOrganization': 'true'}
self.assertEqual(http_parameters_result, http_parameters_should_be)
def test_createInitiativeFromDeedDetails(self):
self.scraper = CoronaHelpersScraper()
test_feed_details = {
"id": 1337,
"fullType": "volunteering_long",
"summary": "test",
"address": "someplace",
"coordinates": {"lat": 52.2, "lng": 6.2}
}
initiative = self.scraper.createInitiativeFromDeedDetails(test_feed_details)
self.assertEqual(initiative.source, "https://www.coronahelpers.nl/api/deeds/1337")
self.assertEqual(initiative.category, "volunteering_long")
self.assertEqual(initiative.description, "test")
self.assertEqual(initiative.location, "someplace")
self.assertEqual(initiative.latitude, 52.2)
self.assertEqual(initiative.longitude, 6.2)
def test_getAPIDeedDetailsURL(self):
self.scraper = CoronaHelpersScraper()
url = self.scraper.getAPIDeedDetailsURL(123)
self.assertEqual(url, "https://www.coronahelpers.nl/api/deeds/123")
"--limit",
help="LIM is the amount of items each scraper scrapes.",
type="int",
metavar="LIM")
parser.add_option("-g",
"--nogeo",
help="Disables the geocoder",
action="store_true",
dest="nogeo")
# HeldNodig().scrape()
# MensenDieWillenHelpen().scrape()
# Zorgheldenauto().scrape()
# PuurPapendrecht().scrape()
# NijmegenOost().scrape()
scrapers = [NLvoorElkaar(), WijAmsterdam(), CoronaHelpersScraper()]
def docs():
parser.print_help()
print('')
print('If no arguments given, runs all scrapers')
print('Use one or more codes as arguments to specify the scrapers to run:')
print('{: <16}{}'.format("Code", "Name"))
print('{: <16}{}'.format("____", "____"))
for s in scrapers:
print('{: <16}{}'.format(s.code, s.name))
(opts, args) = parser.parse_args()
type=str, dest="group")
parser.add_argument("-gc", "--geocoder=city", help="Enables the geocoder on city level",
action="store_const", const=FeatureType.CITY, dest="geocoder")
parser.add_argument("-gs", "--geocoder=settlement", help="Enables the geocoder on settlement level",
action="store_const", const=FeatureType.SETTLEMENT, dest="geocoder")
parser.add_argument("-ga", "--geocoder=address", help="Enables the geocoder on address level",
action="store_const", const=FeatureType.ADDRESS, dest="geocoder")
parser.add_argument("-s", "--scrapers", help="specifies specific scrapers to run", action="append", nargs="+", type=str)
# HeldNodig().scrape()
# MensenDieWillenHelpen().scrape()
# Zorgheldenauto().scrape()
scrapers = [
NLvoorElkaar(),
WijAmsterdam(),
CoronaHelpersScraper(),
PuurPapendrecht(),
NijmegenOost(),
Maasburen(),
CoronaPioniers()]
def docs():
parser.print_help()
print('')
print('If no arguments given, runs all scrapers')
print('Use one or more codes as arguments to specify the scrapers to run:')
print('{: <16}{}'.format("Code", "Name"))
print('{: <16}{}'.format("____", "____"))
for s in scrapers:
print('{: <16}{}'.format(s.code, s.name))
