def test_get_max_entries():
# test min value
assert MeetupApiClient.get_max_entries(max_entries=-1) == 1
# test max value
assert MeetupApiClient.get_max_entries(max_entries=100) == 100
# test valid value
assert MeetupApiClient.get_max_entries(max_entries=1000) == 500
def get_group(meetup_group_urlname: Optional[str] = None,
sandbox: bool = False) -> Group:
"""
Load single Meetupgroup from Meetup REST API into elasticsearch
Arguments:
meetup_group_urlname {str} -- meetup group urlname to load the group from meetup
Returns:
Group -- updated group from meetup.com
"""
if sandbox:
meetup_group_urlname = "Meetup-API-Testing"
if not meetup_group_urlname:
print("No meetup_group_urlname was given!")
exit(1)
api_client: MeetupApiClient = MeetupApiClient()
try:
group: Group = api_client.get_group(meetup_group_urlname)
except (GroupDoesNotExistsOnMeetup, MeetupConnectionError) as e:
print(e)
exit(2)
group_events: List[Event] = api_client.update_all_group_events(group=group)
print("Group {} was updatet with {} events".format(group.name,
len(group_events)))
return group
def load_groups(load_events: bool, country: str):
"""
Load all groups from a country of all meetup zips saved in elasticsearch
Arguments:
load_events {bool} -- Load all past events of every group and save them into elasticsearch
country {str} -- Country code like DE for germany
"""
# meetup api client
api_client: MeetupApiClient = MeetupApiClient()
# get all zip codes from elasticsearch
meetup_zips: List[MeetupZip] = MeetupZip.get_all_zips()
print("Start fetching groups from meetup!")
for meetup_zip in meetup_zips:
groups: List[Group] = api_client.search_new_groups(
zip_code=meetup_zip.zip_code, country_code=country)
sleep(1)
if load_events:
for group in groups:
print(group.urlname)
try:
api_client.update_all_group_events(group=group)
except Exception as e:
print(e)
print("{} groups was added to elasticsearch for zip {}!".format(
len(groups), meetup_zip.zip_code))
def api_client() -> MeetupApiClient:
"""
meetup api client
Returns:
MeetupApiClient -- Meetup Api client
"""
return MeetupApiClient()
def test_get_zip_from_meetup(api_client: MeetupApiClient):
# get zip codes
zip_code_list: List[str] = api_client.get_zip_from_meetup(lat=52.520008,
lon=13.404954)
assert len(zip_code_list) > 0
for zip_code in zip_code_list:
assert isinstance(zip_code, str)
def get_groups(meetup_files_path: str,
load_events: bool) -> Dict[str, List[str]]:
"""
parse all JSON files in meetup_files_path, get the group name and index every group into
elasticsearch
Arguments:
meetup_files_path {str} -- path of the JSON files
load_events {bool} -- load all events from groups
Returns:
Dict[str, List[str]] -- dict with valid & invalid group lists
"""
api_client: MeetupApiClient = MeetupApiClient()
mettup_groups_files: List[str] = glob.glob(
"{}/*.json".format(meetup_files_path))
groups_dict: Dict[str, List[str]] = {"valid": [], "invalid": []}
event_counter: int = 0
for mettup_groups_file in mettup_groups_files:
with open(mettup_groups_file) as json_file:
data = json.load(json_file)
for group_data in data:
try:
group: Group = api_client.get_group(
data[group_data]["urlname"])
except (GroupDoesNotExistsOnMeetup,
MeetupConnectionError) as e:
print(e)
groups_dict["invalid"].append(data[group_data]["urlname"])
continue
groups_dict["valid"].append(data[group_data]["urlname"])
if load_events:
group_events: List[
Event] = api_client.update_all_group_events(
group=group)
event_counter = event_counter + len(group_events)
print("Group {} was updatet with {} events".format(
group.name, len(group_events)))
else:
print("Group {} was updatet without events".format(
group.name, ))
print(
"{} groups was updatet with {} new events & {} do not exists anymore".
format(len(groups_dict["valid"]), event_counter,
len(groups_dict["invalid"])))
return groups_dict
def test_get_all_zip_from_meetup(api_client: MeetupApiClient):
# get all zip codes from Switzerland
zip_code_list: List[str] = api_client.get_all_zip_from_meetup(
min_lat=47.270114,
max_lat=55.099161,
min_lon=5.8663153,
max_lon=15.0418087)
assert len(zip_code_list) > 10000
for zip_code in zip_code_list:
assert isinstance(zip_code, str)
def test_update_all_group_events(meetup_groups: dict,
api_client: MeetupApiClient):
# get sandbox group
group_1: Group = api_client.get_group(
group_urlname=meetup_groups["sandbox"]["urlname"])
# get all events
events_1: List[Event] = api_client.update_all_group_events(group=group_1)
assert isinstance(events_1[0], Event)
assert len(events_1) > 200
# load all events from elasticseach
group_2: Group = api_client.get_group(
group_urlname=meetup_groups["sandbox"]["urlname"])
# check if all events was saved into elasticsearch
assert len(events_1) == len(group_2.events)
# check if there still some events
events_3: List[Event] = api_client.update_all_group_events(group=group_2)
assert len(events_3) == 0
def update_groups():
"""
update for all groups new events
"""
# get all groups
groups: List[Group] = Group.get_all_groups()
# init api client
api_client: MeetupApiClient = MeetupApiClient()
# update all groups
for group in groups:
api_client.update_all_group_events(group=group)
def test_update_group_events(meetup_groups: dict, api_client: MeetupApiClient):
# get sandbox group
group_1: Group = api_client.get_group(
group_urlname=meetup_groups["sandbox"]["urlname"])
# get events when group has no event
events_1: List[Event] = api_client.update_group_events(group=group_1,
max_entries=10)
assert isinstance(events_1[0], Event)
assert len(events_1) == 10
# save events into group
group_1.add_events(events=events_1)
group_1.save()
sleep(1)
# check if there was no double request
events_2: List[Event] = api_client.update_group_events(group=group_1,
max_entries=10)
assert isinstance(events_1[0], Event)
assert len(events_1) == 10
for event_1 in events_1:
for event_2 in events_2:
assert event_1.meetup_id != event_2.meetup_id
# check for min max_entries
events_3: List[Event] = api_client.update_group_events(group=group_1,
max_entries=-10)
assert isinstance(events_3[0], Event)
assert len(events_3) == 1
# check for max max_entries
events_4: List[Event] = api_client.update_group_events(group=group_1,
max_entries=1000)
assert isinstance(events_3[0], Event)
assert len(events_4) == 200
def load_zip_codes(lat_min: float, lat_max: float, lon_min: float,
lon_max: float):
"""
Load all meetup zip codes from a boundingbox [min_lat, max_lat, min_lon, max_lon]
Arguments:
lat_min {float} -- boundingbox lat min
lat_max {float} -- boundingbox lat max
lon_min {float} -- boundingbox lon min
lon_max {float} -- boundingbox lon max
"""
# init api client
api_client: MeetupApiClient = MeetupApiClient()
try:
# get all zip codes from Switzerland
zip_code_list: List[str] = api_client.get_all_zip_from_meetup(
min_lat=lat_min,
max_lat=lat_max,
min_lon=lon_min,
max_lon=lon_max,
)
except (
HttpNotFoundError,
HttpNotAccessibleError,
HttpNoSuccess,
HttpNoXRateLimitHeader,
) as e:
print(e)
print("Network error, please try again later!")
print("There was no zip code added to elasticsearch!")
exit(1)
meetup_zips: List[MeetupZip] = MeetupZip.get_or_create_zips(
zip_code_list=zip_code_list)
print("{} zip codes was updated!".format(len(meetup_zips)))
def test_get_group(httpserver: HTTPServer, meetup_groups: dict,
api_client: MeetupApiClient):
# check existing group
group_1: Group = api_client.get_group(
group_urlname=meetup_groups["sandbox"]["urlname"])
assert isinstance(group_1, Group)
assert group_1.meetup_id == meetup_groups["sandbox"]["meetup_id"]
# check not existing group
with pytest.raises(GroupDoesNotExistsOnMeetup):
api_client.get_group(
group_urlname=meetup_groups["not-exist"]["urlname"])
# create gone group object in elasticsearch
group_2 = create_group(urlname=meetup_groups["gone"]["urlname"])
group_2.save()
sleep(1)
# check gone group
with pytest.raises(GroupDoesNotExistsOnMeetup):
api_client.get_group(group_urlname=meetup_groups["gone"]["urlname"])
sleep(1)
# check if gone group was deleted
with pytest.raises(GroupDoesNotExists):
Group.get_group(urlname=meetup_groups["gone"]["urlname"])
# create gone group object in elasticsearch
group_2 = create_group(urlname=meetup_groups["gone"]["urlname"])
group_2.save()
sleep(1)
# test for HttpNoXRateLimitHeader execption
for _ in range(4):
httpserver.expect_oneshot_request(
"/HttpNoXRateLimitHeader").respond_with_data("OK")
api_client.base_url = httpserver.url_for("/HttpNoXRateLimitHeader")
with pytest.raises(MeetupConnectionError):
api_client.get_group(group_urlname=meetup_groups["gone"]["urlname"])
# check if gone group was not deleted
assert Group.get_group(
urlname=meetup_groups["gone"]["urlname"]) is not None
# test for HttpNoSuccess execption
for _ in range(4):
httpserver.expect_oneshot_request("/HttpNoSuccess")
api_client.base_url = httpserver.url_for("")
with pytest.raises(MeetupConnectionError):
api_client.get_group(group_urlname=meetup_groups["gone"]["urlname"])
# check if gone group was not deleted
assert Group.get_group(
urlname=meetup_groups["gone"]["urlname"]) is not None
def test_get(httpserver: HTTPServer, meetup_groups: dict,
api_client: MeetupApiClient):
with pytest.raises(HttpNotFoundError):
api_client.get(meetup_groups["not-exist"]["urlname"])
with pytest.raises(HttpNotAccessibleError):
api_client.get(meetup_groups["gone"]["urlname"])
json: dict = api_client.get(meetup_groups["sandbox"]["urlname"])
assert isinstance(json, dict)
assert json["id"] == meetup_groups["sandbox"]["meetup_id"]
# test for HttpNoXRateLimitHeader execption
for _ in range(4):
httpserver.expect_oneshot_request(
"/HttpNoXRateLimitHeader").respond_with_data("OK")
api_client.base_url = httpserver.url_for("/HttpNoXRateLimitHeader")
with pytest.raises(HttpNoXRateLimitHeader):
api_client.get(url_path="", reset_time=2)
# test for HttpNoSuccess execption
for _ in range(4):
httpserver.expect_oneshot_request("/HttpNoSuccess")
api_client.base_url = httpserver.url_for("")
with pytest.raises(HttpNoSuccess):
api_client.get(url_path="/", reset_time=2)