def _wait_data_worker(self, latest, proto_to_wait_for, timestamp):
if latest is None:
logger.debug("Nothing received since MAD started")
time.sleep(0.5)
elif 156 in latest and latest[156].get('timestamp', 0) >= timestamp:
return LatestReceivedType.GYM
elif 102 in latest and latest[102].get('timestamp', 0) >= timestamp:
return LatestReceivedType.MON
elif proto_to_wait_for not in latest:
logger.debug(
"No data linked to the requested proto since MAD started.")
time.sleep(0.5)
else:
# proto has previously been received, let's check the timestamp...
# TODO: int vs str-key?
latest_proto = latest.get(proto_to_wait_for, None)
latest_timestamp = latest_proto.get("timestamp", 0)
if latest_timestamp >= timestamp:
# TODO: consider reseting timestamp here since we clearly received SOMETHING
latest_data = latest_proto.get("values", None)
logger.debug4("Latest data received: {}".format(str(latest_data)))
if latest_data is None:
time.sleep(0.5)
return None
elif proto_to_wait_for == 101:
payload: dict = latest_data.get("payload", None)
if payload is None:
return None
result: int = latest_data.get("payload", {}).get("result", 0)
if result == 1 and len(payload.get('items_awarded', [])) > 0:
return FortSearchResultTypes.QUEST
elif (result == 1
and len(payload.get('items_awarded', [])) == 0):
return FortSearchResultTypes.TIME
elif result == 2:
return FortSearchResultTypes.OUT_OF_RANGE
elif result == 3:
return FortSearchResultTypes.COOLDOWN
elif result == 4:
return FortSearchResultTypes.INVENTORY
elif result == 5:
return FortSearchResultTypes.LIMIT
elif proto_to_wait_for == 104:
fort_type: int = latest_data.get("payload").get("type", 0)
if fort_type == 0:
return LatestReceivedType.GYM
else:
return LatestReceivedType.STOP
if proto_to_wait_for == 4 and 'inventory_delta' in latest_data['payload'] and \
len(latest_data['payload']['inventory_delta']['inventory_items']) > 0:
return LatestReceivedType.CLEAR
if proto_to_wait_for == PROTO_NUMBER_FOR_GMO:
return LatestReceivedType.GMO
else:
logger.debug("latest timestamp of proto {} ({}) is older than {}", str(
proto_to_wait_for), str(latest_timestamp), str(timestamp))
# TODO: timeoutopen error instead of data_error_counter? Differentiate timeout vs missing data (the
# TODO: latter indicates too high speeds for example
time.sleep(0.5)
return LatestReceivedType.UNDEFINED
def process_data(self, received_timestamp, data, origin):
type = data.get("type", None)
raw = data.get("raw", False)
if raw:
logger.debug5("Received raw payload: {}", data["payload"])
if type and not raw:
self.__mitm_mapper.run_stats_collector(origin)
logger.debug4("Received payload: {}", data["payload"])
if type == 106:
# process GetMapObject
logger.success(
"Processing GMO received from {}. Received at {}",
str(origin),
str(datetime.fromtimestamp(received_timestamp)))
if self.__application_args.weather:
self.__db_wrapper.submit_weather_map_proto(
origin, data["payload"], received_timestamp)
self.__db_wrapper.submit_pokestops_map_proto(
origin, data["payload"])
self.__db_wrapper.submit_gyms_map_proto(
origin, data["payload"])
self.__db_wrapper.submit_raids_map_proto(
origin, data["payload"], self.__mitm_mapper)
self.__db_wrapper.submit_spawnpoints_map_proto(
origin, data["payload"])
mon_ids_iv = self.__mitm_mapper.get_mon_ids_iv(origin)
self.__db_wrapper.submit_mons_map_proto(
origin, data["payload"], mon_ids_iv, self.__mitm_mapper)
elif type == 102:
playerlevel = self.__mitm_mapper.get_playerlevel(origin)
if playerlevel >= 30:
logger.info("Processing Encounter received from {} at {}",
str(origin), str(received_timestamp))
self.__db_wrapper.submit_mon_iv(origin, received_timestamp,
data["payload"],
self.__mitm_mapper)
else:
logger.debug(
'Playerlevel lower than 30 - not processing encounter Data'
)
elif type == 101:
self.__db_wrapper.submit_quest_proto(origin, data["payload"],
self.__mitm_mapper)
elif type == 104:
self.__db_wrapper.submit_pokestops_details_map_proto(
data["payload"])
elif type == 4:
self.__mitm_mapper.generate_player_stats(
origin, data["payload"])
def execute(self, sql, args=None, commit=False, **kwargs):
"""
Execute a sql, it could be with args and with out args. The usage is
similar with execute() function in module pymysql.
:param sql: sql clause
:param args: args need by sql clause
:param commit: whether to commit
:return: if commit, return None, else, return result
"""
self._connection_semaphore.acquire()
conn = self._pool.get_connection()
cursor = self.setup_cursor(conn, **kwargs)
get_id = kwargs.get('get_id', False)
get_dict = kwargs.get('get_dict', False)
raise_exc = kwargs.get('raise_exc', False)
try:
if args:
if type(args) != tuple:
args = (args, )
cursor.execute(sql, args)
else:
cursor.execute(sql)
logger.debug4(cursor.statement)
if commit is True:
affected_rows = cursor.rowcount
conn.commit()
if get_id:
return cursor.lastrowid
else:
return affected_rows
else:
res = cursor.fetchall()
if get_dict:
return self.__convert_to_dict(cursor.column_names, res)
return res
except mysql.connector.Error as err:
logger.error("Failed executing query: {}, error: {}", str(sql),
str(err))
logger.debug(sql)
logger.debug(args)
return None
except Exception as e:
logger.error("Unspecified exception in dbWrapper: {}", str(e))
return None
finally:
self.close(conn, cursor)
self._connection_semaphore.release()
def submit_gmo_for_location(self, origin, payload):
logger.debug4("submit_gmo_for_location of {}", origin)
cells = payload.get("cells", None)
if cells is None:
return
current_cells_id = sorted(list(map(lambda x: x['id'], cells)))
if origin in self.__last_cellsid:
last_cells_id = self.__last_cellsid[origin]
self.__last_cellsid[origin] = current_cells_id
if last_cells_id != current_cells_id:
self.__last_possibly_moved[origin] = time.time()
else:
self.__last_cellsid[origin] = current_cells_id
self.__last_possibly_moved[origin] = time.time()
logger.debug4("Done submit_gmo_for_location of {} with {}", origin,
current_cells_id)
def proto_endpoint(self, origin, data):
logger.debug2("Receiving proto from {}".format(origin))
logger.debug4("Proto data received from {}: {}".format(origin, str(data)))
type = data.get("type", None)
if type is None or type == 0:
logger.warning(
"Could not read method ID. Stopping processing of proto")
return None
if not self.__mitm_mapper.get_injection_status(origin):
logger.info("Worker {} is injected now", str(origin))
self.__mitm_mapper.set_injection_status(origin)
# extract timestamp from data
timestamp: float = data.get("timestamp", int(time.time()))
self.__mitm_mapper.update_latest(
origin, timestamp_received_raw=timestamp, timestamp_received_receiver=time.time(), key=type,
values_dict=data)
logger.debug3("Placing data received by {} to data_queue".format(origin))
self._data_queue.put(
(timestamp, data, origin)
)
return None
def process_data(self, received_timestamp, data, origin):
data_type = data.get("type", None)
raw = data.get("raw", False)
logger.debug2("Processing data of {}".format(origin))
if raw:
logger.debug5("Received raw payload: {}", data["payload"])
if data_type and not raw:
logger.debug2("Running stats collector of {}".format(origin))
if self.__application_args.game_stats:
self.__mitm_mapper.run_stats_collector(origin)
logger.debug4("Received payload: {}", data["payload"])
if data_type == 106:
# process GetMapObject
logger.success(
"Processing GMO received from {}. Received at {}",
str(origin),
str(datetime.fromtimestamp(received_timestamp)))
if self.__application_args.weather:
self.__db_submit.weather(origin, data["payload"],
received_timestamp)
self.__db_submit.stops(origin, data["payload"])
self.__db_submit.gyms(origin, data["payload"])
self.__db_submit.raids(origin, data["payload"],
self.__mitm_mapper)
self.__db_submit.spawnpoints(origin, data["payload"])
mon_ids_iv = self.__mitm_mapper.get_mon_ids_iv(origin)
self.__db_submit.mons(origin, data["payload"], mon_ids_iv,
self.__mitm_mapper)
self.__db_submit.cells(origin, data["payload"])
self.__mitm_mapper.submit_gmo_for_location(
origin, data["payload"])
logger.debug2("Done processing GMO of {}".format(origin))
elif data_type == 102:
playerlevel = self.__mitm_mapper.get_playerlevel(origin)
if playerlevel >= 30:
logger.info("Processing Encounter received from {} at {}",
str(origin), str(received_timestamp))
self.__db_submit.mon_iv(origin, received_timestamp,
data["payload"],
self.__mitm_mapper)
logger.debug2(
"Done processing encounter of {}".format(origin))
else:
logger.debug(
'Playerlevel lower than 30 - not processing encounter Data'
)
elif data_type == 101:
logger.debug2("Processing proto 101 of {}".format(origin))
self.__db_submit.quest(origin, data["payload"],
self.__mitm_mapper)
logger.debug2("Done processing proto 101 of {}".format(origin))
elif data_type == 104:
logger.debug2("Processing proto 104 of {}".format(origin))
self.__db_submit.stop_details(data["payload"])
logger.debug2("Done processing proto 104 of {}".format(origin))
elif data_type == 4:
logger.debug2("Processing proto 4 of {}".format(origin))
self.__mitm_mapper.generate_player_stats(
origin, data["payload"])
logger.debug2("Done processing proto 4 of {}".format(origin))
elif data_type == 156:
logger.debug2("Processing proto 156 of {}".format(origin))
self.__db_submit.gym(origin, data["payload"])
logger.debug2("Done processing proto 156 of {}".format(origin))
def __send_webhook(self, payload):
if len(payload) == 0:
logger.debug("Payload empty. Skip sending to webhook.")
return
# get list of urls
webhooks = self.__args.webhook_url.replace(" ", "").split(",")
webhook_count = len(webhooks)
current_wh_num = 1
for webhook in webhooks:
payloadToSend = []
subTypes = "all"
url = webhook.strip()
if url.startswith("["):
endIndex = webhook.rindex("]")
endIndex += 1
subTypes = webhook[:endIndex]
url = url[endIndex:]
for payloadData in payload:
if payloadData["type"] in subTypes:
payloadToSend.append(payloadData)
else:
payloadToSend = payload
if len(payloadToSend) == 0:
logger.debug("Payload empty. Skip sending to: {} (Filter: {})",
url, subTypes)
continue
else:
logger.debug("Sending to webhook url: {} (Filter: {})", url,
subTypes)
payload_list = self.__payload_chunk(
payloadToSend, self.__args.webhook_max_payload_size)
current_pl_num = 1
for payload_chunk in payload_list:
logger.debug4("Python data for payload: {}",
str(payload_chunk))
logger.debug3("Payload: {}", str(json.dumps(payload_chunk)))
try:
response = requests.post(
url,
data=json.dumps(payload_chunk),
headers={"Content-Type": "application/json"},
timeout=5,
)
if response.status_code != 200:
logger.warning(
"Got status code other than 200 OK from webhook destination: {}",
str(response.status_code),
)
else:
if webhook_count > 1:
whcount_text = " [wh {}/{}]".format(
current_wh_num, webhook_count)
else:
whcount_text = ""
if len(payload_list) > 1:
whchunk_text = " [pl {}/{}]".format(
current_pl_num, len(payload_list))
else:
whchunk_text = ""
logger.success(
"Successfully sent payload to webhook{}{}. Stats: {}",
whchunk_text,
whcount_text,
json.dumps(
self.__payload_type_count(payload_chunk)),
)
except Exception as e:
logger.warning(
"Exception occured while sending webhook: {}", str(e))
current_pl_num += 1
current_wh_num += 1