def __call__(self, *args, **kwargs):
logger.debug3("HTTP Request from {}".format(str(request.remote_addr)))
origin = request.headers.get('Origin')
abort = False
if request.url_rule is not None and str(request.url_rule) == '/status/':
auth = request.headers.get('Authorization', False)
if self.application_args.mitm_status_password != "" and \
(not auth or auth != self.application_args.mitm_status_password):
self.response = Response(status=500, headers={})
abort = True
else:
abort = False
elif request.url is not None and str(request.url_rule) == '/origin_generator':
auth = request.headers.get('Authorization', None)
if auth is None or not check_auth(auth, self.application_args,
self.mapping_manager.get_auths()):
logger.warning(
"Unauthorized attempt to POST from {}", str(request.remote_addr))
self.response = Response(status=403, headers={})
abort = True
else:
if not origin:
logger.warning("Missing Origin header in request")
self.response = Response(status=500, headers={})
abort = True
elif (self.mapping_manager.get_all_devicemappings().keys() is not None
and (origin is None or origin not in self.mapping_manager.get_all_devicemappings().keys())):
logger.warning("MITMReceiver request without Origin or disallowed Origin: {}".format(origin))
self.response = Response(status=403, headers={})
abort = True
elif self.mapping_manager.get_auths() is not None:
auth = request.headers.get('Authorization', None)
if auth is None or not check_auth(auth, self.application_args,
self.mapping_manager.get_auths()):
logger.warning(
"Unauthorized attempt to POST from {}", str(request.remote_addr))
self.response = Response(status=403, headers={})
abort = True
if not abort:
try:
# TODO: use response data
if len(request.data) > 0:
request_data = json.loads(request.data)
else:
request_data = {}
response_payload = self.action(origin, request_data, *args, **kwargs)
if response_payload is None:
response_payload = ""
if type(response_payload) is Response:
self.response = response_payload
else:
self.response = Response(status=200, headers={"Content-Type": "application/json"})
self.response.data = response_payload
except Exception as e: # TODO: catch exact exception
logger.warning(
"Could not get JSON data from request: {}", str(e))
self.response = Response(status=500, headers={})
return self.response
def update_latest(self,
origin: str,
key: str,
values_dict,
timestamp_received_raw: float = None,
timestamp_received_receiver: float = None,
location: Location = None):
if timestamp_received_raw is None:
timestamp_received_raw = time.time()
if timestamp_received_receiver is None:
timestamp_received_receiver = time.time()
updated = False
logger.debug3(
"Trying to acquire lock and update proto {} received by {}".format(
origin, key))
with self.__mapping_mutex:
if origin not in self.__mapping.keys(
) and origin in self.__mapping_manager.get_all_devicemappings(
).keys():
logger.info(
"New device detected, {}. Setting up the device configuration",
origin)
self.__add_new_device(origin)
if origin in self.__mapping.keys():
logger.debug(
"Updating timestamp of {} at {} with method {} to {}",
str(origin), location, str(key),
str(timestamp_received_raw))
if self.__mapping.get(origin) is not None and self.__mapping[
origin].get(key) is not None:
del self.__mapping[origin][key]
self.__mapping[origin][key] = {}
self.__mapping[origin]["location"] = location
self.__mapping[origin][key][
"timestamp"] = timestamp_received_raw
self.__mapping[origin][
"timestamp_last_data"] = timestamp_received_raw
self.__mapping[origin][
"timestamp_receiver"] = timestamp_received_receiver
self.__mapping[origin][key]["values"] = values_dict
updated = True
else:
logger.warning(
"Not updating timestamp of {} since origin is unknown",
str(origin))
logger.debug3("Done updating proto {} of {}".format(key, origin))
return updated
def __handle_proto_data_dict(self, origin: str, data: dict) -> None:
type = data.get("type", None)
if type is None or type == 0:
logger.warning(
"Could not read method ID. Stopping processing of proto")
return
timestamp: float = data.get("timestamp", int(time.time()))
self.__mitm_mapper.update_latest(
origin, timestamp_received_raw=timestamp, timestamp_received_receiver=time.time(), key=type,
values_dict=data)
logger.debug3("Placing data received by {} to data_queue".format(origin))
self._data_queue.put(
(timestamp, data, origin)
)
def __handle_proto_data_dict(self, origin: str, data: dict) -> None:
type = data.get("type", None)
if type is None or type == 0:
logger.warning(
"Could not read method ID. Stopping processing of proto")
return
timestamp: float = data.get("timestamp", int(time.time()))
location_of_data: Location = Location(data.get("lat", 0.0), data.get("lng", 0.0))
if (location_of_data.lat > 90 or location_of_data.lat < -90
or location_of_data.lng > 180 or location_of_data.lng < -180):
logger.warning("Received invalid location from {} in data: {}".format(origin, str(location_of_data)))
location_of_data: Location = Location(0, 0)
self.__mitm_mapper.update_latest(
origin, timestamp_received_raw=timestamp, timestamp_received_receiver=time.time(), key=type,
values_dict=data, location=location_of_data)
logger.debug3("Placing data received by {} to data_queue".format(origin))
self._data_queue.put(
(timestamp, data, origin)
)
def proto_endpoint(self, origin, data):
logger.debug2("Receiving proto from {}".format(origin))
logger.debug4("Proto data received from {}: {}".format(
origin, str(data)))
type = data.get("type", None)
if type is None or type == 0:
logger.warning(
"Could not read method ID. Stopping processing of proto")
return None
if not self.__mitm_mapper.get_injection_status(origin):
logger.info("Worker {} is injected now", str(origin))
self.__mitm_mapper.set_injection_status(origin)
# extract timestamp from data
timestamp: float = data.get("timestamp", int(time.time()))
self.__mitm_mapper.update_latest(
origin,
timestamp_received_raw=timestamp,
timestamp_received_receiver=time.time(),
key=type,
values_dict=data)
logger.debug3(
"Placing data received by {} to data_queue".format(origin))
self._data_queue.put((timestamp, data, origin))
return None
def __call__(self, *args, **kwargs):
logger.debug3("HTTP Request from {}".format(str(request.remote_addr)))
origin = request.headers.get('Origin')
abort = False
if request.url_rule is not None and str(request.url_rule) == '/status/':
auth = request.headers.get('Authorization', False)
if self.application_args.mitm_status_password != "" and \
(not auth or auth != self.application_args.mitm_status_password):
self.response = Response(status=500, headers={})
abort = True
else:
abort = False
elif request.url is not None and str(request.url_rule) == '/origin_generator':
auth = request.headers.get('Authorization', None)
if auth is None or not check_auth(auth, self.application_args,
self.mapping_manager.get_auths()):
logger.warning(
"Unauthorized attempt to POST from {}", str(request.remote_addr))
self.response = Response(status=403, headers={})
abort = True
else:
if not origin:
logger.warning("Missing Origin header in request")
self.response = Response(status=500, headers={})
abort = True
elif (self.mapping_manager.get_all_devicemappings().keys() is not None
and (origin is None or origin not in self.mapping_manager.get_all_devicemappings().keys())):
logger.warning("MITMReceiver request without Origin or disallowed Origin: {}".format(origin))
self.response = Response(status=403, headers={})
abort = True
elif self.mapping_manager.get_auths() is not None:
auth = request.headers.get('Authorization', None)
if auth is None or not check_auth(auth, self.application_args,
self.mapping_manager.get_auths()):
logger.warning(
"Unauthorized attempt to POST from {}", str(request.remote_addr))
self.response = Response(status=403, headers={})
abort = True
if not abort:
try:
content_encoding = request.headers.get('Content-Encoding', None)
if content_encoding and content_encoding == "gzip":
# we need to unpack the data first
# https://stackoverflow.com/questions/28304515/receiving-gzip-with-flask
compressed_data = io.BytesIO(request.data)
text_data = gzip.GzipFile(fileobj=compressed_data, mode='r')
request_data = json.loads(text_data.read())
else:
request_data = request.data
content_type = request.headers.get('Content-Type', None)
if content_type and content_type == "application/json":
request_data = json.loads(request_data)
else:
request_data = request_data
response_payload = self.action(origin, request_data, *args, **kwargs)
if response_payload is None:
response_payload = ""
if type(response_payload) is Response:
self.response = response_payload
else:
self.response = Response(status=200, headers={"Content-Type": "application/json"})
self.response.data = response_payload
except Exception as e: # TODO: catch exact exception
logger.warning(
"Could not get JSON data from request: {}", str(e))
self.response = Response(status=500, headers={})
import traceback
traceback.print_exc()
return self.response
def __send_webhook(self, payload):
if len(payload) == 0:
logger.debug("Payload empty. Skip sending to webhook.")
return
# get list of urls
webhooks = self.__args.webhook_url.replace(" ", "").split(",")
webhook_count = len(webhooks)
current_wh_num = 1
for webhook in webhooks:
payloadToSend = []
subTypes = "all"
url = webhook.strip()
if url.startswith("["):
endIndex = webhook.rindex("]")
endIndex += 1
subTypes = webhook[:endIndex]
url = url[endIndex:]
for payloadData in payload:
if payloadData["type"] in subTypes:
payloadToSend.append(payloadData)
else:
payloadToSend = payload
if len(payloadToSend) == 0:
logger.debug("Payload empty. Skip sending to: {} (Filter: {})",
url, subTypes)
continue
else:
logger.debug("Sending to webhook url: {} (Filter: {})", url,
subTypes)
payload_list = self.__payload_chunk(
payloadToSend, self.__args.webhook_max_payload_size)
current_pl_num = 1
for payload_chunk in payload_list:
logger.debug4("Python data for payload: {}",
str(payload_chunk))
logger.debug3("Payload: {}", str(json.dumps(payload_chunk)))
try:
response = requests.post(
url,
data=json.dumps(payload_chunk),
headers={"Content-Type": "application/json"},
timeout=5,
)
if response.status_code != 200:
logger.warning(
"Got status code other than 200 OK from webhook destination: {}",
str(response.status_code),
)
else:
if webhook_count > 1:
whcount_text = " [wh {}/{}]".format(
current_wh_num, webhook_count)
else:
whcount_text = ""
if len(payload_list) > 1:
whchunk_text = " [pl {}/{}]".format(
current_pl_num, len(payload_list))
else:
whchunk_text = ""
logger.success(
"Successfully sent payload to webhook{}{}. Stats: {}",
whchunk_text,
whcount_text,
json.dumps(
self.__payload_type_count(payload_chunk)),
)
except Exception as e:
logger.warning(
"Exception occured while sending webhook: {}", str(e))
current_pl_num += 1
current_wh_num += 1