def sanity_check(self):
if not super(HTTP_CONNECT, self).sanity_check():
return False
if not self.config.has_option(self.get_module_configname(), "proxyip"):
common.internal_print(
"'proxyip' option is missing from '{0}' section".format(
self.get_module_configname()), -1)
return False
if not self.config.has_option(self.get_module_configname(),
"proxyport"):
common.internal_print(
"'proxyport' option is missing from '{0}' section".format(
self.get_module_configname()), -1)
return False
if not common.is_ipv4(
self.config.get(self.get_module_configname(), "proxyip")
) and not common.is_ipv6(
self.config.get(self.get_module_configname(), "proxyip")):
common.internal_print(
"'proxyip' should be ipv4 or ipv6 address in '{0}' section".
format(self.get_module_configname()), -1)
return False
return True
def get_intermediate_hop(self, config): if config.has_option(self.get_module_configname(), "proxyip"): if common.is_ipv4(config.get(self.get_module_configname(), "proxyip")) or common.is_ipv6(config.get(self.get_module_configname(), "proxyip")): remoteserverip = config.get(self.get_module_configname(), "proxyip") return remoteserverip return ""
def http_connect_request(self, server_socket):
if self.config.has_option("Global", "remoteserverhost"):
if not common.is_hostname(self.config.get("Global", "remoteserverhost")):
common.internal_print("[Global] remoteserverhost value is not a hostname", -1)
return False
else:
remoteserver = self.config.get("Global", "remoteserverhost")
else:
if self.config.has_option("Global", "remoteserverip"):
if not common.is_ipv4(self.config.get("Global", "remoteserverip")):
common.internal_print("[Global] remoteserverip value is not an IPv4 address", -1)
return False
remoteserver = self.config.get("Global", "remoteserverip")
serverport = int(self.config.get(self.get_module_configname(), "serverport"))
request = "CONNECT %s:%d HTTP/1.1\r\nHost: %s\r\n\r\n" % (remoteserver, serverport, remoteserver)
server_socket.send(request)
response = server_socket.recv(4096)
if response[9:12] != "200":
common.internal_print("Connection failed: {0}".format(response[0:response.find("\n")]), -1)
return False
return True
def __init__(self, name, type, source=None, subtype=None, case_id=None):
self.name = name
self.created = timestamp()
self.type = type
self.subtype = subtype
self.source = source
self.parent = None
self.children = []
self.case_id = case_id
self.tags = []
self.notes = []
self.data = {}
if self.type == 'host':
if is_ipv4(self.name):
self.subtype = 'ipv4'
elif is_ipv6(self.name):
self.subtype = 'ipv6'
elif is_fqdn(self.name):
self.subtype = 'fqdn'
else:
warning('host type cannot be determined. must be one of: ipv4, ipv6, fqdn')
self.subtype = 'unknown'
elif self.type == 'hash':
result = is_hash(self.name)
if result is None:
warning('hash is not a valid md5, sha1, sha256, or sha512')
self.subtype = 'unknown'
else:
self.subtype = result
def sanity_check(self):
if not super(WebSocket, self).sanity_check():
return False
if not self.config.has_option(self.get_module_configname(), "proxyip"):
common.internal_print(
"'proxyip' option is missing from '{0}' section".format(
self.get_module_configname()), -1)
return False
if not self.config.has_option(self.get_module_configname(),
"proxyport"):
common.internal_print(
"'proxyport' option is missing from '{0}' section".format(
self.get_module_configname()), -1)
return False
if not common.is_ipv4(
self.config.get(self.get_module_configname(), "proxyip")
) and not common.is_ipv6(
self.config.get(self.get_module_configname(), "proxyip")):
common.internal_print(
"'proxyip' should be ipv4 or ipv6 address in '{0}' section".
format(self.get_module_configname()), -1)
return False
if not self.config.has_option("Global", "remoteserverhost"):
common.internal_print(
"'remoteserverhost' option is missing from 'Global' section",
-1)
return False
if not common.is_hostname(self.config.get(
"Global", "remoteserverhost")) and not common.is_ipv4(
self.config.get(
"Global", "remoteserverhost")) and not common.is_ipv6(
self.config.get("Global", "remoteserverhost")):
common.internal_print(
"'remoteserverhost' should be a hostname 'Global' section", -1)
return False
return True
def sanity_check(self):
if not super(SOCKS, self).sanity_check():
return False
if not self.config.has_option(self.get_module_configname(), "proxyip"):
common.internal_print(
"'proxyip' option is missing from '{0}' section".format(
self.get_module_configname()), -1)
return False
if not self.config.has_option(self.get_module_configname(),
"proxyport"):
common.internal_print(
"'proxyport' option is missing from '{0}' section".format(
self.get_module_configname()), -1)
return False
if not common.is_ipv4(
self.config.get(self.get_module_configname(), "proxyip")
) and not common.is_ipv6(
self.config.get(self.get_module_configname(), "proxyip")):
common.internal_print(
"'proxyip' should be ipv4 or ipv6 address in '{0}' section".
format(self.get_module_configname()), -1)
return False
if not self.config.has_option(self.get_module_configname(), "version"):
common.internal_print(
"'version' option is missing from '{0}' section".format(
self.get_module_configname()), -1)
return False
version = self.config.get(self.get_module_configname(), "version")
if version == "5":
if not self.config.has_option(self.get_module_configname(),
"usernamev5"):
common.internal_print(
"'usernamev5' option is missing from '{0}' section".format(
self.get_module_configname()), -1)
return False
if not self.config.has_option(self.get_module_configname(),
"passwordv5"):
common.internal_print(
"'passwordv5' option is missing from '{0}' section".format(
self.get_module_configname()), -1)
return False
return True
def freebsd_set_default_route(self, serverip, clientip, ip):
# get default gateway
ps = subprocess.Popen(["route", "-n", "get", "default"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = ps.communicate()
# is there a default gateway entry?
if "route has not been found" in stderr:
common.internal_print(
"No default route. Please set up your routing before executing the tool",
-1)
sys.exit(-1)
self.orig_default_gw = stdout.split("gateway: ")[1].split("\n")[0]
# is it an ipv4 address?
if not common.is_ipv4(self.orig_default_gw):
common.internal_print("Default gateway is not an IPv4 address.",
-1)
sys.exit(-1)
ps = subprocess.Popen([
"route", "add", "-net", serverip, self.orig_default_gw,
"255.255.255.255"
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = ps.communicate()
if stderr:
if not "File exists" in stderr:
common.internal_print(
"Error: adding server route: {0}".format(stderr), -1)
sys.exit(-1)
ps = subprocess.Popen(["route", "delete", "default"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = ps.communicate()
if stderr:
common.internal_print(
"Error: deleting default route: {0}".format(stderr), -1)
sys.exit(-1)
ps = subprocess.Popen(["route", "add", "default", ip],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = ps.communicate()
if stderr:
common.internal_print(
"Error: adding new default route: {0}".format(stderr), -1)
sys.exit(-1)
return
def get_nameserver(self):
if common.get_os_type() == common.OS_LINUX:
f = open("/etc/resolv.conf", "r")
content = f.read()
f.close()
for line in content.split("\n"):
if line.replace(" ", "").replace("\t", "")[0:1] == "#":
continue
if line.find("nameserver") != -1:
break
nameserver = line[line.find("nameserver")+len("nameserver "):len(line)+line.find("nameserver")]
if common.is_ipv4(nameserver) or common.is_ipv6(nameserver):
return nameserver
else:
return None
def __init__(self, name, type, source=None, subtype=None, case_id=None):
self.name = name
self.created = timestamp()
self.type = type
self.subtype = subtype
self.source = source
self.parent = None
self.children = []
self.case_id = case_id
self.tags = []
self.notes = []
self.data = {}
if self.subtype is None:
if self.type == 'host':
if is_ipv4(name):
self.subtype = 'ipv4'
elif is_ipv6(name):
self.subtype = 'ipv6'
elif is_fqdn(name):
self.subtype = 'fqdn'
else:
warning('host subtype is not one of: ipv4, ipv6, fqdn')
elif self.type == 'hash':
hash_type = is_hash(name)
if hash_type is None:
warning('hash is not a valid md5, sha1, sha256, or sha512')
else:
self.subtype = hash_type
elif self.type == 'user':
self.subtype = 'account'
elif self.type == 'email':
self.subtype = 'account'
elif self.type == 'btc':
self.subtype = 'cryptocurrency address'
def mac_set_default_route(self, serverip, clientip, ip):
# https://developer.apple.com/documentation/kernel/rt_msghdr?language=objc
# s = socket(PF_ROUTE, SOCK_RAW, 0)
# not sure which is the better way, calling external tools like
# 'route' or implementing the messaging...
# get default gateway
ps = subprocess.Popen(["route", "-n", "get", "default"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = ps.communicate()
# is there a default gateway entry?
if "not in table" in stderr:
common.internal_print(
"No default route. Please set up your routing before executing the tool",
-1)
sys.exit(-1)
self.orig_default_gw = stdout.split("gateway: ")[1].split("\n")[0]
# is it an ipv4 address?
if not common.is_ipv4(self.orig_default_gw):
common.internal_print("Default gateway is not an IPv4 address.",
-1)
sys.exit(-1)
ps = subprocess.Popen([
"route", "add", "-net", serverip, self.orig_default_gw,
"255.255.255.255"
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = ps.communicate()
if stderr:
if not "File exists" in stderr:
common.internal_print(
"Error: adding server route: {0}".format(stderr), -1)
sys.exit(-1)
ps = subprocess.Popen(["route", "delete", "default"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = ps.communicate()
if stderr:
common.internal_print(
"Error: deleting default route: {0}".format(stderr), -1)
sys.exit(-1)
ps = subprocess.Popen(["route", "add", "default", ip],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = ps.communicate()
if stderr:
common.internal_print(
"Error: adding new default route: {0}".format(stderr), -1)
sys.exit(-1)
'''
# keeping this, in case I can test with tun, not utun
ps = subprocess.Popen(["route", "add", "-net", clientip, serverip, "255.255.255.255"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(stdout, stderr) = ps.communicate()
if stderr:
if not "File exists" in stderr:
common.internal_print("Error: adding new route: {0}".format(stderr), -1)
sys.exit(-1)
'''
return
def perform_semantic_inference(cluster_collection):
"""
This function performs semantic inference on a list of clusters given
For each message in these clusters semantics are inferred by analyzing the token
resp. its context.
At the moment only two semantics are automatically inferred: numeric and IPv4 address
TODO: Add more semantics, e.g. EOL identifier, lenght fields, ...
"""
# Try to perform semantic inferences
# Walk through every cluster and check messages for obvious results
cluster = cluster_collection.get_all_cluster()
for c in cluster:
messages = c.get_messages()
for message in messages:
tokenlist = message.get_tokenlist()
iterator = peekable(tokenlist)
idx = 0
while not iterator.isLast():
#for tokenRepresentation in tokenlist:
tokenRepresentation = iterator.next()
# TODO: do we need to keep semantics which involve multiple cluster? e.g. sessionids?
previous_semantics = tokenRepresentation.get_semantics()
tokenRepresentation.set_semantics(
[]) # Clear existing semantics from previous run
#for s in previous_semantics:
# if s.startswith("sessionid"):
# tokenRepresentation.add_semantic(s)
# break
if "sessionid" in previous_semantics:
# Check if we have at least 2 messages and we are not of type Const
if len(messages) > 1 and c.get_format(
idx) != Message.typeConst:
tokenRepresentation.add_semantic("sessionid")
if "FD" in previous_semantics:
tokenRepresentation.add_semantic("FD")
token = tokenRepresentation.get_token()
# Check whether it is numeric
try:
isNumber = tokenRepresentation.get_tokenType(
) == Message.typeText and common.is_number(token)
except TypeError:
if Globals.getConfig().debug:
print "Error checking token {0} for number semantics".format(
token)
isNumber = False
if isNumber:
tokenRepresentation.add_semantic("numeric")
#c.add_semantics(idx,"numeric")
#print "Inferred semantic inference 'numeric' for token ", token
# Check whether it is an IP address
if isinstance(token, str) and common.is_ipv4(token):
tokenRepresentation.add_semantic("ipv4 address")
# Do not add to cluster unless it is valid for all c.add_semantics(idx,"ipv4 address")
#print "Inferred semantic inference 'ipv4 address' for token ", token
# Check for carriage return identifiers
# When 0d is followed by 0a we've got a CR-LF
# Sensible? When 0d or 0a is the last token, we've got a single CR resp LF
# In all other cases assume 0d/0a is just a hex value of the protocol
if token == 0xd:
nextOne = iterator.peek()
if isinstance(nextOne, TokenRepresentation):
if nextOne.get_token() == 0xa:
inferred_formats = c.get_format_inference()
if inferred_formats[idx].getType(
) == Message.typeConst and inferred_formats[
idx + 1].getType() == Message.typeConst:
tokenRepresentation.add_semantic("CR")
#c.add_semantics(idx,"CR")
nextOne = iterator.next()
nextOne.set_semantics(["LF"])
#c.add_semantics(idx+1, "LF")
idx += 1
idx += 1
# Perform other tests like "is length field?"
# explicitely iterate through all messages like stated in the paper
# we could also postpone this to the call of 'pushToClusterSeminatics" but..
reference_message = messages[0]
tokenlist = reference_message.get_tokenlist()
idx = 0
for tokenRepresentation in tokenlist:
if tokenRepresentation.get_tokenType(
) == Message.typeBinary and idx + 1 < len(tokenlist):
ref_value = tokenRepresentation.get_token()
if not tokenlist[idx + 1].get_tokenType(
) == Message.typeText: # We require that the next token is the text token in question
idx += 1
continue
ref_next_length = tokenlist[idx + 1].get_length()
if not ref_value == ref_next_length: # This is no length field
idx += 1
continue
ref_message_length = reference_message.get_length()
is_length = True
for message in messages:
cmp_value = message.get_tokenlist()[idx].get_token()
cmp_next_length = message.get_tokenlist()[idx +
1].get_length()
cmp_message_length = message.get_length()
try:
diff_val = abs(cmp_value - ref_value)
except TypeError: # Could happen if a short text token is mistaken as a binary value
break
diff_next_length = abs(cmp_next_length - ref_next_length)
# The next line also takes total msg length differences into account. This might not be true for
# all protocols
diff_msg_length = abs(cmp_message_length -
ref_message_length)
if Globals.getConfig(
).requireTotalLengthChangeForLengthField:
if not (diff_val == diff_next_length ==
diff_msg_length):
is_length = False
break
else:
if not (diff_val == diff_next_length):
is_length = False
break
if is_length: # set "lengthfield" semantic for every message in the cluster at the given position
for message in messages: # TODO: What if there's only one message in the cluster? Sensible?
message.get_tokenlist()[idx].add_semantic(
"lengthfield")
c.add_semantic_for_token(idx, "lengthfield")
idx += 1
# Try to identify sessionid fields
reference_message = messages[0]
nextInFlow = reference_message.getNextInFlow()
if nextInFlow != None and not (
len(messages) == 1 and Globals.getConfig(
).sessionIDOnlyWithClustersWithMoreThanOneMessage):
tokenlist = reference_message.get_tokenlist()
next_tokenlist = nextInFlow.get_tokenlist()
ref_idx = 0
for tokenRepresentation in tokenlist:
tokType = tokenRepresentation.get_tokenType()
# If its not a binary, it cannot be a cookie
if tokType != Message.typeBinary:
ref_idx += 1
continue
fmt = c.get_format(ref_idx)
# If its a binary but const, it cannot be a cookie
if fmt[1] == Message.typeConst:
ref_idx += 1
continue
# Set reference value
ref_val = tokenRepresentation.get_token()
# Walk next flow for reference value
next_idx = 0
for next_tokenRepresentation in next_tokenlist:
# Retrieve next token type
nextTokType = next_tokenRepresentation.get_tokenType()
# If it is not a binary we don't see it as a cookie
if Globals.getConfig().sessionIDOnlyWithBinary:
if nextTokType != Message.typeBinary:
next_idx += 1
continue
next_cluster = nextInFlow.getCluster()
# Get format of comparating message
comp_fmt = next_cluster.get_format(next_idx)
# If it is const, it cannot be a sessonid
if comp_fmt[1] == Message.typeConst:
next_idx += 1
continue
# Load comparator value
comp_val = next_tokenRepresentation.get_token()
if ref_val == comp_val: # We've got a potential hit, now compare all messages for the same idx pairs
isCookie = True
for cmp_ref_msg in messages:
if not isCookie:
break
if cmp_ref_msg == messages[
0]: # Skip first message (we've already checked that one
continue
cmp_ref_tok_list = cmp_ref_msg.get_tokenlist()
cmp_ref_val = cmp_ref_tok_list[ref_idx].get_token()
cmp_cmp_msg = cmp_ref_msg.getNextInFlow()
if cmp_cmp_msg == None:
isCookie = False
else:
cmp_cmp_tok_list = cmp_cmp_msg.get_tokenlist()
if next_idx >= len(cmp_cmp_tok_list):
# Obviously "next" points to messages in different clusters
# so the len might differ from the reference next cluster
# used to find our reference cookie value
# Therefore this cannot be a cookie
isCookie = False
continue
# Make sure the comparing token is also not constant
cmp_cmp_fmt = cmp_cmp_msg.getCluster(
).get_format(next_idx)
# If it is const, it cannot be a sessonid
if cmp_cmp_fmt == Message.typeConst:
isCookie = False
continue
# Finally compare the values
cmp_cmp_val = cmp_cmp_tok_list[
next_idx].get_token()
if (cmp_ref_val != cmp_cmp_val) or (
(cmp_ref_val == cmp_cmp_val) and
(cmp_ref_val == ref_val)):
isCookie = False
if isCookie:
# Set cookie semantic in this message and the other
#sessionid = uuid.uuid1()
for message in messages: # Set for every message and the cluster itself
#message.get_tokenlist()[ref_idx].add_semantic("sessionid_{0}".format(sessionid))
message.get_tokenlist()[ref_idx].add_semantic(
"sessionid")
nextMsg = message.getNextInFlow()
#nextMsg.get_tokenlist()[next_idx].add_semantic("sessionid_{0}".format(sessionid))
nextMsg.get_tokenlist()[next_idx].add_semantic(
"sessionid")
c.add_semantic_for_token(ref_idx, "sessionid")
#c.add_semantic_for_token(ref_idx,"sessionid_{0}".format(sessionid))
next_idx += 1
ref_idx += 1
# Try to find random fields (16 bit)
token_formats = c.get_formats()
idx = 0
for token_format in token_formats:
rep, form, semantics = token_format
if form.getType(
) == Message.typeVariable and rep == Message.typeBinary:
try:
variance = c.getVariableStatistics()[idx].getVariance()
except Exception:
pass
if variance > 1000 and len(semantics) == 0:
# We've got a very high variance and no assigned semantics --> candidate for random
# Have a look at the last but one token
if idx - 1 >= 0:
rep, form, semantics = token_formats[idx - 1]
if form.getType(
) == Message.typeVariable and rep == Message.typeBinary:
stats = c.getVariableStatistics()[idx - 1]
if stats != None:
variance2 = stats.getVariance()
else:
logging.error(
"Did not receive cluster statistics for token {0} (len of formats {1}, len of stats {2})"
.format(idx, len(token_formats),
len(c.getVariableStatistics())))
idx += 1
continue
if variance2 > 1000 and len(semantics) == 0:
# Consider the two as a CRC-16
for message in messages: # Set for every message and the cluster itself
message.get_tokenlist()[
idx - 1].add_semantic("random")
message.get_tokenlist()[idx].add_semantic(
"random")
c.add_semantic_for_token(idx - 1, "random")
c.add_semantic_for_token(idx, "random")
idx += 1
# Try to find sets (valued limited in variability with lower and upper bound)
token_formats = c.get_formats()
idx = 0
for token_format in token_formats:
rep, form, semantics = token_format
if form.getType() == Message.typeVariable:
stats = c.getVariableStatistics()[idx]
if stats != None:
distinct = stats.numberOfDistinctSamples()
else:
logging.error(
"Did not receive cluster statistics for token {0} (len of formats {1}, len of stats {2})"
.format(idx, len(token_formats),
len(c.getVariableStatistics())))
idx += 1
continue
# How will be find out whether a number of variable values is a set or really variable?
# We assume that there is an absolute maximum amount of distinct values which is independent
# of the actual number of messages. However we also need to consider that when the number of
# messages in a cluster definitily falls below the setAbsoluteMax value, we have to adapt
# the local maximum in this cluster.
# For the moment we take a multiplier for the number of messages (default 0.3 == 30%) and
# assume it is a set, when both, setAbsoluteMax and the localThreshold is underrun
# In addition we assume that we have no semantics for this token, as other semantics conflict
# with the notion of a set
if (distinct <= Globals.getConfig().setAbsoluteMax
and distinct <=
(c.getNumberOfMessages() *
Globals.getConfig().setPercentageThreshold)
and len(semantics) == 0):
for message in messages: # Set for every message and the cluster itself
message.get_tokenlist()[idx].add_semantic("set")
c.add_semantic_for_token(idx - 1, "set")
idx += 1
# Push to cluster
pushUpToCluster(cluster_collection)
def run(self, argv):
self.banner()
try:
opts, args = getopt.getopt(argv, self.short, self.long)
except getopt.GetoptError:
self.usage()
sys.exit(-1)
for opt, arg in opts:
if opt in ("-h", "--help"):
self.usage()
sys.exit(0)
elif opt in ("-s", "--server"):
self.servermode = 1
elif opt in ("-c", "--client"):
self.clientmode = 1
elif opt in ("--check"):
self.checkmode = 1
elif opt in ("--config"):
self.configfile = arg
elif opt in ("--split"):
self.splitmode = 1
elif opt in ("--ignore-dependencies"):
common.internal_print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", -1)
common.internal_print("!IGNORING MISSING DEPENDENCIES, THIS COULD RESULT IN UNHANDLED EXCEPTIONS!", -1)
common.internal_print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", -1)
self.ignoredependencies = 1
elif opt in ("--verbose"):
try:
self.verbosity = int(arg)
except:
common.internal_print("Invalid verbose value: {0}".format(arg), -1)
sys.exit(-1)
if not common.get_privilege_level():
common.internal_print("The tool needs superuser or admin privileges. Run as root or Administrator.", -1)
sys.exit(-1)
# set the servermode when it was not specified expicitly
if (not self.clientmode and not self.checkmode):
self.servermode = 1
# checking for the config file
if not os.path.isfile(self.configfile):
common.internal_print("config file missing!", -1)
sys.exit(-1)
# Looking for and parsing config file.
common.internal_print("Parsing config file")
config = ConfigParser.ConfigParser()
try:
config.read(self.configfile)
except:
common.internal_print("Invalid or malformed configuration file specified", -1)
sys.exit(-1)
# sanity check on configuration, exit on error
if not common.config_sanity_check(config, (self.servermode)):
sys.exit(-1)
# load authentication module
auth_module = self.authentication.load_auth_module(config)
# sanity check for the auth module
if not auth_module.sanity_check(config):
sys.exit(-1)
# initialize authentication
if not auth_module.init(config, self.servermode):
sys.exit(-1)
# load encryption module
encryption_module = self.encryption.load_encryption_module(config)
# sanity check for the encryption module
if not encryption_module.sanity_check(config):
sys.exit(-1)
# initialize encryption
if not encryption_module.init(config, self.servermode):
sys.exit(-1)
if self.splitmode:
self.scope = common.parse_scope_file(config.get("Global", "scope"))
if self.scope == []:
common.internal_print("Split tunnelling mode enabled, but no scope was defined or entries were invalid", -1)
sys.exit(-1)
# Check system config for routing. If the OS was not set up for IP
# forwarding then we need to exit.
if self.servermode:
if not common.check_router_settings(config):
sys.exit(-1)
# Loading modules from modules/ directory
# 1. listing and loading all modules except forbidden ones
common.internal_print("Loading all modules from 'modules' directory")
for module in os.listdir("./modules/"):
if module == '__init__.py' or module[-3:] != '.py' or (module[:-3] in self.forbidden_modules):
continue
module_list = __import__("modules."+module[:-3], locals(), globals())
modules = []
for m in dir(module_list):
# 2. going thru all the modules
if m != 'Generic_module' and m[:2] != "__":
module_attributes = getattr(module_list, m)
# 3. get the classes from the modules
module_classes = [c for c in module_attributes.__dict__.values() if inspect.isclass(c)]
# 4. select classes that are XFLTReaT modules
real_modules = [c for c in module_classes if (issubclass(c, Generic_module) and (c not in self.forbidden_modules_instance))]
for r in real_modules:
# 5. actual instantiation of a module
try:
modules.append(r())
except Exception as e:
if self.ignoredependencies:
common.internal_print("This module cannot be used without the dependencies installed: {0}".format(m), -1)
pass
else:
raise
# if the module is enabled from config, we store it in modules_enabled[]
modules_enabled = []
for m in modules:
enabled = "no"
# is there any section in the config for the module?
if not config.has_section(m.get_module_configname()):
common.internal_print("No section in config for module: {0}".format(m.get_module_configname()), -1)
continue
# is the 'enabled' option there for the module?
if not config.has_option(m.get_module_configname(), "enabled"):
common.internal_print("No option 'enabled' in config for module: {0}".format(m.get_module_configname()), -1)
sys.exit(-1)
enabled = config.get(m.get_module_configname(), "enabled")
if enabled == "yes":
# looks like the module is enabled, adding to modules_enabled[]
modules_enabled.append(m)
# check if more than one module is enabled for client mode
if self.clientmode and (len(modules_enabled)>1):
common.internal_print("In client mode only one module can be used.", -1)
sys.exit(-1)
if not len(modules_enabled):
common.internal_print("No modules were enabled in configuration", -1)
sys.exit(-1)
# One Interface to rule them all, One Interface to find them,
# One Interface to bring them all and in the darkness bind them
common.internal_print("Setting up interface")
interface = Interface()
# Setting up interface related things for server mode
if self.servermode:
server_tunnel = interface.tun_alloc(config.get("Global", "serverif"), interface.IFF_TUN|interface.IFF_NO_PI)
interface.set_ip_address(config.get("Global", "serverif"),
config.get("Global", "serverip"), config.get("Global", "serverip"), config.get("Global", "servernetmask"))
interface.set_mtu(config.get("Global", "serverif"), int(config.get("Global", "mtu")))
# start thread with socket-interface related pipes
ps = PacketSelector(server_tunnel)
ps.start()
# Setting up interface related things for client mode
if self.clientmode:
client_tunnel = interface.tun_alloc(config.get("Global", "clientif"), interface.IFF_TUN|interface.IFF_NO_PI)
interface.set_ip_address(config.get("Global", "clientif"),
config.get("Global", "clientip"), config.get("Global", "serverip"), config.get("Global", "clientnetmask"))
if not self.splitmode:
interface.set_default_route(config.get("Global", "remoteserverip"), config.get("Global", "clientip"), config.get("Global", "serverip"))
else:
interface.set_split_route(self.scope, config.get("Global", "serverip"))
interface.set_mtu(config.get("Global", "clientif"), int(config.get("Global", "mtu")))
common.internal_print("Please use CTRL+C to exit...")
module_threads = []
module_thread_num = 0
for m in modules_enabled:
# Executing module in server mode
if self.servermode:
module_thread_num = module_thread_num + 1
if m.__init_thread__(module_thread_num, config, server_tunnel, ps, auth_module, encryption_module, self.verbosity):
m.start()
module_threads.append(m)
# Executing module in check mode
if self.checkmode:
interface.check_default_route()
if m.__init_thread__(0, config, None, None, None, None, self.verbosity):
try:
m.check()
except KeyboardInterrupt:
pass
# Executing module in client mode
if self.clientmode:
try:
remoteserverip = config.get("Global", "remoteserverip")
if not config.has_section(m.get_module_configname()):
common.internal_print("No section in config for module: {0}".format(m.get_module_configname()), -1)
continue
# if the module requires an indirect connection (proxy,
# dns) then we need to amend the routing table
intermediate_hop = m.get_intermediate_hop(config)
if config.has_option("Global", "overriderouter"):
if common.is_ipv4(config.get("Global", "overriderouter")):
intermediate_hop = config.get("Global", "overriderouter")
if intermediate_hop and (not self.splitmode):
remoteserverip = intermediate_hop
interface.set_intermediate_route(config.get("Global", "remoteserverip"), remoteserverip)
# init "thread" for client mode, this will not be run in a thread.
if m.__init_thread__(0, config, client_tunnel, None, auth_module, encryption_module, self.verbosity):
# run in client mode
m.connect()
# client finished, closing down tunnel and restoring routes
interface.close_tunnel(client_tunnel)
if self.splitmode:
interface.del_split_route(self.scope, config.get("Global", "serverip"))
else:
interface.restore_routes(remoteserverip, config.get("Global", "clientip"), config.get("Global", "serverip"))
except KeyboardInterrupt:
# CTRL+C was pushed
interface.close_tunnel(client_tunnel)
if self.splitmode:
interface.del_split_route(self.scope, config.get("Global", "serverip"))
else:
interface.restore_routes(remoteserverip, config.get("Global", "clientip"), config.get("Global", "serverip"))
pass
except socket.error as e:
# socket related error
interface.close_tunnel(client_tunnel)
if self.splitmode:
interface.del_split_route(self.scope, config.get("Global", "serverip"))
else:
interface.restore_routes(remoteserverip, config.get("Global", "clientip"), config.get("Global", "serverip"))
if e.errno == errno.ECONNREFUSED:
common.internal_print("Socket does not seem to answer.", -1)
else:
common.internal_print("Socket died, probably the server went down. ({0})".format(e.errno), -1)
except:
interface.close_tunnel(client_tunnel)
if self.splitmode:
interface.del_split_route(self.scope, config.get("Global", "serverip"))
else:
interface.restore_routes(remoteserverip, config.get("Global", "clientip"), config.get("Global", "serverip"))
raise
# No modules are running
if not module_threads:
common.internal_print("Exiting...")
if self.servermode:
ps.stop()
else:
try:
time.sleep(0.5)
common.internal_print("Please use CTRL+C to exit...")
# found no better solution to keep the main thread and catch CTRL+C
# if you know any, you know how to tell me ;)
while True:
time.sleep(1000)
except KeyboardInterrupt:
common.internal_print("Interrupted. Exiting...")
ps.stop()
for t in module_threads:
t.stop()
if self.servermode:
time.sleep(1.0)
interface.close_tunnel(server_tunnel)
# give one sec to clean up for modules, otherwise some objects just
# disapper and cannot be closed properly like the tunnel interface
try:
time.sleep(1.0)
except KeyboardInterrupt:
common.internal_print("Are you really this impatient????", -1)
pass
def socks_handshake(self, server_socket):
version = self.config.get(self.get_module_configname(), "version")
if (version == "4") or (version == "4a"):
if self.config.has_option(self.get_module_configname(), "userid"):
userid = self.config.get(self.get_module_configname(),
"userid") + "\x00"
else:
userid = "\x00"
if version == "4":
server_socket.send(
struct.pack(
">BBH4s", self.socks.SOCKS4_VERSION,
self.socks.SOCKS4_CD,
int(
self.config.get(self.get_module_configname(),
"serverport")),
socket.inet_aton(
self.config.get("Global", "remoteserverip"))) +
userid)
else:
if self.config.has_option("Global", "remoteserverhost"):
if not common.is_hostname(
self.config.get("Global", "remoteserverhost")):
common.internal_print(
"[Global] remoteserverhost value is not a hostname",
-1)
return False
domain = self.config.get("Global",
"remoteserverhost") + "\x00"
server_socket.send(
struct.pack(
">BBH4s", self.socks.SOCKS4_VERSION,
self.socks.SOCKS4_CD,
int(
self.config.get(self.get_module_configname(),
"serverport")),
socket.inet_aton("0.0.0.1")) + userid + domain)
else:
common.internal_print(
"Missing remoteserverhost attribute from config [Global] section",
-1)
return False
response = server_socket.recv(8)
if (response[0:1] == self.socks.SOCKS4_OK) and (
response[1:2] == self.socks.SOCKS4_RESPONSES[0]):
common.internal_print(
"Connection was made through the proxy server", 1)
else:
if len(response) > 1:
if (response[1:2] in self.socks.SOCKS4_RESPONSES):
for i in range(len(self.socks.SOCKS4_RESPONSES)):
if response[1:2] == self.socks.SOCKS4_RESPONSES[i]:
common.internal_print(
"Connection failed through the proxy server: {0}"
.format(
self.socks.SOCKS4_RESPONSES_STR[i]),
-1)
return False
common.internal_print(
"Connection failed through the proxy server: Unknown error",
-1)
return False
if version == "5":
# send greeting with auth method list
greeting = struct.pack(">BB", self.socks.SOCKS5_VERSION,
len(self.socks.SOCKS5_AUTH_METHODS))
for i in range(len(self.socks.SOCKS5_AUTH_METHODS)):
greeting += self.socks.SOCKS5_AUTH_METHODS[i][0]
server_socket.send(greeting)
# receive response with selected auth method
response = server_socket.recv(2)
if (len(response) != 2) or (response[0:1] != chr(
self.socks.SOCKS5_VERSION)):
common.internal_print(
"Connection failed through the proxy server: Unknown error",
-1)
return False
if response[1:2] == self.socks.SOCKS5_REJECT_METHODS:
common.internal_print(
"Connection failed through the proxy server: Authentication methods rejected",
-1)
return False
for i in range(len(self.socks.SOCKS5_AUTH_METHODS)):
if response[1:2] == self.socks.SOCKS5_AUTH_METHODS[i][0]:
if self.socks.SOCKS5_AUTH_METHODS[i][1](self.config,
server_socket):
if self.config.has_option("Global",
"remoteserverhost"):
remoteserverhost = self.config.get(
"Global", "remoteserverhost")
if not (common.is_hostname(remoteserverhost)
or common.is_ipv4(remoteserverhost)
or common.is_ipv6(remoteserverhost)):
common.internal_print(
"[Global] remoteserverhost value is not ipv4, ipv6 or a hostname",
-1)
return False
else:
common.internal_print(
"Missing remoteserverhost attribute from config [Global] section",
-1)
return False
if common.is_ipv4(remoteserverhost):
host_type = self.socks.SOCKS5_ADDR_TYPE[0]
connect_string = struct.pack(
">BBBB4sH", self.socks.SOCKS5_VERSION,
self.socks.SOCKS5_CD, 0, host_type,
socket.inet_aton(remoteserverhost),
int(
self.config.get(
self.get_module_configname(),
"serverport")))
if common.is_hostname(remoteserverhost):
host_type = self.socks.SOCKS5_ADDR_TYPE[1]
connect_string = struct.pack(
">BBBBB", self.socks.SOCKS5_VERSION,
self.socks.SOCKS5_CD, 0, host_type,
len(remoteserverhost))
connect_string += remoteserverhost + struct.pack(
">H",
int(
self.config.get(
self.get_module_configname(),
"serverport")))
if common.is_ipv6(remoteserverhost):
host_type = self.socks.SOCKS5_ADDR_TYPE[2]
connect_string = struct.pack(
">BBBB16sH", self.socks.SOCKS5_VERSION,
self.socks.SOCKS5_CD, 0, host_type,
socket.inet_pton(socket.AF_INET6,
remoteserverhost),
int(
self.config.get(
self.get_module_configname(),
"serverport")))
server_socket.send(connect_string)
response = server_socket.recv(4096)
if (len(response) < 4) or (response[0:1] != chr(
self.socks.SOCKS5_VERSION)):
common.internal_print(
"Connection failed through the proxy server: Connection error to the server",
-1)
return False
if response[1:2] == self.socks.SOCKS5_RESPONSES[0]:
return True
if response[1:2] not in self.socks.SOCKS5_RESPONSES:
common.internal_print(
"Connection failed through the proxy server: Unknown error code",
-1)
return False
for i in range(len(self.socks.SOCKS5_RESPONSES)):
if response[1:2] == self.socks.SOCKS5_RESPONSES[i]:
common.internal_print(
"Connection failed through the proxy server: {0}"
.format(
self.socks.SOCKS5_RESPONSES_STR[i]),
-1)
return False
return False
else:
return False
common.internal_print(
"Connection failed through the proxy server: Strange response (Authentication method)",
-1)
return False
return True
def get_ssr_conf(ssr_conf_path: str) -> Dict:
_ssr_conf: Dict = utils.parse_json(ssr_conf_path)
if not _ssr_conf:
exit.error('Require ssr-config.')
# -- check params --
port = _ssr_conf.get('server_port')
if port is None:
exit.error('Require \'server_port\'.')
if type(port) != int or port <= 0:
exit.error('Illegal \'server_port\'.')
password = _ssr_conf.get('password')
if common.is_blank(password):
exit.error('Require \'password\'.')
method = _ssr_conf.get('method')
if common.is_blank(method):
exit.error('Require \'method\'.')
if not encrypt.is_supported(method):
exit.error(f'Not supported method [{method}]')
protocol = _ssr_conf.get('protocol')
if common.is_blank(protocol):
exit.error('Require \'protocol\'.')
obfs = _ssr_conf.get('obfs')
if common.is_blank(obfs):
exit.error('Require \'obfs\'.')
dns_list = _ssr_conf.get('dns')
if dns_list is None or type(dns_list) != list or len(dns_list) == 0:
exit.error('Require \'dns server\'.')
for dns in dns_list:
host = dns.get('host')
if ':' in host:
exit.error("Not support ipv6 dns host.")
if not common.is_ipv4(host):
exit.error("Illegal ipv4 dns host.")
# -- default params --
_ssr_conf['server'] = '::'
_ssr_conf['password'] = common.to_bytes(_ssr_conf['password'])
_ssr_conf['protocol_param'] = _ssr_conf.get('protocol_param', '')
_ssr_conf['obfs_param'] = _ssr_conf.get('obfs_param', '')
_ssr_conf['udp_enable'] = _ssr_conf.get('udp_enable', False)
# process default data
try:
_ssr_conf['forbidden_ip'] = \
common.IPNetwork(_ssr_conf.get('forbidden_ip', '127.0.0.0/8,::1/128'))
except Exception as e:
exit.error('error configuration \'forbidden_ip\'.')
try:
_ssr_conf['forbidden_port'] = common.PortRange(
_ssr_conf.get('forbidden_port', ''))
except Exception as e:
exit.error('error configuration \'forbidden_port\'.')
try:
_ssr_conf['ignore_bind'] = \
common.IPNetwork(_ssr_conf.get('ignore_bind', '127.0.0.0/8,::1/128,10.0.0.0/8,192.168.0.0/16'))
except Exception as e:
exit.error('error configuration \'ignore_bind\'.')
return _ssr_conf
def __init__(self, argv):
self.verbosity = 0
self.configfile = "xfltreat.conf"
self.servermode = 0
self.clientmode = 0
self.checkmode = 0
self.short = "hsc"
self.long = ["help", "server", "client", "check", "config=", "verbose="]
self.forbidden_modules = ["Generic_module", "Stateful_module", "Stateless_module"]
self.forbidden_modules_instance = [Generic_module, Stateful_module, Stateless_module]
self.banner()
try:
opts, args = getopt.getopt(argv, self.short, self.long)
except getopt.GetoptError:
self.usage()
exit(-1)
for opt, arg in opts:
if opt in ("-h", "--help"):
self.usage()
exit(0)
elif opt in ("-s", "--server"):
self.servermode = 1
elif opt in ("-c", "--client"):
self.clientmode = 1
elif opt in ("--check"):
self.checkmode = 1
elif opt in ("--config"):
self.configfile = arg
elif opt in ("--verbose"):
try:
self.verbosity = int(arg)
except:
common.internal_print("Invalid verbose value: {0}".format(arg), -1)
exit(-1)
if (not self.clientmode and not self.checkmode):
self.servermode = 1
# checking for the config file
if not os.path.isfile(self.configfile):
common.internal_print("config file missing!", -1)
exit(-1)
# Looking for and parsing config file.
common.internal_print("Parsing config file")
config = ConfigParser.ConfigParser()
try:
config.read(self.configfile)
except:
common.internal_print("Invalid or malformed configuration file specified", -1)
exit(-1)
if not common.config_sanity_check(config, (self.servermode or (not self.clientmode and not self.checkmode))):
exit(-1)
# Loading modules from modules/ directory
# 1. listing and loading all modules except forbidden ones
common.internal_print("Loading all modules from 'modules' directory")
for module in os.listdir("./modules/"):
if module == '__init__.py' or module[-3:] != '.py' or (module[:-3] in self.forbidden_modules):
continue
module_list = __import__("modules."+module[:-3], locals(), globals())
modules = []
for m in dir(module_list):
# 2. going thru all the modules
if m != 'Generic_module' and m[:2] != "__":
module_attributes = getattr(module_list, m)
# 3. get the classes from the modules
module_classes = [c for c in module_attributes.__dict__.values() if inspect.isclass(c)]
# 4. select classes that are XFLTReaT modules
real_modules = [c for c in module_classes if (issubclass(c, Generic_module) and (c not in self.forbidden_modules_instance))]
for r in real_modules:
# 5. actual instantiation of a module
modules.append(r())
# if the module is enabled from config, we make store it in modules_enabled[]
modules_enabled = []
for m in modules:
enabled = "no"
if not config.has_section(m.get_module_configname()):
common.internal_print("No section in config for module: {0}".format(m.get_module_configname()), -1)
continue
enabled = config.get(m.get_module_configname(), "enabled")
if enabled == "yes":
# looks like the module is enabled, adding to modules|_enabled[]
modules_enabled.append(m)
# check if more than one module is enabled for client mode
if self.clientmode and (len(modules_enabled)>1):
common.internal_print("In client mode only one module can be used.", -1)
exit(-1)
# One Interface to rule them all, One Interface to find them,
# One Interface to bring them all and in the darkness bind them
common.internal_print("Setting up interface")
interface = Interface()
# Operating in server mode
if self.servermode:
server_tunnel = interface.tun_alloc(config.get("Global", "serverif"), interface.IFF_TUN|interface.IFF_NO_PI)
interface.set_ip_address(config.get("Global", "serverif"),
config.get("Global", "serverip"), config.get("Global", "servernetmask"))
interface.set_mtu(config.get("Global", "serverif"), int(config.get("Global", "mtu")))
# start thread with socket-interface related pipes
ps = PacketSelector(server_tunnel)
ps.start()
# Operating in client mode
if self.clientmode:
client_tunnel = interface.tun_alloc(config.get("Global", "clientif"), interface.IFF_TUN|interface.IFF_NO_PI)
interface.set_ip_address(config.get("Global", "clientif"),
config.get("Global", "clientip"), config.get("Global", "clientnetmask"))
interface.set_default_route(config.get("Global", "remoteserverip"), config.get("Global", "serverip"))
interface.set_mtu(config.get("Global", "clientif"), int(config.get("Global", "mtu")))
module_threads = []
module_thread_num = 0
for m in modules_enabled:
if (self.servermode or (not self.clientmode and not self.checkmode)):
module_thread_num = module_thread_num + 1
m.__init_thread__(module_thread_num, config, server_tunnel, ps, self.verbosity)
m.start()
module_threads.append(m)
if self.clientmode:
try:
remoteserverip = config.get("Global", "remoteserverip")
if not config.has_section(m.get_module_configname()):
common.internal_print("No section in config for module: {0}".format(m.get_module_configname()), -1)
continue
if config.has_option(m.get_module_configname(), "proxyip"):
if common.is_ipv4(config.get(m.get_module_configname(), "proxyip")) or common.is_ipv6(config.get(m.get_module_configname(), "proxyip")):
remoteserverip = config.get(m.get_module_configname(), "proxyip")
m.__init_thread__(0, config, client_tunnel, None, self.verbosity)
if config.has_option(m.get_module_configname(), "proxyip"):
interface.set_proxy_route(config.get("Global", "remoteserverip"), remoteserverip)
m.client()
interface.close_tunnel(client_tunnel)
interface.restore_routes(remoteserverip)
except KeyboardInterrupt:
interface.close_tunnel(client_tunnel)
interface.restore_routes(remoteserverip)
pass
except socket.error as e:
interface.close_tunnel(client_tunnel)
interface.restore_routes(remoteserverip)
if e.errno == errno.ECONNREFUSED:
common.internal_print("Socket does not seem to answer.", -1)
else:
common.internal_print("Socket died, probably the server went down.", -1)
except:
interface.close_tunnel(client_tunnel)
interface.restore_routes(remoteserverip)
raise
if self.checkmode:
m.__init_thread__(0, config, None, None, self.verbosity)
try:
m.check()
except KeyboardInterrupt:
pass
#
if not module_threads:
common.internal_print("Exiting...")
if (self.servermode or (not self.clientmode and not self.checkmode)):
ps.stop()
else:
try:
common.internal_print("Please use CTRL+C to exit...")
while True:
time.sleep(1000)
except KeyboardInterrupt:
common.internal_print("Interrupted. Exiting...")
ps.stop()
for t in module_threads:
t.stop()
def perform_semantic_inference(cluster_collection):
"""
This function performs semantic inference on a list of clusters given
For each message in these clusters semantics are inferred by analyzing the token
resp. its context.
At the moment only two semantics are automatically inferred: numeric and IPv4 address
TODO: Add more semantics, e.g. EOL identifier, lenght fields, ...
"""
# Try to perform semantic inferences
# Walk through every cluster and check messages for obvious results
cluster = cluster_collection.get_all_cluster()
for c in cluster:
messages = c.get_messages()
for message in messages:
tokenlist = message.get_tokenlist()
iterator = peekable(tokenlist)
idx = 0
while not iterator.isLast():
# for tokenRepresentation in tokenlist:
tokenRepresentation = iterator.next()
# TODO: do we need to keep semantics which involve multiple cluster? e.g. sessionids?
previous_semantics = tokenRepresentation.get_semantics()
tokenRepresentation.set_semantics([]) # Clear existing semantics from previous run
# for s in previous_semantics:
# if s.startswith("sessionid"):
# tokenRepresentation.add_semantic(s)
# break
if "sessionid" in previous_semantics:
# Check if we have at least 2 messages and we are not of type Const
if len(messages) > 1 and c.get_format(idx) != Message.typeConst:
tokenRepresentation.add_semantic("sessionid")
if "FD" in previous_semantics:
tokenRepresentation.add_semantic("FD")
token = tokenRepresentation.get_token()
# Check whether it is numeric
try:
isNumber = tokenRepresentation.get_tokenType() == Message.typeText and common.is_number(token)
except TypeError:
if Globals.getConfig().debug:
print "Error checking token {0} for number semantics".format(token)
isNumber = False
if isNumber:
tokenRepresentation.add_semantic("numeric")
# c.add_semantics(idx,"numeric")
# print "Inferred semantic inference 'numeric' for token ", token
# Check whether it is an IP address
if isinstance(token, str) and common.is_ipv4(token):
tokenRepresentation.add_semantic("ipv4 address")
# Do not add to cluster unless it is valid for all c.add_semantics(idx,"ipv4 address")
# print "Inferred semantic inference 'ipv4 address' for token ", token
# Check for carriage return identifiers
# When 0d is followed by 0a we've got a CR-LF
# Sensible? When 0d or 0a is the last token, we've got a single CR resp LF
# In all other cases assume 0d/0a is just a hex value of the protocol
if token == 0xD:
nextOne = iterator.peek()
if isinstance(nextOne, TokenRepresentation):
if nextOne.get_token() == 0xA:
inferred_formats = c.get_format_inference()
if (
inferred_formats[idx].getType() == Message.typeConst
and inferred_formats[idx + 1].getType() == Message.typeConst
):
tokenRepresentation.add_semantic("CR")
# c.add_semantics(idx,"CR")
nextOne = iterator.next()
nextOne.set_semantics(["LF"])
# c.add_semantics(idx+1, "LF")
idx += 1
idx += 1
# Perform other tests like "is length field?"
# explicitely iterate through all messages like stated in the paper
# we could also postpone this to the call of 'pushToClusterSeminatics" but..
reference_message = messages[0]
tokenlist = reference_message.get_tokenlist()
idx = 0
for tokenRepresentation in tokenlist:
if tokenRepresentation.get_tokenType() == Message.typeBinary and idx + 1 < len(tokenlist):
ref_value = tokenRepresentation.get_token()
if (
not tokenlist[idx + 1].get_tokenType() == Message.typeText
): # We require that the next token is the text token in question
idx += 1
continue
ref_next_length = tokenlist[idx + 1].get_length()
if not ref_value == ref_next_length: # This is no length field
idx += 1
continue
ref_message_length = reference_message.get_length()
is_length = True
for message in messages:
cmp_value = message.get_tokenlist()[idx].get_token()
cmp_next_length = message.get_tokenlist()[idx + 1].get_length()
cmp_message_length = message.get_length()
try:
diff_val = abs(cmp_value - ref_value)
except TypeError: # Could happen if a short text token is mistaken as a binary value
break
diff_next_length = abs(cmp_next_length - ref_next_length)
# The next line also takes total msg length differences into account. This might not be true for
# all protocols
diff_msg_length = abs(cmp_message_length - ref_message_length)
if Globals.getConfig().requireTotalLengthChangeForLengthField:
if not (diff_val == diff_next_length == diff_msg_length):
is_length = False
break
else:
if not (diff_val == diff_next_length):
is_length = False
break
if is_length: # set "lengthfield" semantic for every message in the cluster at the given position
for message in messages: # TODO: What if there's only one message in the cluster? Sensible?
message.get_tokenlist()[idx].add_semantic("lengthfield")
c.add_semantic_for_token(idx, "lengthfield")
idx += 1
# Try to identify sessionid fields
reference_message = messages[0]
nextInFlow = reference_message.getNextInFlow()
if nextInFlow != None and not (
len(messages) == 1 and Globals.getConfig().sessionIDOnlyWithClustersWithMoreThanOneMessage
):
tokenlist = reference_message.get_tokenlist()
next_tokenlist = nextInFlow.get_tokenlist()
ref_idx = 0
for tokenRepresentation in tokenlist:
tokType = tokenRepresentation.get_tokenType()
# If its not a binary, it cannot be a cookie
if tokType != Message.typeBinary:
ref_idx += 1
continue
fmt = c.get_format(ref_idx)
# If its a binary but const, it cannot be a cookie
if fmt[1] == Message.typeConst:
ref_idx += 1
continue
# Set reference value
ref_val = tokenRepresentation.get_token()
# Walk next flow for reference value
next_idx = 0
for next_tokenRepresentation in next_tokenlist:
# Retrieve next token type
nextTokType = next_tokenRepresentation.get_tokenType()
# If it is not a binary we don't see it as a cookie
if Globals.getConfig().sessionIDOnlyWithBinary:
if nextTokType != Message.typeBinary:
next_idx += 1
continue
next_cluster = nextInFlow.getCluster()
# Get format of comparating message
comp_fmt = next_cluster.get_format(next_idx)
# If it is const, it cannot be a sessonid
if comp_fmt[1] == Message.typeConst:
next_idx += 1
continue
# Load comparator value
comp_val = next_tokenRepresentation.get_token()
if (
ref_val == comp_val
): # We've got a potential hit, now compare all messages for the same idx pairs
isCookie = True
for cmp_ref_msg in messages:
if not isCookie:
break
if cmp_ref_msg == messages[0]: # Skip first message (we've already checked that one
continue
cmp_ref_tok_list = cmp_ref_msg.get_tokenlist()
cmp_ref_val = cmp_ref_tok_list[ref_idx].get_token()
cmp_cmp_msg = cmp_ref_msg.getNextInFlow()
if cmp_cmp_msg == None:
isCookie = False
else:
cmp_cmp_tok_list = cmp_cmp_msg.get_tokenlist()
if next_idx >= len(cmp_cmp_tok_list):
# Obviously "next" points to messages in different clusters
# so the len might differ from the reference next cluster
# used to find our reference cookie value
# Therefore this cannot be a cookie
isCookie = False
continue
# Make sure the comparing token is also not constant
cmp_cmp_fmt = cmp_cmp_msg.getCluster().get_format(next_idx)
# If it is const, it cannot be a sessonid
if cmp_cmp_fmt == Message.typeConst:
isCookie = False
continue
# Finally compare the values
cmp_cmp_val = cmp_cmp_tok_list[next_idx].get_token()
if (cmp_ref_val != cmp_cmp_val) or (
(cmp_ref_val == cmp_cmp_val) and (cmp_ref_val == ref_val)
):
isCookie = False
if isCookie:
# Set cookie semantic in this message and the other
# sessionid = uuid.uuid1()
for message in messages: # Set for every message and the cluster itself
# message.get_tokenlist()[ref_idx].add_semantic("sessionid_{0}".format(sessionid))
message.get_tokenlist()[ref_idx].add_semantic("sessionid")
nextMsg = message.getNextInFlow()
# nextMsg.get_tokenlist()[next_idx].add_semantic("sessionid_{0}".format(sessionid))
nextMsg.get_tokenlist()[next_idx].add_semantic("sessionid")
c.add_semantic_for_token(ref_idx, "sessionid")
# c.add_semantic_for_token(ref_idx,"sessionid_{0}".format(sessionid))
next_idx += 1
ref_idx += 1
# Try to find random fields (16 bit)
token_formats = c.get_formats()
idx = 0
for token_format in token_formats:
rep, form, semantics = token_format
if form.getType() == Message.typeVariable and rep == Message.typeBinary:
try:
variance = c.getVariableStatistics()[idx].getVariance()
except Exception:
pass
if variance > 1000 and len(semantics) == 0:
# We've got a very high variance and no assigned semantics --> candidate for random
# Have a look at the last but one token
if idx - 1 >= 0:
rep, form, semantics = token_formats[idx - 1]
if form.getType() == Message.typeVariable and rep == Message.typeBinary:
stats = c.getVariableStatistics()[idx - 1]
if stats != None:
variance2 = stats.getVariance()
else:
logging.error(
"Did not receive cluster statistics for token {0} (len of formats {1}, len of stats {2})".format(
idx, len(token_formats), len(c.getVariableStatistics())
)
)
idx += 1
continue
if variance2 > 1000 and len(semantics) == 0:
# Consider the two as a CRC-16
for message in messages: # Set for every message and the cluster itself
message.get_tokenlist()[idx - 1].add_semantic("random")
message.get_tokenlist()[idx].add_semantic("random")
c.add_semantic_for_token(idx - 1, "random")
c.add_semantic_for_token(idx, "random")
idx += 1
# Try to find sets (valued limited in variability with lower and upper bound)
token_formats = c.get_formats()
idx = 0
for token_format in token_formats:
rep, form, semantics = token_format
if form.getType() == Message.typeVariable:
stats = c.getVariableStatistics()[idx]
if stats != None:
distinct = stats.numberOfDistinctSamples()
else:
logging.error(
"Did not receive cluster statistics for token {0} (len of formats {1}, len of stats {2})".format(
idx, len(token_formats), len(c.getVariableStatistics())
)
)
idx += 1
continue
# How will be find out whether a number of variable values is a set or really variable?
# We assume that there is an absolute maximum amount of distinct values which is independent
# of the actual number of messages. However we also need to consider that when the number of
# messages in a cluster definitily falls below the setAbsoluteMax value, we have to adapt
# the local maximum in this cluster.
# For the moment we take a multiplier for the number of messages (default 0.3 == 30%) and
# assume it is a set, when both, setAbsoluteMax and the localThreshold is underrun
# In addition we assume that we have no semantics for this token, as other semantics conflict
# with the notion of a set
if (
distinct <= Globals.getConfig().setAbsoluteMax
and distinct <= (c.getNumberOfMessages() * Globals.getConfig().setPercentageThreshold)
and len(semantics) == 0
):
for message in messages: # Set for every message and the cluster itself
message.get_tokenlist()[idx].add_semantic("set")
c.add_semantic_for_token(idx - 1, "set")
idx += 1
# Push to cluster
pushUpToCluster(cluster_collection)
