def sanity_check(self): if not super(HTTP_CONNECT, self).sanity_check(): return False if not self.config.has_option(self.get_module_configname(), "proxyip"): common.internal_print( "'proxyip' option is missing from '{0}' section".format( self.get_module_configname()), -1) return False if not self.config.has_option(self.get_module_configname(), "proxyport"): common.internal_print( "'proxyport' option is missing from '{0}' section".format( self.get_module_configname()), -1) return False if not common.is_ipv4( self.config.get(self.get_module_configname(), "proxyip") ) and not common.is_ipv6( self.config.get(self.get_module_configname(), "proxyip")): common.internal_print( "'proxyip' should be ipv4 or ipv6 address in '{0}' section". format(self.get_module_configname()), -1) return False return True
def get_intermediate_hop(self, config): if config.has_option(self.get_module_configname(), "proxyip"): if common.is_ipv4(config.get(self.get_module_configname(), "proxyip")) or common.is_ipv6(config.get(self.get_module_configname(), "proxyip")): remoteserverip = config.get(self.get_module_configname(), "proxyip") return remoteserverip return ""
def http_connect_request(self, server_socket): if self.config.has_option("Global", "remoteserverhost"): if not common.is_hostname(self.config.get("Global", "remoteserverhost")): common.internal_print("[Global] remoteserverhost value is not a hostname", -1) return False else: remoteserver = self.config.get("Global", "remoteserverhost") else: if self.config.has_option("Global", "remoteserverip"): if not common.is_ipv4(self.config.get("Global", "remoteserverip")): common.internal_print("[Global] remoteserverip value is not an IPv4 address", -1) return False remoteserver = self.config.get("Global", "remoteserverip") serverport = int(self.config.get(self.get_module_configname(), "serverport")) request = "CONNECT %s:%d HTTP/1.1\r\nHost: %s\r\n\r\n" % (remoteserver, serverport, remoteserver) server_socket.send(request) response = server_socket.recv(4096) if response[9:12] != "200": common.internal_print("Connection failed: {0}".format(response[0:response.find("\n")]), -1) return False return True
def __init__(self, name, type, source=None, subtype=None, case_id=None): self.name = name self.created = timestamp() self.type = type self.subtype = subtype self.source = source self.parent = None self.children = [] self.case_id = case_id self.tags = [] self.notes = [] self.data = {} if self.type == 'host': if is_ipv4(self.name): self.subtype = 'ipv4' elif is_ipv6(self.name): self.subtype = 'ipv6' elif is_fqdn(self.name): self.subtype = 'fqdn' else: warning('host type cannot be determined. must be one of: ipv4, ipv6, fqdn') self.subtype = 'unknown' elif self.type == 'hash': result = is_hash(self.name) if result is None: warning('hash is not a valid md5, sha1, sha256, or sha512') self.subtype = 'unknown' else: self.subtype = result
def sanity_check(self): if not super(WebSocket, self).sanity_check(): return False if not self.config.has_option(self.get_module_configname(), "proxyip"): common.internal_print( "'proxyip' option is missing from '{0}' section".format( self.get_module_configname()), -1) return False if not self.config.has_option(self.get_module_configname(), "proxyport"): common.internal_print( "'proxyport' option is missing from '{0}' section".format( self.get_module_configname()), -1) return False if not common.is_ipv4( self.config.get(self.get_module_configname(), "proxyip") ) and not common.is_ipv6( self.config.get(self.get_module_configname(), "proxyip")): common.internal_print( "'proxyip' should be ipv4 or ipv6 address in '{0}' section". format(self.get_module_configname()), -1) return False if not self.config.has_option("Global", "remoteserverhost"): common.internal_print( "'remoteserverhost' option is missing from 'Global' section", -1) return False if not common.is_hostname(self.config.get( "Global", "remoteserverhost")) and not common.is_ipv4( self.config.get( "Global", "remoteserverhost")) and not common.is_ipv6( self.config.get("Global", "remoteserverhost")): common.internal_print( "'remoteserverhost' should be a hostname 'Global' section", -1) return False return True
def sanity_check(self): if not super(SOCKS, self).sanity_check(): return False if not self.config.has_option(self.get_module_configname(), "proxyip"): common.internal_print( "'proxyip' option is missing from '{0}' section".format( self.get_module_configname()), -1) return False if not self.config.has_option(self.get_module_configname(), "proxyport"): common.internal_print( "'proxyport' option is missing from '{0}' section".format( self.get_module_configname()), -1) return False if not common.is_ipv4( self.config.get(self.get_module_configname(), "proxyip") ) and not common.is_ipv6( self.config.get(self.get_module_configname(), "proxyip")): common.internal_print( "'proxyip' should be ipv4 or ipv6 address in '{0}' section". format(self.get_module_configname()), -1) return False if not self.config.has_option(self.get_module_configname(), "version"): common.internal_print( "'version' option is missing from '{0}' section".format( self.get_module_configname()), -1) return False version = self.config.get(self.get_module_configname(), "version") if version == "5": if not self.config.has_option(self.get_module_configname(), "usernamev5"): common.internal_print( "'usernamev5' option is missing from '{0}' section".format( self.get_module_configname()), -1) return False if not self.config.has_option(self.get_module_configname(), "passwordv5"): common.internal_print( "'passwordv5' option is missing from '{0}' section".format( self.get_module_configname()), -1) return False return True
def freebsd_set_default_route(self, serverip, clientip, ip): # get default gateway ps = subprocess.Popen(["route", "-n", "get", "default"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = ps.communicate() # is there a default gateway entry? if "route has not been found" in stderr: common.internal_print( "No default route. Please set up your routing before executing the tool", -1) sys.exit(-1) self.orig_default_gw = stdout.split("gateway: ")[1].split("\n")[0] # is it an ipv4 address? if not common.is_ipv4(self.orig_default_gw): common.internal_print("Default gateway is not an IPv4 address.", -1) sys.exit(-1) ps = subprocess.Popen([ "route", "add", "-net", serverip, self.orig_default_gw, "255.255.255.255" ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = ps.communicate() if stderr: if not "File exists" in stderr: common.internal_print( "Error: adding server route: {0}".format(stderr), -1) sys.exit(-1) ps = subprocess.Popen(["route", "delete", "default"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = ps.communicate() if stderr: common.internal_print( "Error: deleting default route: {0}".format(stderr), -1) sys.exit(-1) ps = subprocess.Popen(["route", "add", "default", ip], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = ps.communicate() if stderr: common.internal_print( "Error: adding new default route: {0}".format(stderr), -1) sys.exit(-1) return
def get_nameserver(self): if common.get_os_type() == common.OS_LINUX: f = open("/etc/resolv.conf", "r") content = f.read() f.close() for line in content.split("\n"): if line.replace(" ", "").replace("\t", "")[0:1] == "#": continue if line.find("nameserver") != -1: break nameserver = line[line.find("nameserver")+len("nameserver "):len(line)+line.find("nameserver")] if common.is_ipv4(nameserver) or common.is_ipv6(nameserver): return nameserver else: return None
def __init__(self, name, type, source=None, subtype=None, case_id=None): self.name = name self.created = timestamp() self.type = type self.subtype = subtype self.source = source self.parent = None self.children = [] self.case_id = case_id self.tags = [] self.notes = [] self.data = {} if self.subtype is None: if self.type == 'host': if is_ipv4(name): self.subtype = 'ipv4' elif is_ipv6(name): self.subtype = 'ipv6' elif is_fqdn(name): self.subtype = 'fqdn' else: warning('host subtype is not one of: ipv4, ipv6, fqdn') elif self.type == 'hash': hash_type = is_hash(name) if hash_type is None: warning('hash is not a valid md5, sha1, sha256, or sha512') else: self.subtype = hash_type elif self.type == 'user': self.subtype = 'account' elif self.type == 'email': self.subtype = 'account' elif self.type == 'btc': self.subtype = 'cryptocurrency address'
def mac_set_default_route(self, serverip, clientip, ip): # https://developer.apple.com/documentation/kernel/rt_msghdr?language=objc # s = socket(PF_ROUTE, SOCK_RAW, 0) # not sure which is the better way, calling external tools like # 'route' or implementing the messaging... # get default gateway ps = subprocess.Popen(["route", "-n", "get", "default"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = ps.communicate() # is there a default gateway entry? if "not in table" in stderr: common.internal_print( "No default route. Please set up your routing before executing the tool", -1) sys.exit(-1) self.orig_default_gw = stdout.split("gateway: ")[1].split("\n")[0] # is it an ipv4 address? if not common.is_ipv4(self.orig_default_gw): common.internal_print("Default gateway is not an IPv4 address.", -1) sys.exit(-1) ps = subprocess.Popen([ "route", "add", "-net", serverip, self.orig_default_gw, "255.255.255.255" ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = ps.communicate() if stderr: if not "File exists" in stderr: common.internal_print( "Error: adding server route: {0}".format(stderr), -1) sys.exit(-1) ps = subprocess.Popen(["route", "delete", "default"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = ps.communicate() if stderr: common.internal_print( "Error: deleting default route: {0}".format(stderr), -1) sys.exit(-1) ps = subprocess.Popen(["route", "add", "default", ip], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = ps.communicate() if stderr: common.internal_print( "Error: adding new default route: {0}".format(stderr), -1) sys.exit(-1) ''' # keeping this, in case I can test with tun, not utun ps = subprocess.Popen(["route", "add", "-net", clientip, serverip, "255.255.255.255"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = ps.communicate() if stderr: if not "File exists" in stderr: common.internal_print("Error: adding new route: {0}".format(stderr), -1) sys.exit(-1) ''' return
def perform_semantic_inference(cluster_collection): """ This function performs semantic inference on a list of clusters given For each message in these clusters semantics are inferred by analyzing the token resp. its context. At the moment only two semantics are automatically inferred: numeric and IPv4 address TODO: Add more semantics, e.g. EOL identifier, lenght fields, ... """ # Try to perform semantic inferences # Walk through every cluster and check messages for obvious results cluster = cluster_collection.get_all_cluster() for c in cluster: messages = c.get_messages() for message in messages: tokenlist = message.get_tokenlist() iterator = peekable(tokenlist) idx = 0 while not iterator.isLast(): #for tokenRepresentation in tokenlist: tokenRepresentation = iterator.next() # TODO: do we need to keep semantics which involve multiple cluster? e.g. sessionids? previous_semantics = tokenRepresentation.get_semantics() tokenRepresentation.set_semantics( []) # Clear existing semantics from previous run #for s in previous_semantics: # if s.startswith("sessionid"): # tokenRepresentation.add_semantic(s) # break if "sessionid" in previous_semantics: # Check if we have at least 2 messages and we are not of type Const if len(messages) > 1 and c.get_format( idx) != Message.typeConst: tokenRepresentation.add_semantic("sessionid") if "FD" in previous_semantics: tokenRepresentation.add_semantic("FD") token = tokenRepresentation.get_token() # Check whether it is numeric try: isNumber = tokenRepresentation.get_tokenType( ) == Message.typeText and common.is_number(token) except TypeError: if Globals.getConfig().debug: print "Error checking token {0} for number semantics".format( token) isNumber = False if isNumber: tokenRepresentation.add_semantic("numeric") #c.add_semantics(idx,"numeric") #print "Inferred semantic inference 'numeric' for token ", token # Check whether it is an IP address if isinstance(token, str) and common.is_ipv4(token): tokenRepresentation.add_semantic("ipv4 address") # Do not add to cluster unless it is valid for all c.add_semantics(idx,"ipv4 address") #print "Inferred semantic inference 'ipv4 address' for token ", token # Check for carriage return identifiers # When 0d is followed by 0a we've got a CR-LF # Sensible? When 0d or 0a is the last token, we've got a single CR resp LF # In all other cases assume 0d/0a is just a hex value of the protocol if token == 0xd: nextOne = iterator.peek() if isinstance(nextOne, TokenRepresentation): if nextOne.get_token() == 0xa: inferred_formats = c.get_format_inference() if inferred_formats[idx].getType( ) == Message.typeConst and inferred_formats[ idx + 1].getType() == Message.typeConst: tokenRepresentation.add_semantic("CR") #c.add_semantics(idx,"CR") nextOne = iterator.next() nextOne.set_semantics(["LF"]) #c.add_semantics(idx+1, "LF") idx += 1 idx += 1 # Perform other tests like "is length field?" # explicitely iterate through all messages like stated in the paper # we could also postpone this to the call of 'pushToClusterSeminatics" but.. reference_message = messages[0] tokenlist = reference_message.get_tokenlist() idx = 0 for tokenRepresentation in tokenlist: if tokenRepresentation.get_tokenType( ) == Message.typeBinary and idx + 1 < len(tokenlist): ref_value = tokenRepresentation.get_token() if not tokenlist[idx + 1].get_tokenType( ) == Message.typeText: # We require that the next token is the text token in question idx += 1 continue ref_next_length = tokenlist[idx + 1].get_length() if not ref_value == ref_next_length: # This is no length field idx += 1 continue ref_message_length = reference_message.get_length() is_length = True for message in messages: cmp_value = message.get_tokenlist()[idx].get_token() cmp_next_length = message.get_tokenlist()[idx + 1].get_length() cmp_message_length = message.get_length() try: diff_val = abs(cmp_value - ref_value) except TypeError: # Could happen if a short text token is mistaken as a binary value break diff_next_length = abs(cmp_next_length - ref_next_length) # The next line also takes total msg length differences into account. This might not be true for # all protocols diff_msg_length = abs(cmp_message_length - ref_message_length) if Globals.getConfig( ).requireTotalLengthChangeForLengthField: if not (diff_val == diff_next_length == diff_msg_length): is_length = False break else: if not (diff_val == diff_next_length): is_length = False break if is_length: # set "lengthfield" semantic for every message in the cluster at the given position for message in messages: # TODO: What if there's only one message in the cluster? Sensible? message.get_tokenlist()[idx].add_semantic( "lengthfield") c.add_semantic_for_token(idx, "lengthfield") idx += 1 # Try to identify sessionid fields reference_message = messages[0] nextInFlow = reference_message.getNextInFlow() if nextInFlow != None and not ( len(messages) == 1 and Globals.getConfig( ).sessionIDOnlyWithClustersWithMoreThanOneMessage): tokenlist = reference_message.get_tokenlist() next_tokenlist = nextInFlow.get_tokenlist() ref_idx = 0 for tokenRepresentation in tokenlist: tokType = tokenRepresentation.get_tokenType() # If its not a binary, it cannot be a cookie if tokType != Message.typeBinary: ref_idx += 1 continue fmt = c.get_format(ref_idx) # If its a binary but const, it cannot be a cookie if fmt[1] == Message.typeConst: ref_idx += 1 continue # Set reference value ref_val = tokenRepresentation.get_token() # Walk next flow for reference value next_idx = 0 for next_tokenRepresentation in next_tokenlist: # Retrieve next token type nextTokType = next_tokenRepresentation.get_tokenType() # If it is not a binary we don't see it as a cookie if Globals.getConfig().sessionIDOnlyWithBinary: if nextTokType != Message.typeBinary: next_idx += 1 continue next_cluster = nextInFlow.getCluster() # Get format of comparating message comp_fmt = next_cluster.get_format(next_idx) # If it is const, it cannot be a sessonid if comp_fmt[1] == Message.typeConst: next_idx += 1 continue # Load comparator value comp_val = next_tokenRepresentation.get_token() if ref_val == comp_val: # We've got a potential hit, now compare all messages for the same idx pairs isCookie = True for cmp_ref_msg in messages: if not isCookie: break if cmp_ref_msg == messages[ 0]: # Skip first message (we've already checked that one continue cmp_ref_tok_list = cmp_ref_msg.get_tokenlist() cmp_ref_val = cmp_ref_tok_list[ref_idx].get_token() cmp_cmp_msg = cmp_ref_msg.getNextInFlow() if cmp_cmp_msg == None: isCookie = False else: cmp_cmp_tok_list = cmp_cmp_msg.get_tokenlist() if next_idx >= len(cmp_cmp_tok_list): # Obviously "next" points to messages in different clusters # so the len might differ from the reference next cluster # used to find our reference cookie value # Therefore this cannot be a cookie isCookie = False continue # Make sure the comparing token is also not constant cmp_cmp_fmt = cmp_cmp_msg.getCluster( ).get_format(next_idx) # If it is const, it cannot be a sessonid if cmp_cmp_fmt == Message.typeConst: isCookie = False continue # Finally compare the values cmp_cmp_val = cmp_cmp_tok_list[ next_idx].get_token() if (cmp_ref_val != cmp_cmp_val) or ( (cmp_ref_val == cmp_cmp_val) and (cmp_ref_val == ref_val)): isCookie = False if isCookie: # Set cookie semantic in this message and the other #sessionid = uuid.uuid1() for message in messages: # Set for every message and the cluster itself #message.get_tokenlist()[ref_idx].add_semantic("sessionid_{0}".format(sessionid)) message.get_tokenlist()[ref_idx].add_semantic( "sessionid") nextMsg = message.getNextInFlow() #nextMsg.get_tokenlist()[next_idx].add_semantic("sessionid_{0}".format(sessionid)) nextMsg.get_tokenlist()[next_idx].add_semantic( "sessionid") c.add_semantic_for_token(ref_idx, "sessionid") #c.add_semantic_for_token(ref_idx,"sessionid_{0}".format(sessionid)) next_idx += 1 ref_idx += 1 # Try to find random fields (16 bit) token_formats = c.get_formats() idx = 0 for token_format in token_formats: rep, form, semantics = token_format if form.getType( ) == Message.typeVariable and rep == Message.typeBinary: try: variance = c.getVariableStatistics()[idx].getVariance() except Exception: pass if variance > 1000 and len(semantics) == 0: # We've got a very high variance and no assigned semantics --> candidate for random # Have a look at the last but one token if idx - 1 >= 0: rep, form, semantics = token_formats[idx - 1] if form.getType( ) == Message.typeVariable and rep == Message.typeBinary: stats = c.getVariableStatistics()[idx - 1] if stats != None: variance2 = stats.getVariance() else: logging.error( "Did not receive cluster statistics for token {0} (len of formats {1}, len of stats {2})" .format(idx, len(token_formats), len(c.getVariableStatistics()))) idx += 1 continue if variance2 > 1000 and len(semantics) == 0: # Consider the two as a CRC-16 for message in messages: # Set for every message and the cluster itself message.get_tokenlist()[ idx - 1].add_semantic("random") message.get_tokenlist()[idx].add_semantic( "random") c.add_semantic_for_token(idx - 1, "random") c.add_semantic_for_token(idx, "random") idx += 1 # Try to find sets (valued limited in variability with lower and upper bound) token_formats = c.get_formats() idx = 0 for token_format in token_formats: rep, form, semantics = token_format if form.getType() == Message.typeVariable: stats = c.getVariableStatistics()[idx] if stats != None: distinct = stats.numberOfDistinctSamples() else: logging.error( "Did not receive cluster statistics for token {0} (len of formats {1}, len of stats {2})" .format(idx, len(token_formats), len(c.getVariableStatistics()))) idx += 1 continue # How will be find out whether a number of variable values is a set or really variable? # We assume that there is an absolute maximum amount of distinct values which is independent # of the actual number of messages. However we also need to consider that when the number of # messages in a cluster definitily falls below the setAbsoluteMax value, we have to adapt # the local maximum in this cluster. # For the moment we take a multiplier for the number of messages (default 0.3 == 30%) and # assume it is a set, when both, setAbsoluteMax and the localThreshold is underrun # In addition we assume that we have no semantics for this token, as other semantics conflict # with the notion of a set if (distinct <= Globals.getConfig().setAbsoluteMax and distinct <= (c.getNumberOfMessages() * Globals.getConfig().setPercentageThreshold) and len(semantics) == 0): for message in messages: # Set for every message and the cluster itself message.get_tokenlist()[idx].add_semantic("set") c.add_semantic_for_token(idx - 1, "set") idx += 1 # Push to cluster pushUpToCluster(cluster_collection)
def run(self, argv): self.banner() try: opts, args = getopt.getopt(argv, self.short, self.long) except getopt.GetoptError: self.usage() sys.exit(-1) for opt, arg in opts: if opt in ("-h", "--help"): self.usage() sys.exit(0) elif opt in ("-s", "--server"): self.servermode = 1 elif opt in ("-c", "--client"): self.clientmode = 1 elif opt in ("--check"): self.checkmode = 1 elif opt in ("--config"): self.configfile = arg elif opt in ("--split"): self.splitmode = 1 elif opt in ("--ignore-dependencies"): common.internal_print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", -1) common.internal_print("!IGNORING MISSING DEPENDENCIES, THIS COULD RESULT IN UNHANDLED EXCEPTIONS!", -1) common.internal_print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", -1) self.ignoredependencies = 1 elif opt in ("--verbose"): try: self.verbosity = int(arg) except: common.internal_print("Invalid verbose value: {0}".format(arg), -1) sys.exit(-1) if not common.get_privilege_level(): common.internal_print("The tool needs superuser or admin privileges. Run as root or Administrator.", -1) sys.exit(-1) # set the servermode when it was not specified expicitly if (not self.clientmode and not self.checkmode): self.servermode = 1 # checking for the config file if not os.path.isfile(self.configfile): common.internal_print("config file missing!", -1) sys.exit(-1) # Looking for and parsing config file. common.internal_print("Parsing config file") config = ConfigParser.ConfigParser() try: config.read(self.configfile) except: common.internal_print("Invalid or malformed configuration file specified", -1) sys.exit(-1) # sanity check on configuration, exit on error if not common.config_sanity_check(config, (self.servermode)): sys.exit(-1) # load authentication module auth_module = self.authentication.load_auth_module(config) # sanity check for the auth module if not auth_module.sanity_check(config): sys.exit(-1) # initialize authentication if not auth_module.init(config, self.servermode): sys.exit(-1) # load encryption module encryption_module = self.encryption.load_encryption_module(config) # sanity check for the encryption module if not encryption_module.sanity_check(config): sys.exit(-1) # initialize encryption if not encryption_module.init(config, self.servermode): sys.exit(-1) if self.splitmode: self.scope = common.parse_scope_file(config.get("Global", "scope")) if self.scope == []: common.internal_print("Split tunnelling mode enabled, but no scope was defined or entries were invalid", -1) sys.exit(-1) # Check system config for routing. If the OS was not set up for IP # forwarding then we need to exit. if self.servermode: if not common.check_router_settings(config): sys.exit(-1) # Loading modules from modules/ directory # 1. listing and loading all modules except forbidden ones common.internal_print("Loading all modules from 'modules' directory") for module in os.listdir("./modules/"): if module == '__init__.py' or module[-3:] != '.py' or (module[:-3] in self.forbidden_modules): continue module_list = __import__("modules."+module[:-3], locals(), globals()) modules = [] for m in dir(module_list): # 2. going thru all the modules if m != 'Generic_module' and m[:2] != "__": module_attributes = getattr(module_list, m) # 3. get the classes from the modules module_classes = [c for c in module_attributes.__dict__.values() if inspect.isclass(c)] # 4. select classes that are XFLTReaT modules real_modules = [c for c in module_classes if (issubclass(c, Generic_module) and (c not in self.forbidden_modules_instance))] for r in real_modules: # 5. actual instantiation of a module try: modules.append(r()) except Exception as e: if self.ignoredependencies: common.internal_print("This module cannot be used without the dependencies installed: {0}".format(m), -1) pass else: raise # if the module is enabled from config, we store it in modules_enabled[] modules_enabled = [] for m in modules: enabled = "no" # is there any section in the config for the module? if not config.has_section(m.get_module_configname()): common.internal_print("No section in config for module: {0}".format(m.get_module_configname()), -1) continue # is the 'enabled' option there for the module? if not config.has_option(m.get_module_configname(), "enabled"): common.internal_print("No option 'enabled' in config for module: {0}".format(m.get_module_configname()), -1) sys.exit(-1) enabled = config.get(m.get_module_configname(), "enabled") if enabled == "yes": # looks like the module is enabled, adding to modules_enabled[] modules_enabled.append(m) # check if more than one module is enabled for client mode if self.clientmode and (len(modules_enabled)>1): common.internal_print("In client mode only one module can be used.", -1) sys.exit(-1) if not len(modules_enabled): common.internal_print("No modules were enabled in configuration", -1) sys.exit(-1) # One Interface to rule them all, One Interface to find them, # One Interface to bring them all and in the darkness bind them common.internal_print("Setting up interface") interface = Interface() # Setting up interface related things for server mode if self.servermode: server_tunnel = interface.tun_alloc(config.get("Global", "serverif"), interface.IFF_TUN|interface.IFF_NO_PI) interface.set_ip_address(config.get("Global", "serverif"), config.get("Global", "serverip"), config.get("Global", "serverip"), config.get("Global", "servernetmask")) interface.set_mtu(config.get("Global", "serverif"), int(config.get("Global", "mtu"))) # start thread with socket-interface related pipes ps = PacketSelector(server_tunnel) ps.start() # Setting up interface related things for client mode if self.clientmode: client_tunnel = interface.tun_alloc(config.get("Global", "clientif"), interface.IFF_TUN|interface.IFF_NO_PI) interface.set_ip_address(config.get("Global", "clientif"), config.get("Global", "clientip"), config.get("Global", "serverip"), config.get("Global", "clientnetmask")) if not self.splitmode: interface.set_default_route(config.get("Global", "remoteserverip"), config.get("Global", "clientip"), config.get("Global", "serverip")) else: interface.set_split_route(self.scope, config.get("Global", "serverip")) interface.set_mtu(config.get("Global", "clientif"), int(config.get("Global", "mtu"))) common.internal_print("Please use CTRL+C to exit...") module_threads = [] module_thread_num = 0 for m in modules_enabled: # Executing module in server mode if self.servermode: module_thread_num = module_thread_num + 1 if m.__init_thread__(module_thread_num, config, server_tunnel, ps, auth_module, encryption_module, self.verbosity): m.start() module_threads.append(m) # Executing module in check mode if self.checkmode: interface.check_default_route() if m.__init_thread__(0, config, None, None, None, None, self.verbosity): try: m.check() except KeyboardInterrupt: pass # Executing module in client mode if self.clientmode: try: remoteserverip = config.get("Global", "remoteserverip") if not config.has_section(m.get_module_configname()): common.internal_print("No section in config for module: {0}".format(m.get_module_configname()), -1) continue # if the module requires an indirect connection (proxy, # dns) then we need to amend the routing table intermediate_hop = m.get_intermediate_hop(config) if config.has_option("Global", "overriderouter"): if common.is_ipv4(config.get("Global", "overriderouter")): intermediate_hop = config.get("Global", "overriderouter") if intermediate_hop and (not self.splitmode): remoteserverip = intermediate_hop interface.set_intermediate_route(config.get("Global", "remoteserverip"), remoteserverip) # init "thread" for client mode, this will not be run in a thread. if m.__init_thread__(0, config, client_tunnel, None, auth_module, encryption_module, self.verbosity): # run in client mode m.connect() # client finished, closing down tunnel and restoring routes interface.close_tunnel(client_tunnel) if self.splitmode: interface.del_split_route(self.scope, config.get("Global", "serverip")) else: interface.restore_routes(remoteserverip, config.get("Global", "clientip"), config.get("Global", "serverip")) except KeyboardInterrupt: # CTRL+C was pushed interface.close_tunnel(client_tunnel) if self.splitmode: interface.del_split_route(self.scope, config.get("Global", "serverip")) else: interface.restore_routes(remoteserverip, config.get("Global", "clientip"), config.get("Global", "serverip")) pass except socket.error as e: # socket related error interface.close_tunnel(client_tunnel) if self.splitmode: interface.del_split_route(self.scope, config.get("Global", "serverip")) else: interface.restore_routes(remoteserverip, config.get("Global", "clientip"), config.get("Global", "serverip")) if e.errno == errno.ECONNREFUSED: common.internal_print("Socket does not seem to answer.", -1) else: common.internal_print("Socket died, probably the server went down. ({0})".format(e.errno), -1) except: interface.close_tunnel(client_tunnel) if self.splitmode: interface.del_split_route(self.scope, config.get("Global", "serverip")) else: interface.restore_routes(remoteserverip, config.get("Global", "clientip"), config.get("Global", "serverip")) raise # No modules are running if not module_threads: common.internal_print("Exiting...") if self.servermode: ps.stop() else: try: time.sleep(0.5) common.internal_print("Please use CTRL+C to exit...") # found no better solution to keep the main thread and catch CTRL+C # if you know any, you know how to tell me ;) while True: time.sleep(1000) except KeyboardInterrupt: common.internal_print("Interrupted. Exiting...") ps.stop() for t in module_threads: t.stop() if self.servermode: time.sleep(1.0) interface.close_tunnel(server_tunnel) # give one sec to clean up for modules, otherwise some objects just # disapper and cannot be closed properly like the tunnel interface try: time.sleep(1.0) except KeyboardInterrupt: common.internal_print("Are you really this impatient????", -1) pass
def socks_handshake(self, server_socket): version = self.config.get(self.get_module_configname(), "version") if (version == "4") or (version == "4a"): if self.config.has_option(self.get_module_configname(), "userid"): userid = self.config.get(self.get_module_configname(), "userid") + "\x00" else: userid = "\x00" if version == "4": server_socket.send( struct.pack( ">BBH4s", self.socks.SOCKS4_VERSION, self.socks.SOCKS4_CD, int( self.config.get(self.get_module_configname(), "serverport")), socket.inet_aton( self.config.get("Global", "remoteserverip"))) + userid) else: if self.config.has_option("Global", "remoteserverhost"): if not common.is_hostname( self.config.get("Global", "remoteserverhost")): common.internal_print( "[Global] remoteserverhost value is not a hostname", -1) return False domain = self.config.get("Global", "remoteserverhost") + "\x00" server_socket.send( struct.pack( ">BBH4s", self.socks.SOCKS4_VERSION, self.socks.SOCKS4_CD, int( self.config.get(self.get_module_configname(), "serverport")), socket.inet_aton("0.0.0.1")) + userid + domain) else: common.internal_print( "Missing remoteserverhost attribute from config [Global] section", -1) return False response = server_socket.recv(8) if (response[0:1] == self.socks.SOCKS4_OK) and ( response[1:2] == self.socks.SOCKS4_RESPONSES[0]): common.internal_print( "Connection was made through the proxy server", 1) else: if len(response) > 1: if (response[1:2] in self.socks.SOCKS4_RESPONSES): for i in range(len(self.socks.SOCKS4_RESPONSES)): if response[1:2] == self.socks.SOCKS4_RESPONSES[i]: common.internal_print( "Connection failed through the proxy server: {0}" .format( self.socks.SOCKS4_RESPONSES_STR[i]), -1) return False common.internal_print( "Connection failed through the proxy server: Unknown error", -1) return False if version == "5": # send greeting with auth method list greeting = struct.pack(">BB", self.socks.SOCKS5_VERSION, len(self.socks.SOCKS5_AUTH_METHODS)) for i in range(len(self.socks.SOCKS5_AUTH_METHODS)): greeting += self.socks.SOCKS5_AUTH_METHODS[i][0] server_socket.send(greeting) # receive response with selected auth method response = server_socket.recv(2) if (len(response) != 2) or (response[0:1] != chr( self.socks.SOCKS5_VERSION)): common.internal_print( "Connection failed through the proxy server: Unknown error", -1) return False if response[1:2] == self.socks.SOCKS5_REJECT_METHODS: common.internal_print( "Connection failed through the proxy server: Authentication methods rejected", -1) return False for i in range(len(self.socks.SOCKS5_AUTH_METHODS)): if response[1:2] == self.socks.SOCKS5_AUTH_METHODS[i][0]: if self.socks.SOCKS5_AUTH_METHODS[i][1](self.config, server_socket): if self.config.has_option("Global", "remoteserverhost"): remoteserverhost = self.config.get( "Global", "remoteserverhost") if not (common.is_hostname(remoteserverhost) or common.is_ipv4(remoteserverhost) or common.is_ipv6(remoteserverhost)): common.internal_print( "[Global] remoteserverhost value is not ipv4, ipv6 or a hostname", -1) return False else: common.internal_print( "Missing remoteserverhost attribute from config [Global] section", -1) return False if common.is_ipv4(remoteserverhost): host_type = self.socks.SOCKS5_ADDR_TYPE[0] connect_string = struct.pack( ">BBBB4sH", self.socks.SOCKS5_VERSION, self.socks.SOCKS5_CD, 0, host_type, socket.inet_aton(remoteserverhost), int( self.config.get( self.get_module_configname(), "serverport"))) if common.is_hostname(remoteserverhost): host_type = self.socks.SOCKS5_ADDR_TYPE[1] connect_string = struct.pack( ">BBBBB", self.socks.SOCKS5_VERSION, self.socks.SOCKS5_CD, 0, host_type, len(remoteserverhost)) connect_string += remoteserverhost + struct.pack( ">H", int( self.config.get( self.get_module_configname(), "serverport"))) if common.is_ipv6(remoteserverhost): host_type = self.socks.SOCKS5_ADDR_TYPE[2] connect_string = struct.pack( ">BBBB16sH", self.socks.SOCKS5_VERSION, self.socks.SOCKS5_CD, 0, host_type, socket.inet_pton(socket.AF_INET6, remoteserverhost), int( self.config.get( self.get_module_configname(), "serverport"))) server_socket.send(connect_string) response = server_socket.recv(4096) if (len(response) < 4) or (response[0:1] != chr( self.socks.SOCKS5_VERSION)): common.internal_print( "Connection failed through the proxy server: Connection error to the server", -1) return False if response[1:2] == self.socks.SOCKS5_RESPONSES[0]: return True if response[1:2] not in self.socks.SOCKS5_RESPONSES: common.internal_print( "Connection failed through the proxy server: Unknown error code", -1) return False for i in range(len(self.socks.SOCKS5_RESPONSES)): if response[1:2] == self.socks.SOCKS5_RESPONSES[i]: common.internal_print( "Connection failed through the proxy server: {0}" .format( self.socks.SOCKS5_RESPONSES_STR[i]), -1) return False return False else: return False common.internal_print( "Connection failed through the proxy server: Strange response (Authentication method)", -1) return False return True
def get_ssr_conf(ssr_conf_path: str) -> Dict: _ssr_conf: Dict = utils.parse_json(ssr_conf_path) if not _ssr_conf: exit.error('Require ssr-config.') # -- check params -- port = _ssr_conf.get('server_port') if port is None: exit.error('Require \'server_port\'.') if type(port) != int or port <= 0: exit.error('Illegal \'server_port\'.') password = _ssr_conf.get('password') if common.is_blank(password): exit.error('Require \'password\'.') method = _ssr_conf.get('method') if common.is_blank(method): exit.error('Require \'method\'.') if not encrypt.is_supported(method): exit.error(f'Not supported method [{method}]') protocol = _ssr_conf.get('protocol') if common.is_blank(protocol): exit.error('Require \'protocol\'.') obfs = _ssr_conf.get('obfs') if common.is_blank(obfs): exit.error('Require \'obfs\'.') dns_list = _ssr_conf.get('dns') if dns_list is None or type(dns_list) != list or len(dns_list) == 0: exit.error('Require \'dns server\'.') for dns in dns_list: host = dns.get('host') if ':' in host: exit.error("Not support ipv6 dns host.") if not common.is_ipv4(host): exit.error("Illegal ipv4 dns host.") # -- default params -- _ssr_conf['server'] = '::' _ssr_conf['password'] = common.to_bytes(_ssr_conf['password']) _ssr_conf['protocol_param'] = _ssr_conf.get('protocol_param', '') _ssr_conf['obfs_param'] = _ssr_conf.get('obfs_param', '') _ssr_conf['udp_enable'] = _ssr_conf.get('udp_enable', False) # process default data try: _ssr_conf['forbidden_ip'] = \ common.IPNetwork(_ssr_conf.get('forbidden_ip', '127.0.0.0/8,::1/128')) except Exception as e: exit.error('error configuration \'forbidden_ip\'.') try: _ssr_conf['forbidden_port'] = common.PortRange( _ssr_conf.get('forbidden_port', '')) except Exception as e: exit.error('error configuration \'forbidden_port\'.') try: _ssr_conf['ignore_bind'] = \ common.IPNetwork(_ssr_conf.get('ignore_bind', '127.0.0.0/8,::1/128,10.0.0.0/8,192.168.0.0/16')) except Exception as e: exit.error('error configuration \'ignore_bind\'.') return _ssr_conf
def __init__(self, argv): self.verbosity = 0 self.configfile = "xfltreat.conf" self.servermode = 0 self.clientmode = 0 self.checkmode = 0 self.short = "hsc" self.long = ["help", "server", "client", "check", "config=", "verbose="] self.forbidden_modules = ["Generic_module", "Stateful_module", "Stateless_module"] self.forbidden_modules_instance = [Generic_module, Stateful_module, Stateless_module] self.banner() try: opts, args = getopt.getopt(argv, self.short, self.long) except getopt.GetoptError: self.usage() exit(-1) for opt, arg in opts: if opt in ("-h", "--help"): self.usage() exit(0) elif opt in ("-s", "--server"): self.servermode = 1 elif opt in ("-c", "--client"): self.clientmode = 1 elif opt in ("--check"): self.checkmode = 1 elif opt in ("--config"): self.configfile = arg elif opt in ("--verbose"): try: self.verbosity = int(arg) except: common.internal_print("Invalid verbose value: {0}".format(arg), -1) exit(-1) if (not self.clientmode and not self.checkmode): self.servermode = 1 # checking for the config file if not os.path.isfile(self.configfile): common.internal_print("config file missing!", -1) exit(-1) # Looking for and parsing config file. common.internal_print("Parsing config file") config = ConfigParser.ConfigParser() try: config.read(self.configfile) except: common.internal_print("Invalid or malformed configuration file specified", -1) exit(-1) if not common.config_sanity_check(config, (self.servermode or (not self.clientmode and not self.checkmode))): exit(-1) # Loading modules from modules/ directory # 1. listing and loading all modules except forbidden ones common.internal_print("Loading all modules from 'modules' directory") for module in os.listdir("./modules/"): if module == '__init__.py' or module[-3:] != '.py' or (module[:-3] in self.forbidden_modules): continue module_list = __import__("modules."+module[:-3], locals(), globals()) modules = [] for m in dir(module_list): # 2. going thru all the modules if m != 'Generic_module' and m[:2] != "__": module_attributes = getattr(module_list, m) # 3. get the classes from the modules module_classes = [c for c in module_attributes.__dict__.values() if inspect.isclass(c)] # 4. select classes that are XFLTReaT modules real_modules = [c for c in module_classes if (issubclass(c, Generic_module) and (c not in self.forbidden_modules_instance))] for r in real_modules: # 5. actual instantiation of a module modules.append(r()) # if the module is enabled from config, we make store it in modules_enabled[] modules_enabled = [] for m in modules: enabled = "no" if not config.has_section(m.get_module_configname()): common.internal_print("No section in config for module: {0}".format(m.get_module_configname()), -1) continue enabled = config.get(m.get_module_configname(), "enabled") if enabled == "yes": # looks like the module is enabled, adding to modules|_enabled[] modules_enabled.append(m) # check if more than one module is enabled for client mode if self.clientmode and (len(modules_enabled)>1): common.internal_print("In client mode only one module can be used.", -1) exit(-1) # One Interface to rule them all, One Interface to find them, # One Interface to bring them all and in the darkness bind them common.internal_print("Setting up interface") interface = Interface() # Operating in server mode if self.servermode: server_tunnel = interface.tun_alloc(config.get("Global", "serverif"), interface.IFF_TUN|interface.IFF_NO_PI) interface.set_ip_address(config.get("Global", "serverif"), config.get("Global", "serverip"), config.get("Global", "servernetmask")) interface.set_mtu(config.get("Global", "serverif"), int(config.get("Global", "mtu"))) # start thread with socket-interface related pipes ps = PacketSelector(server_tunnel) ps.start() # Operating in client mode if self.clientmode: client_tunnel = interface.tun_alloc(config.get("Global", "clientif"), interface.IFF_TUN|interface.IFF_NO_PI) interface.set_ip_address(config.get("Global", "clientif"), config.get("Global", "clientip"), config.get("Global", "clientnetmask")) interface.set_default_route(config.get("Global", "remoteserverip"), config.get("Global", "serverip")) interface.set_mtu(config.get("Global", "clientif"), int(config.get("Global", "mtu"))) module_threads = [] module_thread_num = 0 for m in modules_enabled: if (self.servermode or (not self.clientmode and not self.checkmode)): module_thread_num = module_thread_num + 1 m.__init_thread__(module_thread_num, config, server_tunnel, ps, self.verbosity) m.start() module_threads.append(m) if self.clientmode: try: remoteserverip = config.get("Global", "remoteserverip") if not config.has_section(m.get_module_configname()): common.internal_print("No section in config for module: {0}".format(m.get_module_configname()), -1) continue if config.has_option(m.get_module_configname(), "proxyip"): if common.is_ipv4(config.get(m.get_module_configname(), "proxyip")) or common.is_ipv6(config.get(m.get_module_configname(), "proxyip")): remoteserverip = config.get(m.get_module_configname(), "proxyip") m.__init_thread__(0, config, client_tunnel, None, self.verbosity) if config.has_option(m.get_module_configname(), "proxyip"): interface.set_proxy_route(config.get("Global", "remoteserverip"), remoteserverip) m.client() interface.close_tunnel(client_tunnel) interface.restore_routes(remoteserverip) except KeyboardInterrupt: interface.close_tunnel(client_tunnel) interface.restore_routes(remoteserverip) pass except socket.error as e: interface.close_tunnel(client_tunnel) interface.restore_routes(remoteserverip) if e.errno == errno.ECONNREFUSED: common.internal_print("Socket does not seem to answer.", -1) else: common.internal_print("Socket died, probably the server went down.", -1) except: interface.close_tunnel(client_tunnel) interface.restore_routes(remoteserverip) raise if self.checkmode: m.__init_thread__(0, config, None, None, self.verbosity) try: m.check() except KeyboardInterrupt: pass # if not module_threads: common.internal_print("Exiting...") if (self.servermode or (not self.clientmode and not self.checkmode)): ps.stop() else: try: common.internal_print("Please use CTRL+C to exit...") while True: time.sleep(1000) except KeyboardInterrupt: common.internal_print("Interrupted. Exiting...") ps.stop() for t in module_threads: t.stop()
def perform_semantic_inference(cluster_collection): """ This function performs semantic inference on a list of clusters given For each message in these clusters semantics are inferred by analyzing the token resp. its context. At the moment only two semantics are automatically inferred: numeric and IPv4 address TODO: Add more semantics, e.g. EOL identifier, lenght fields, ... """ # Try to perform semantic inferences # Walk through every cluster and check messages for obvious results cluster = cluster_collection.get_all_cluster() for c in cluster: messages = c.get_messages() for message in messages: tokenlist = message.get_tokenlist() iterator = peekable(tokenlist) idx = 0 while not iterator.isLast(): # for tokenRepresentation in tokenlist: tokenRepresentation = iterator.next() # TODO: do we need to keep semantics which involve multiple cluster? e.g. sessionids? previous_semantics = tokenRepresentation.get_semantics() tokenRepresentation.set_semantics([]) # Clear existing semantics from previous run # for s in previous_semantics: # if s.startswith("sessionid"): # tokenRepresentation.add_semantic(s) # break if "sessionid" in previous_semantics: # Check if we have at least 2 messages and we are not of type Const if len(messages) > 1 and c.get_format(idx) != Message.typeConst: tokenRepresentation.add_semantic("sessionid") if "FD" in previous_semantics: tokenRepresentation.add_semantic("FD") token = tokenRepresentation.get_token() # Check whether it is numeric try: isNumber = tokenRepresentation.get_tokenType() == Message.typeText and common.is_number(token) except TypeError: if Globals.getConfig().debug: print "Error checking token {0} for number semantics".format(token) isNumber = False if isNumber: tokenRepresentation.add_semantic("numeric") # c.add_semantics(idx,"numeric") # print "Inferred semantic inference 'numeric' for token ", token # Check whether it is an IP address if isinstance(token, str) and common.is_ipv4(token): tokenRepresentation.add_semantic("ipv4 address") # Do not add to cluster unless it is valid for all c.add_semantics(idx,"ipv4 address") # print "Inferred semantic inference 'ipv4 address' for token ", token # Check for carriage return identifiers # When 0d is followed by 0a we've got a CR-LF # Sensible? When 0d or 0a is the last token, we've got a single CR resp LF # In all other cases assume 0d/0a is just a hex value of the protocol if token == 0xD: nextOne = iterator.peek() if isinstance(nextOne, TokenRepresentation): if nextOne.get_token() == 0xA: inferred_formats = c.get_format_inference() if ( inferred_formats[idx].getType() == Message.typeConst and inferred_formats[idx + 1].getType() == Message.typeConst ): tokenRepresentation.add_semantic("CR") # c.add_semantics(idx,"CR") nextOne = iterator.next() nextOne.set_semantics(["LF"]) # c.add_semantics(idx+1, "LF") idx += 1 idx += 1 # Perform other tests like "is length field?" # explicitely iterate through all messages like stated in the paper # we could also postpone this to the call of 'pushToClusterSeminatics" but.. reference_message = messages[0] tokenlist = reference_message.get_tokenlist() idx = 0 for tokenRepresentation in tokenlist: if tokenRepresentation.get_tokenType() == Message.typeBinary and idx + 1 < len(tokenlist): ref_value = tokenRepresentation.get_token() if ( not tokenlist[idx + 1].get_tokenType() == Message.typeText ): # We require that the next token is the text token in question idx += 1 continue ref_next_length = tokenlist[idx + 1].get_length() if not ref_value == ref_next_length: # This is no length field idx += 1 continue ref_message_length = reference_message.get_length() is_length = True for message in messages: cmp_value = message.get_tokenlist()[idx].get_token() cmp_next_length = message.get_tokenlist()[idx + 1].get_length() cmp_message_length = message.get_length() try: diff_val = abs(cmp_value - ref_value) except TypeError: # Could happen if a short text token is mistaken as a binary value break diff_next_length = abs(cmp_next_length - ref_next_length) # The next line also takes total msg length differences into account. This might not be true for # all protocols diff_msg_length = abs(cmp_message_length - ref_message_length) if Globals.getConfig().requireTotalLengthChangeForLengthField: if not (diff_val == diff_next_length == diff_msg_length): is_length = False break else: if not (diff_val == diff_next_length): is_length = False break if is_length: # set "lengthfield" semantic for every message in the cluster at the given position for message in messages: # TODO: What if there's only one message in the cluster? Sensible? message.get_tokenlist()[idx].add_semantic("lengthfield") c.add_semantic_for_token(idx, "lengthfield") idx += 1 # Try to identify sessionid fields reference_message = messages[0] nextInFlow = reference_message.getNextInFlow() if nextInFlow != None and not ( len(messages) == 1 and Globals.getConfig().sessionIDOnlyWithClustersWithMoreThanOneMessage ): tokenlist = reference_message.get_tokenlist() next_tokenlist = nextInFlow.get_tokenlist() ref_idx = 0 for tokenRepresentation in tokenlist: tokType = tokenRepresentation.get_tokenType() # If its not a binary, it cannot be a cookie if tokType != Message.typeBinary: ref_idx += 1 continue fmt = c.get_format(ref_idx) # If its a binary but const, it cannot be a cookie if fmt[1] == Message.typeConst: ref_idx += 1 continue # Set reference value ref_val = tokenRepresentation.get_token() # Walk next flow for reference value next_idx = 0 for next_tokenRepresentation in next_tokenlist: # Retrieve next token type nextTokType = next_tokenRepresentation.get_tokenType() # If it is not a binary we don't see it as a cookie if Globals.getConfig().sessionIDOnlyWithBinary: if nextTokType != Message.typeBinary: next_idx += 1 continue next_cluster = nextInFlow.getCluster() # Get format of comparating message comp_fmt = next_cluster.get_format(next_idx) # If it is const, it cannot be a sessonid if comp_fmt[1] == Message.typeConst: next_idx += 1 continue # Load comparator value comp_val = next_tokenRepresentation.get_token() if ( ref_val == comp_val ): # We've got a potential hit, now compare all messages for the same idx pairs isCookie = True for cmp_ref_msg in messages: if not isCookie: break if cmp_ref_msg == messages[0]: # Skip first message (we've already checked that one continue cmp_ref_tok_list = cmp_ref_msg.get_tokenlist() cmp_ref_val = cmp_ref_tok_list[ref_idx].get_token() cmp_cmp_msg = cmp_ref_msg.getNextInFlow() if cmp_cmp_msg == None: isCookie = False else: cmp_cmp_tok_list = cmp_cmp_msg.get_tokenlist() if next_idx >= len(cmp_cmp_tok_list): # Obviously "next" points to messages in different clusters # so the len might differ from the reference next cluster # used to find our reference cookie value # Therefore this cannot be a cookie isCookie = False continue # Make sure the comparing token is also not constant cmp_cmp_fmt = cmp_cmp_msg.getCluster().get_format(next_idx) # If it is const, it cannot be a sessonid if cmp_cmp_fmt == Message.typeConst: isCookie = False continue # Finally compare the values cmp_cmp_val = cmp_cmp_tok_list[next_idx].get_token() if (cmp_ref_val != cmp_cmp_val) or ( (cmp_ref_val == cmp_cmp_val) and (cmp_ref_val == ref_val) ): isCookie = False if isCookie: # Set cookie semantic in this message and the other # sessionid = uuid.uuid1() for message in messages: # Set for every message and the cluster itself # message.get_tokenlist()[ref_idx].add_semantic("sessionid_{0}".format(sessionid)) message.get_tokenlist()[ref_idx].add_semantic("sessionid") nextMsg = message.getNextInFlow() # nextMsg.get_tokenlist()[next_idx].add_semantic("sessionid_{0}".format(sessionid)) nextMsg.get_tokenlist()[next_idx].add_semantic("sessionid") c.add_semantic_for_token(ref_idx, "sessionid") # c.add_semantic_for_token(ref_idx,"sessionid_{0}".format(sessionid)) next_idx += 1 ref_idx += 1 # Try to find random fields (16 bit) token_formats = c.get_formats() idx = 0 for token_format in token_formats: rep, form, semantics = token_format if form.getType() == Message.typeVariable and rep == Message.typeBinary: try: variance = c.getVariableStatistics()[idx].getVariance() except Exception: pass if variance > 1000 and len(semantics) == 0: # We've got a very high variance and no assigned semantics --> candidate for random # Have a look at the last but one token if idx - 1 >= 0: rep, form, semantics = token_formats[idx - 1] if form.getType() == Message.typeVariable and rep == Message.typeBinary: stats = c.getVariableStatistics()[idx - 1] if stats != None: variance2 = stats.getVariance() else: logging.error( "Did not receive cluster statistics for token {0} (len of formats {1}, len of stats {2})".format( idx, len(token_formats), len(c.getVariableStatistics()) ) ) idx += 1 continue if variance2 > 1000 and len(semantics) == 0: # Consider the two as a CRC-16 for message in messages: # Set for every message and the cluster itself message.get_tokenlist()[idx - 1].add_semantic("random") message.get_tokenlist()[idx].add_semantic("random") c.add_semantic_for_token(idx - 1, "random") c.add_semantic_for_token(idx, "random") idx += 1 # Try to find sets (valued limited in variability with lower and upper bound) token_formats = c.get_formats() idx = 0 for token_format in token_formats: rep, form, semantics = token_format if form.getType() == Message.typeVariable: stats = c.getVariableStatistics()[idx] if stats != None: distinct = stats.numberOfDistinctSamples() else: logging.error( "Did not receive cluster statistics for token {0} (len of formats {1}, len of stats {2})".format( idx, len(token_formats), len(c.getVariableStatistics()) ) ) idx += 1 continue # How will be find out whether a number of variable values is a set or really variable? # We assume that there is an absolute maximum amount of distinct values which is independent # of the actual number of messages. However we also need to consider that when the number of # messages in a cluster definitily falls below the setAbsoluteMax value, we have to adapt # the local maximum in this cluster. # For the moment we take a multiplier for the number of messages (default 0.3 == 30%) and # assume it is a set, when both, setAbsoluteMax and the localThreshold is underrun # In addition we assume that we have no semantics for this token, as other semantics conflict # with the notion of a set if ( distinct <= Globals.getConfig().setAbsoluteMax and distinct <= (c.getNumberOfMessages() * Globals.getConfig().setPercentageThreshold) and len(semantics) == 0 ): for message in messages: # Set for every message and the cluster itself message.get_tokenlist()[idx].add_semantic("set") c.add_semantic_for_token(idx - 1, "set") idx += 1 # Push to cluster pushUpToCluster(cluster_collection)