def update_trails(server=None, force=False): """ Update trails from feeds """ trails = {} duplicates = {} if server: print "[i] retrieving trails from provided 'UPDATE_SERVER' server..." _ = retrieve_content(server) if not _: exit("[!] unable to retrieve data from '%s'" % server) else: with _fopen(TRAILS_FILE, "w+b") as f: f.write(_) trails = load_trails() trail_files = set() for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(ROOT_DIR, "trails"))) : for filename in filenames: trail_files.add(os.path.abspath(os.path.join(dirpath, filename))) if config.CUSTOM_TRAILS_DIR: for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(ROOT_DIR, os.path.expanduser(config.CUSTOM_TRAILS_DIR)))) : for filename in filenames: trail_files.add(os.path.abspath(os.path.join(dirpath, filename))) try: if not os.path.isdir(USERS_DIR): os.makedirs(USERS_DIR, 0755) except Exception, ex: exit("[!] something went wrong during creation of directory '%s' ('%s')" % (USERS_DIR, ex))
def update_timer(): retries = 0 while retries < CHECK_CONNECTION_MAX_RETRIES and not check_connection(): sys.stdout.write("[!] can't update because of lack of network connection (waiting..." if not retries else '.') sys.stdout.flush() time.sleep(10) retries += 1 if retries: print(")") if retries == CHECK_CONNECTION_MAX_RETRIES: print("[x] going to continue without update") _ = {} else: _ = update_trails(server=config.UPDATE_SERVER) update_ipcat() if _: trails.clear() trails.update(_) elif not trails: trails.update(load_trails()) thread = threading.Timer(config.UPDATE_PERIOD, update_timer) thread.daemon = True thread.start()
def update_timer(): retries = 0 if not config.no_updates: while retries < CHECK_CONNECTION_MAX_RETRIES and not check_connection(): sys.stdout.write("[ERROR]: can't update because of lack of Internet connection (waiting..." if not retries else '.') sys.stdout.flush() time.sleep(10) retries += 1 if retries: sys.stdout.write(")\n") if config.no_updates or retries == CHECK_CONNECTION_MAX_RETRIES: if retries == CHECK_CONNECTION_MAX_RETRIES: logger.error("going to continue without online update") _ = update_trails(offline=True) else: _ = update_trails(server=config.UPDATE_SERVER) update_ipcat() if _: trails.clear() trails.update(_) elif not trails: trails.update(load_trails()) thread = threading.Timer(config.UPDATE_PERIOD, update_timer) thread.start()
def update_trails(server=None, force=False): """ Update trails from feeds """ trails = {} duplicates = {} if server: print "[i] retrieving trails from provided 'UPDATE_SERVER' server..." _ = retrieve_content(server) if not _: exit("[!] unable to retrieve data from '%s'" % server) else: with _fopen(TRAILS_FILE, "w+b") as f: f.write(_) trails = load_trails() trail_files = [] for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(ROOT_DIR, "trails"))) : for filename in filenames: trail_files.append(os.path.abspath(os.path.join(dirpath, filename))) if config.CUSTOM_TRAILS_DIR: for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(ROOT_DIR, os.path.expanduser(config.CUSTOM_TRAILS_DIR)))) : for filename in filenames: trail_files.append(os.path.abspath(os.path.join(dirpath, filename))) try: if not os.path.isdir(USERS_DIR): os.makedirs(USERS_DIR, 0755) except Exception, ex: exit("[!] something went wrong during creation of directory '%s' ('%s')" % (USERS_DIR, ex))
def update(server=None): """ Update trails from feeds """ trails = {} if server: print "[i] retrieving trails from provided 'UPDATE_SERVER' server..." _ = retrieve_content(server) if not _: print "[!] unable to retrieve data from '%s'" % server else: with _fopen_trails("w+b") as f: f.write(_) trails = load_trails() if not trails and ( (not os.path.isfile(TRAILS_FILE) or (time.time() - os.stat(TRAILS_FILE).st_mtime) >= config.UPDATE_PERIOD or os.stat(TRAILS_FILE).st_size == 0)): try: if not os.path.isdir(USERS_DIRECTORY): os.makedirs(USERS_DIRECTORY, 0755) except Exception, ex: exit( "[!] something went wrong during creation of directory '%s' ('%s')" % (USERS_DIRECTORY, ex)) print "[i] updating trails..." sys.path.append( os.path.abspath(os.path.join(ROOT_DIR, "trails", "feeds"))) filenames = glob.glob(os.path.join(sys.path[-1], "*.py")) sys.path.append(os.path.abspath(os.path.join(ROOT_DIR, "trails"))) filenames += [os.path.join(sys.path[-1], "static")] filenames += [os.path.join(sys.path[-1], "custom")] for filename in filenames: try: module = __import__(os.path.basename(filename).split(".py")[0]) except (ImportError, SyntaxError), ex: print "[!] something went wrong during import of feed file '%s' ('%s')" % ( filename, ex) continue for name, function in inspect.getmembers(module, inspect.isfunction): if name == "fetch": print(" [o] '%s'" % module.__url__) results = function() for item in results.items(): if not (any(_ in item[1][0] for _ in LOW_PRIORITY_INFO_KEYWORDS) and item[0] in trails): trails[item[0]] = item[1] if not results: print "[!] something went wrong during remote data retrieval ('%s')" % module.__url__
def update_timer(): _ = update(server=config.UPDATE_SERVER) if _: trails.clear() trails.update(_) elif not trails: trails.update(load_trails()) thread = threading.Timer(config.UPDATE_PERIOD, update_timer) thread.daemon = True thread.start()
def update_timer(): if (time.time() - os.stat(TRAILS_FILE).st_mtime) >= config.UPDATE_PERIOD: _ = None while True: _ = load_trails(True) if _: trails.clear() trails.update(_) break else: time.sleep(LOAD_TRAILS_RETRY_SLEEP_TIME) threading.Timer(config.UPDATE_PERIOD, update_timer).start()
def update_trails(): """ Update trails from feeds """ print datetime.now().strftime('%Y-%m-%d:%H') trails = TrailsDict() trails.update(load_trails()) #load trails list_trails = [] if not (os.path.isfile('trail_info.csv') ): # create a csv for first time of updating print "can't find collecting_info.csv! then init a new csv" old = pd.read_csv(TRAILS_CSV, names=['trail', 'info', 'ref']) old['ftime'] = datetime.now().strftime('%Y-%m-%d:%H') old['ltime'] = datetime.now().strftime('%Y-%m-%d:%H') old.to_csv('trail_info.csv', index=False) else: old = pd.read_csv('trail_info.csv') old_trails = set(old.trail) print "[i] collecting trails information (ftime,ltime)..." filenames = sorted(glob.glob(os.path.join(FEEDS, "*.py"))) filenames = [_ for _ in filenames if "__init__.py" not in _] filenames += sorted(glob.glob(os.path.join( STATIC, "*.py"))) # in static folder, __init__.py has fetch() filenames += sorted(glob.glob(os.path.join( CUSTOM, "*.py"))) # in custom folder, __init__.py has fetch() #remove __init__.py in feeds folder #filenames = [_ for _ in filenames if "__init__.py" not in _] #print filenames init_sys_path() time = datetime.now().strftime('%Y-%m-%d:%H') for i in xrange(len(filenames)): f = filenames[i] try: module = __import__(os.path.basename(f).split(".py")[0]) except (ImportError, SyntaxError), ex: print "[x] Failed: import feed file '%s' ('%s')" % (f, ex) continue for name, function in inspect.getmembers(module, inspect.isfunction): if name == "fetch": print "[o] '%s'" % (module.__url__) sys.stdout.write("[?] progress: %d/%d (%d%%)\r" % \ (i, len(filenames), i * 100 / len(filenames))) sys.stdout.flush() results = function() for item in results.items(): list_trails.append( (item[0], item[1][0], item[1][1], time, time))
def update(server=None): """ Update trails from feeds """ trails = {} if server: print "[i] retrieving trails from provided 'UPDATE_SERVER' server..." _ = retrieve_content(server) if not _: print "[!] unable to retrieve data from '%s'" % server else: with _fopen_trails("w+b") as f: f.write(_) trails = load_trails() if not trails and ((not os.path.isfile(TRAILS_FILE) or (time.time() - os.stat(TRAILS_FILE).st_mtime) >= config.UPDATE_PERIOD or os.stat(TRAILS_FILE).st_size == 0)): try: if not os.path.isdir(USERS_DIR): os.makedirs(USERS_DIR, 0755) except Exception, ex: exit("[!] something went wrong during creation of directory '%s' ('%s')" % (USERS_DIR, ex)) print "[i] updating trails..." if config.USE_FEED_UPDATES: sys.path.append(os.path.abspath(os.path.join(ROOT_DIR, "trails", "feeds"))) filenames = glob.glob(os.path.join(sys.path[-1], "*.py")) else: filenames = [] sys.path.append(os.path.abspath(os.path.join(ROOT_DIR, "trails"))) filenames += [os.path.join(sys.path[-1], "static")] filenames += [os.path.join(sys.path[-1], "custom")] for filename in filenames: try: module = __import__(os.path.basename(filename).split(".py")[0]) except (ImportError, SyntaxError), ex: print "[!] something went wrong during import of feed file '%s' ('%s')" % (filename, ex) continue for name, function in inspect.getmembers(module, inspect.isfunction): if name == "fetch": print(" [o] '%s'" % module.__url__) results = function() for item in results.items(): if not (any(_ in item[1][0] for _ in LOW_PRIORITY_INFO_KEYWORDS) and item[0] in trails): trails[item[0]] = item[1] if not results: print "[!] something went wrong during remote data retrieval ('%s')" % module.__url__
def update_timer(): retries = 0 if not config.no_updates: # 判断是否设置不更新,然后会利用抓取页面检测网络状态 while retries < CHECK_CONNECTION_MAX_RETRIES and not check_connection( ): sys.stdout.write( "[!] can't update because of lack of Internet connection (waiting..." if not retries else '.') sys.stdout.flush() log_error( "[!] can't update because of lack of Internet connection (waiting...", "Warning") time.sleep(10) retries += 1 if retries: print(")") # 超出次数,那么使用update_trails的离线模式 if config.no_updates or retries == CHECK_CONNECTION_MAX_RETRIES: if retries == CHECK_CONNECTION_MAX_RETRIES: print("[x] going to continue without online update") log_error("[x] going to continue without online update", "Warning") _ = update_trails(offline=True) else: # 正常进入 _ = update_trails() # update_ipcat() # 有新的trails if _: trails.clear() trails.update(_) elif not trails: # load_trails()只是加载trails()进内存 _ = load_trails() trails.update(_) _regex = "" for trail in trails: if "static" in trails[trail][1]: if re.search(r"[\].][*+]|\[[a-z0-9_.\-]+\]", trail, re.I): try: re.compile(trail) except: pass else: if re.escape(trail) != trail: index = _regex.count("(?P<g") if index < 100: # Reference: https://stackoverflow.com/questions/478458/python-regular-expressions-with-more-than-100-groups _regex += "|(?P<g%s>%s)" % (index, trail) trails._regex = _regex.strip('|')
def update_timer(): # 查看trail.csv的上一次修改时间,差1天更新,其实这里的trails已经在初始化阶段加入了trail.csv的内容。 # **只是利用这个函数来读取trail.csv的新内容** # 因为只是读,所以没有加锁。所以主线程1天更新一次,多线程采用读来同步trails的状态。 if (time.time() - os.stat(config.TRAILS_FILE).st_mtime) >= config.UPDATE_PERIOD: _ = None while True: _ = load_trails(True) if _: trails.clear() trails.update(_) break else: time.sleep(LOAD_TRAILS_RETRY_SLEEP_TIME) # 1天后再次执行 threading.Timer(config.UPDATE_PERIOD, update_timer).start()
def cron_job_load_trails(): ''' 重新加载trails到内存 :return: ''' read_config(CONFIG_FILE) _ = load_trails() trails.update(_) _regex = "" for trail in trails: if "static" in trails[trail][1]: if re.search(r"[\].][*+]|\[[a-z0-9_.\-]+\]", trail, re.I): try: re.compile(trail) except: pass else: if re.escape(trail) != trail: index = _regex.count("(?P<g") if index < 100: # Reference: https://stackoverflow.com/questions/478458/python-regular-expressions-with-more-than-100-groups _regex += "|(?P<g%s>%s)" % (index, trail) trails._regex = _regex.strip('|')
def update_timer(): first = True while not check_connection(): sys.stdout.write("[!] can't update because of lack of network connection (waiting..." if first else '.') sys.stdout.flush() time.sleep(60) first = False if not first: print(")") _ = update_trails(server=config.UPDATE_SERVER) update_ipcat() if _: trails.clear() trails.update(_) elif not trails: trails.update(load_trails()) thread = threading.Timer(config.UPDATE_PERIOD, update_timer) thread.daemon = True thread.start()
except Exception, ex: exit( "[!] something went wrong during creation of directory '%s' ('%s')" % (USERS_DIR, ex)) _chown(USERS_DIR) if server: print "[i] retrieving trails from provided 'UPDATE_SERVER' server..." content = retrieve_content(server) if not content: exit("[!] unable to retrieve data from '%s'" % server) else: with _fopen(TRAILS_FILE, "w+b") as f: f.write(content) trails = load_trails() trail_files = set() for dirpath, dirnames, filenames in os.walk( os.path.abspath(os.path.join(ROOT_DIR, "trails"))): for filename in filenames: trail_files.add(os.path.abspath(os.path.join(dirpath, filename))) if config.CUSTOM_TRAILS_DIR: for dirpath, dirnames, filenames in os.walk( os.path.abspath( os.path.join(ROOT_DIR, os.path.expanduser( config.CUSTOM_TRAILS_DIR)))): for filename in filenames: trail_files.add(
def update(server=None): """ Update trails from feeds """ trails = {} duplicates = {} if server: print "[i] retrieving trails from provided 'UPDATE_SERVER' server..." _ = retrieve_content(server) if not _: print "[!] unable to retrieve data from '%s'" % server else: with _fopen_trails("w+b") as f: f.write(_) trails = load_trails() trail_files = [] for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(ROOT_DIR, "trails"))) : for filename in filenames: trail_files.append(os.path.abspath(os.path.join(dirpath, filename))) if config.CUSTOM_TRAILS_DIR: for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(ROOT_DIR, os.path.expanduser(config.CUSTOM_TRAILS_DIR)))) : for filename in filenames: trail_files.append(os.path.abspath(os.path.join(dirpath, filename))) if not trails and ((not os.path.isfile(TRAILS_FILE) or (time.time() - os.stat(TRAILS_FILE).st_mtime) >= config.UPDATE_PERIOD or os.stat(TRAILS_FILE).st_size == 0 or any(os.stat(_).st_mtime > os.stat(TRAILS_FILE).st_mtime for _ in trail_files))): try: if not os.path.isdir(USERS_DIR): os.makedirs(USERS_DIR, 0755) except Exception, ex: exit("[!] something went wrong during creation of directory '%s' ('%s')" % (USERS_DIR, ex)) print "[i] updating trails (this might take a while)..." if config.USE_FEED_UPDATES: sys.path.append(os.path.abspath(os.path.join(ROOT_DIR, "trails", "feeds"))) filenames = sorted(glob.glob(os.path.join(sys.path[-1], "*.py"))) else: filenames = [] sys.path.append(os.path.abspath(os.path.join(ROOT_DIR, "trails"))) filenames += [os.path.join(sys.path[-1], "static")] filenames += [os.path.join(sys.path[-1], "custom")] for filename in filenames: try: module = __import__(os.path.basename(filename).split(".py")[0]) except (ImportError, SyntaxError), ex: print "[!] something went wrong during import of feed file '%s' ('%s')" % (filename, ex) continue for name, function in inspect.getmembers(module, inspect.isfunction): if name == "fetch": print(" [o] '%s'" % module.__url__) try: results = function() for item in results.items(): if item[0] in trails: if item[0] not in duplicates: duplicates[item[0]] = set((trails[item[0]][1],)) duplicates[item[0]].add(item[1][1]) if not (item[0] in trails and (any(_ in item[1][0] for _ in LOW_PRIORITY_INFO_KEYWORDS) or trails[item[0]][1] in HIGH_PRIORITY_REFERENCES)) or item[1][1] in HIGH_PRIORITY_REFERENCES: trails[item[0]] = item[1] if not results and "abuse.ch" not in module.__url__: print "[!] something went wrong during remote data retrieval ('%s')" % module.__url__ except Exception, ex: print "[!] something went wrong during processing of feed file '%s' ('%s')" % (filename, ex)
def update_trails(): """ Update trails from feeds """ print datetime.now().strftime('%Y-%m-%d:%H') trails = TrailsDict() trails.update(load_trails()) #load trails '''old=pd.read_csv(TRAILS_CSV,names=['trail', 'info', 'ref']) old['ftime']=datetime.now().strftime('%Y-%m-%d:%H') old['ltime']=datetime.now().strftime('%Y-%m-%d:%H') old.to_csv('old_info.csv',index_label="id") exit(1)''' list_trails = [] if not (os.path.isfile('trail_info.csv') ): # create a csv for first time of updating print "can't find collecting_info.csv!" old = pd.read_csv(TRAILS_CSV, names=['trail', 'info', 'ref']) old['ftime'] = datetime.now().strftime('%Y-%m-%d:%H') old['ltime'] = datetime.now().strftime('%Y-%m-%d:%H') old.to_csv('trail_info.csv', index_label="id") else: old = pd.read_csv('trail_info.csv') old_trails = set(old.trail) print "[i] updating trails (this might take a while)..." filenames = sorted(glob.glob(os.path.join(FEEDS, "*.py"))) filenames += [STATIC] # in static folder, __init__.py has fetch() filenames += [CUSTOM] # in custom folder, __init__.py has fetch() # remove __init__.py in feeds folder filenames = [_ for _ in filenames if "__init__.py" not in _] init_sys_path() for i in xrange(len(filenames)): f = filenames[i] try: module = __import__(os.path.basename(f).split(".py")[0]) except (ImportError, SyntaxError), ex: print "[x] Failed: import feed file '%s' ('%s')" % (f, ex) continue for name, function in inspect.getmembers(module, inspect.isfunction): if name == "fetch": print "[o] '%s'" % (module.__url__) sys.stdout.write("[?] progress: %d/%d (%d%%)\r" % \ (i, len(filenames), i * 100 / len(filenames))) sys.stdout.flush() try: results = function() #print(1) for item in results.items(): list_trails.append( (item[0], item[1][0], item[1][1], datetime.now().strftime('%Y-%m-%d:%H'), datetime.now().strftime('%Y-%m-%d:%H'))) '''if(item[0] in trails): tmp=str(old[old.trail == item[0]].ftime) list_trails.append((item[0], item[1][0],item[1][1],tmp[5:18],time)) #print(1) else: list_trails.append((item[0], item[1][0],item[1][1],time,time)) ''' except Exception, ex: print "[x] Failed: process feed file '%s' ('%s')" % ( filename, ex)
from core.settings import trails import sys from core.common import load_trails trails.update(load_trails()) ''' trails contains blacklisted strings: domains and Ips e.g. mqbbsagabardinedazyx.com dgj4gu1xmithip.net moegestnessbiophysicalohax.com rhmxancorml.com 115.28.7.221 jjcwgfwdyqje.pw ''' for line in sys.stdin: try: line = line.strip() data = line.split(',') if data[5] in trails or data[9] in trails: print str(data[6]) + "," + str(100) else: print str(data[6]) + "," + str(0) except: print line
from core.settings import trails import sys from core.common import load_trails trails.update(load_trails()) ''' trails contains blacklisted strings: domains and Ips e.g. mqbbsagabardinedazyx.com dgj4gu1xmithip.net moegestnessbiophysicalohax.com rhmxancorml.com 115.28.7.221 jjcwgfwdyqje.pw ''' for line in sys.stdin: try: line = line.strip() data = line.split(',') if data[5] in trails or data[9] in trails: print str(data[6])+","+str(100) else: print str(data[6])+","+str(0) except: print line
def update_trails(force=False, offline=False): """ Update trails from feeds """ success = False trails = TrailsDict() duplicates = {} try: if not os.path.isdir(USERS_DIR): os.makedirs(USERS_DIR, 0o755) except Exception as ex: exit( "[!] something went wrong during creation of directory '%s' ('%s')" % (USERS_DIR, ex)) _chown(USERS_DIR) if config.UPDATE_SERVER: print("[i] retrieving trails from provided 'UPDATE_SERVER' server...") content = retrieve_content(config.UPDATE_SERVER) if not content or content.count(',') < 2: print("[x] unable to retrieve data from '%s'" % config.UPDATE_SERVER) else: with _fopen(config.TRAILS_FILE, "w+b" if six.PY2 else "w+", open if six.PY2 else codecs.open) as f: f.write(content) trails = load_trails() else: trail_files = set() for dirpath, dirnames, filenames in os.walk( os.path.abspath(os.path.join(ROOT_DIR, "trails"))): for filename in filenames: trail_files.add( os.path.abspath(os.path.join(dirpath, filename))) if config.CUSTOM_TRAILS_DIR: for dirpath, dirnames, filenames in os.walk( os.path.abspath( os.path.join( ROOT_DIR, os.path.expanduser(config.CUSTOM_TRAILS_DIR)))): for filename in filenames: trail_files.add( os.path.abspath(os.path.join(dirpath, filename))) if not trails and ( force or not os.path.isfile(config.TRAILS_FILE) or (time.time() - os.stat(config.TRAILS_FILE).st_mtime) >= config.UPDATE_PERIOD or os.stat(config.TRAILS_FILE).st_size == 0 or any( os.stat(_).st_mtime > os.stat(config.TRAILS_FILE).st_mtime for _ in trail_files)): if not config.no_updates: print("[i] updating trails (this might take a while)...") else: print("[i] checking trails...") if not offline and (force or config.USE_FEED_UPDATES): _ = os.path.abspath(os.path.join(ROOT_DIR, "trails", "feeds")) if _ not in sys.path: sys.path.append(_) filenames = sorted(glob.glob(os.path.join(_, "*.py"))) else: filenames = [] _ = os.path.abspath(os.path.join(ROOT_DIR, "trails")) if _ not in sys.path: sys.path.append(_) filenames += [os.path.join(_, "custom")] filenames += [ os.path.join(_, "static") ] # Note: higher priority than previous one because of dummy user trails (FE) filenames = [_ for _ in filenames if "__init__.py" not in _] if config.DISABLED_FEEDS: filenames = [ filename for filename in filenames if os.path.splitext(os.path.split(filename)[-1])[0] not in re.split(r"[^\w]+", config.DISABLED_FEEDS) ] for i in xrange(len(filenames)): filename = filenames[i] try: module = __import__( os.path.basename(filename).split(".py")[0]) except (ImportError, SyntaxError) as ex: print( "[x] something went wrong during import of feed file '%s' ('%s')" % (filename, ex)) continue for name, function in inspect.getmembers( module, inspect.isfunction): if name == "fetch": url = module.__url__ # Note: to prevent "SyntaxError: can not delete variable 'module' referenced in nested scope" print(" [o] '%s'%s" % (url, " " * 20 if len(url) < 20 else "")) sys.stdout.write( "[?] progress: %d/%d (%d%%)\r" % (i, len(filenames), i * 100 // len(filenames))) sys.stdout.flush() if config.DISABLED_TRAILS_INFO_REGEX and re.search( config.DISABLED_TRAILS_INFO_REGEX, getattr(module, "__info__", "")): continue try: results = function() for item in results.items(): if item[0].startswith( "www.") and '/' not in item[0]: item = [item[0][len("www."):], item[1]] if item[0] in trails: if item[0] not in duplicates: duplicates[item[0]] = set( (trails[item[0]][1], )) duplicates[item[0]].add(item[1][1]) if not ( item[0] in trails and (any(_ in item[1][0] for _ in LOW_PRIORITY_INFO_KEYWORDS) or trails[item[0]][1] in HIGH_PRIORITY_REFERENCES)) or ( item[1][1] in HIGH_PRIORITY_REFERENCES and "history" not in item[1][0] ) or any( _ in item[1][0] for _ in HIGH_PRIORITY_INFO_KEYWORDS): trails[item[0]] = item[1] if not results and not any( _ in url for _ in ("abuse.ch", "cobaltstrike")): print( "[x] something went wrong during remote data retrieval ('%s')" % url) except Exception as ex: print( "[x] something went wrong during processing of feed file '%s' ('%s')" % (filename, ex)) try: sys.modules.pop(module.__name__) del module except Exception: pass # custom trails from remote location if config.CUSTOM_TRAILS_URL: print(" [o] '(remote custom)'%s" % (" " * 20)) for url in re.split(r"[;,]", config.CUSTOM_TRAILS_URL): url = url.strip() if not url: continue url = ("http://%s" % url) if "//" not in url else url content = retrieve_content(url) if not content: print( "[x] unable to retrieve data (or empty response) from '%s'" % url) else: __info__ = "blacklisted" __reference__ = "(remote custom)" # urlparse.urlsplit(url).netloc for line in content.split('\n'): line = line.strip() if not line or line.startswith('#'): continue line = re.sub(r"\s*#.*", "", line) if '://' in line: line = re.search(r"://(.*)", line).group(1) line = line.rstrip('/') if line in trails and any( _ in trails[line][1] for _ in ("custom", "static")): continue if '/' in line: trails[line] = (__info__, __reference__) line = line.split('/')[0] elif re.search(r"\A\d+\.\d+\.\d+\.\d+\Z", line): trails[line] = (__info__, __reference__) else: trails[line.strip('.')] = (__info__, __reference__) for match in re.finditer(r"(\d+\.\d+\.\d+\.\d+)/(\d+)", content): prefix, mask = match.groups() mask = int(mask) if mask > 32: continue start_int = addr_to_int(prefix) & make_mask(mask) end_int = start_int | ((1 << 32 - mask) - 1) if 0 <= end_int - start_int <= 1024: address = start_int while start_int <= address <= end_int: trails[int_to_addr(address)] = ( __info__, __reference__) address += 1 print("[i] post-processing trails (this might take a while)...") # basic cleanup for key in list(trails.keys()): if key not in trails: continue if config.DISABLED_TRAILS_INFO_REGEX: if re.search(config.DISABLED_TRAILS_INFO_REGEX, trails[key][0]): del trails[key] continue try: _key = key.decode(UNICODE_ENCODING) if isinstance( key, bytes) else key _key = _key.encode("idna") if six.PY3: _key = _key.decode(UNICODE_ENCODING) if _key != key: # for domains with non-ASCII letters (e.g. phishing) trails[_key] = trails[key] del trails[key] key = _key except: pass if not key or re.search(r"(?i)\A\.?[a-z]+\Z", key) and not any( _ in trails[key][1] for _ in ("custom", "static")): del trails[key] continue if re.search(r"\A\d+\.\d+\.\d+\.\d+\Z", key): if any( _ in trails[key][0] for _ in ("parking site", "sinkhole") ) and key in duplicates: # Note: delete (e.g.) junk custom trails if static trail is a sinkhole del duplicates[key] if trails[key][0] == "malware": trails[key] = ("potential malware site", trails[key][1]) if config.get("IP_MINIMUM_FEEDS", 3) > 1: if (key not in duplicates or len(duplicates[key]) < config.get("IP_MINIMUM_FEEDS", 3) ) and re.search(r"\b(custom|static)\b", trails[key][1]) is None: del trails[key] continue if any(int(_) > 255 for _ in key.split('.')): del trails[key] continue if trails[key][0] == "ransomware": trails[key] = ("ransomware (malware)", trails[key][1]) if key.startswith("www.") and '/' not in key: _ = trails[key] del trails[key] key = key[len("www."):] if key: trails[key] = _ if '?' in key and not key.startswith('/'): _ = trails[key] del trails[key] key = key.split('?')[0] if key: trails[key] = _ if '//' in key: _ = trails[key] del trails[key] key = key.replace('//', '/') trails[key] = _ if key != key.lower(): _ = trails[key] del trails[key] key = key.lower() trails[key] = _ if key in duplicates: _ = trails[key] others = sorted(duplicates[key] - set((_[1], ))) if others and " (+" not in _[1]: trails[key] = (_[0], "%s (+%s)" % (_[1], ','.join(others))) read_whitelist() for key in list(trails.keys()): match = re.search(r"\A(\d+\.\d+\.\d+\.\d+)\b", key) if check_whitelisted(key) or any( key.startswith(_) for _ in BAD_TRAIL_PREFIXES): del trails[key] elif match and (bogon_ip(match.group(1)) or cdn_ip(match.group(1))) and not any( _ in trails[key][0] for _ in ("parking", "sinkhole")): del trails[key] else: try: key.decode("utf8") if hasattr( key, "decode") else key.encode("utf8") trails[key][0].decode("utf8") if hasattr( trails[key][0], "decode") else trails[key][0].encode("utf8") trails[key][1].decode("utf8") if hasattr( trails[key][1], "decode") else trails[key][1].encode("utf8") except UnicodeError: del trails[key] try: if trails: with _fopen(config.TRAILS_FILE, "w+b" if six.PY2 else "w+", open if six.PY2 else codecs.open) as f: writer = csv.writer(f, delimiter=',', quotechar='\"', quoting=csv.QUOTE_MINIMAL) for trail in trails: row = (trail, trails[trail][0], trails[trail][1]) writer.writerow(row) success = True except Exception as ex: print( "[x] something went wrong during trails file write '%s' ('%s')" % (config.TRAILS_FILE, ex)) print("[i] update finished%s" % (40 * " ")) if success: print("[i] trails stored to '%s'" % config.TRAILS_FILE) return trails
if not os.path.isdir(USERS_DIR): os.makedirs(USERS_DIR, 0755) except Exception, ex: exit("[!] something went wrong during creation of directory '%s' ('%s')" % (USERS_DIR, ex)) _chown(USERS_DIR) if config.UPDATE_SERVER: print "[i] retrieving trails from provided 'UPDATE_SERVER' server..." content = retrieve_content(config.UPDATE_SERVER) if not content or content.count(',') < 2: print "[x] unable to retrieve data from '%s'" % config.UPDATE_SERVER else: with _fopen(TRAILS_FILE, "w+b") as f: f.write(content) trails = load_trails() else: trail_files = set() for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(ROOT_DIR, "trails"))) : for filename in filenames: trail_files.add(os.path.abspath(os.path.join(dirpath, filename))) if config.CUSTOM_TRAILS_DIR: for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(ROOT_DIR, os.path.expanduser(config.CUSTOM_TRAILS_DIR)))) : for filename in filenames: trail_files.add(os.path.abspath(os.path.join(dirpath, filename))) if not trails and (force or not os.path.isfile(TRAILS_FILE) or (time.time() - os.stat(TRAILS_FILE).st_mtime) >= config.UPDATE_PERIOD or os.stat(TRAILS_FILE).st_size == 0 or any(os.stat(_).st_mtime > os.stat(TRAILS_FILE).st_mtime for _ in trail_files)): if not config.no_updates: print "[i] updating trails (this might take a while)..."