def update_trails(server=None, force=False):
"""
Update trails from feeds
"""
trails = {}
duplicates = {}
if server:
print "[i] retrieving trails from provided 'UPDATE_SERVER' server..."
_ = retrieve_content(server)
if not _:
exit("[!] unable to retrieve data from '%s'" % server)
else:
with _fopen(TRAILS_FILE, "w+b") as f:
f.write(_)
trails = load_trails()
trail_files = set()
for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(ROOT_DIR, "trails"))) :
for filename in filenames:
trail_files.add(os.path.abspath(os.path.join(dirpath, filename)))
if config.CUSTOM_TRAILS_DIR:
for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(ROOT_DIR, os.path.expanduser(config.CUSTOM_TRAILS_DIR)))) :
for filename in filenames:
trail_files.add(os.path.abspath(os.path.join(dirpath, filename)))
try:
if not os.path.isdir(USERS_DIR):
os.makedirs(USERS_DIR, 0755)
except Exception, ex:
exit("[!] something went wrong during creation of directory '%s' ('%s')" % (USERS_DIR, ex))
def update_timer():
retries = 0
while retries < CHECK_CONNECTION_MAX_RETRIES and not check_connection():
sys.stdout.write("[!] can't update because of lack of network connection (waiting..." if not retries else '.')
sys.stdout.flush()
time.sleep(10)
retries += 1
if retries:
print(")")
if retries == CHECK_CONNECTION_MAX_RETRIES:
print("[x] going to continue without update")
_ = {}
else:
_ = update_trails(server=config.UPDATE_SERVER)
update_ipcat()
if _:
trails.clear()
trails.update(_)
elif not trails:
trails.update(load_trails())
thread = threading.Timer(config.UPDATE_PERIOD, update_timer)
thread.daemon = True
thread.start()
def update_timer():
retries = 0
if not config.no_updates:
while retries < CHECK_CONNECTION_MAX_RETRIES and not check_connection():
sys.stdout.write("[ERROR]: can't update because of lack of Internet connection (waiting..." if not retries else '.')
sys.stdout.flush()
time.sleep(10)
retries += 1
if retries:
sys.stdout.write(")\n")
if config.no_updates or retries == CHECK_CONNECTION_MAX_RETRIES:
if retries == CHECK_CONNECTION_MAX_RETRIES:
logger.error("going to continue without online update")
_ = update_trails(offline=True)
else:
_ = update_trails(server=config.UPDATE_SERVER)
update_ipcat()
if _:
trails.clear()
trails.update(_)
elif not trails:
trails.update(load_trails())
thread = threading.Timer(config.UPDATE_PERIOD, update_timer)
thread.start()
def update_timer():
retries = 0
while retries < CHECK_CONNECTION_MAX_RETRIES and not check_connection():
sys.stdout.write("[!] can't update because of lack of network connection (waiting..." if not retries else '.')
sys.stdout.flush()
time.sleep(10)
retries += 1
if retries:
print(")")
if retries == CHECK_CONNECTION_MAX_RETRIES:
print("[x] going to continue without update")
_ = {}
else:
_ = update_trails(server=config.UPDATE_SERVER)
update_ipcat()
if _:
trails.clear()
trails.update(_)
elif not trails:
trails.update(load_trails())
thread = threading.Timer(config.UPDATE_PERIOD, update_timer)
thread.daemon = True
thread.start()
def update_trails(server=None, force=False):
"""
Update trails from feeds
"""
trails = {}
duplicates = {}
if server:
print "[i] retrieving trails from provided 'UPDATE_SERVER' server..."
_ = retrieve_content(server)
if not _:
exit("[!] unable to retrieve data from '%s'" % server)
else:
with _fopen(TRAILS_FILE, "w+b") as f:
f.write(_)
trails = load_trails()
trail_files = []
for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(ROOT_DIR, "trails"))) :
for filename in filenames:
trail_files.append(os.path.abspath(os.path.join(dirpath, filename)))
if config.CUSTOM_TRAILS_DIR:
for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(ROOT_DIR, os.path.expanduser(config.CUSTOM_TRAILS_DIR)))) :
for filename in filenames:
trail_files.append(os.path.abspath(os.path.join(dirpath, filename)))
try:
if not os.path.isdir(USERS_DIR):
os.makedirs(USERS_DIR, 0755)
except Exception, ex:
exit("[!] something went wrong during creation of directory '%s' ('%s')" % (USERS_DIR, ex))
def update(server=None):
"""
Update trails from feeds
"""
trails = {}
if server:
print "[i] retrieving trails from provided 'UPDATE_SERVER' server..."
_ = retrieve_content(server)
if not _:
print "[!] unable to retrieve data from '%s'" % server
else:
with _fopen_trails("w+b") as f:
f.write(_)
trails = load_trails()
if not trails and (
(not os.path.isfile(TRAILS_FILE) or
(time.time() - os.stat(TRAILS_FILE).st_mtime) >= config.UPDATE_PERIOD
or os.stat(TRAILS_FILE).st_size == 0)):
try:
if not os.path.isdir(USERS_DIRECTORY):
os.makedirs(USERS_DIRECTORY, 0755)
except Exception, ex:
exit(
"[!] something went wrong during creation of directory '%s' ('%s')"
% (USERS_DIRECTORY, ex))
print "[i] updating trails..."
sys.path.append(
os.path.abspath(os.path.join(ROOT_DIR, "trails", "feeds")))
filenames = glob.glob(os.path.join(sys.path[-1], "*.py"))
sys.path.append(os.path.abspath(os.path.join(ROOT_DIR, "trails")))
filenames += [os.path.join(sys.path[-1], "static")]
filenames += [os.path.join(sys.path[-1], "custom")]
for filename in filenames:
try:
module = __import__(os.path.basename(filename).split(".py")[0])
except (ImportError, SyntaxError), ex:
print "[!] something went wrong during import of feed file '%s' ('%s')" % (
filename, ex)
continue
for name, function in inspect.getmembers(module,
inspect.isfunction):
if name == "fetch":
print(" [o] '%s'" % module.__url__)
results = function()
for item in results.items():
if not (any(_ in item[1][0]
for _ in LOW_PRIORITY_INFO_KEYWORDS)
and item[0] in trails):
trails[item[0]] = item[1]
if not results:
print "[!] something went wrong during remote data retrieval ('%s')" % module.__url__
def update_timer():
_ = update(server=config.UPDATE_SERVER)
if _:
trails.clear()
trails.update(_)
elif not trails:
trails.update(load_trails())
thread = threading.Timer(config.UPDATE_PERIOD, update_timer)
thread.daemon = True
thread.start()
def update_timer():
if (time.time() - os.stat(TRAILS_FILE).st_mtime) >= config.UPDATE_PERIOD:
_ = None
while True:
_ = load_trails(True)
if _:
trails.clear()
trails.update(_)
break
else:
time.sleep(LOAD_TRAILS_RETRY_SLEEP_TIME)
threading.Timer(config.UPDATE_PERIOD, update_timer).start()
def update_timer():
_ = update(server=config.UPDATE_SERVER)
if _:
trails.clear()
trails.update(_)
elif not trails:
trails.update(load_trails())
thread = threading.Timer(config.UPDATE_PERIOD, update_timer)
thread.daemon = True
thread.start()
def update_trails():
"""
Update trails from feeds
"""
print datetime.now().strftime('%Y-%m-%d:%H')
trails = TrailsDict()
trails.update(load_trails()) #load trails
list_trails = []
if not (os.path.isfile('trail_info.csv')
): # create a csv for first time of updating
print "can't find collecting_info.csv! then init a new csv"
old = pd.read_csv(TRAILS_CSV, names=['trail', 'info', 'ref'])
old['ftime'] = datetime.now().strftime('%Y-%m-%d:%H')
old['ltime'] = datetime.now().strftime('%Y-%m-%d:%H')
old.to_csv('trail_info.csv', index=False)
else:
old = pd.read_csv('trail_info.csv')
old_trails = set(old.trail)
print "[i] collecting trails information (ftime,ltime)..."
filenames = sorted(glob.glob(os.path.join(FEEDS, "*.py")))
filenames = [_ for _ in filenames if "__init__.py" not in _]
filenames += sorted(glob.glob(os.path.join(
STATIC, "*.py"))) # in static folder, __init__.py has fetch()
filenames += sorted(glob.glob(os.path.join(
CUSTOM, "*.py"))) # in custom folder, __init__.py has fetch()
#remove __init__.py in feeds folder
#filenames = [_ for _ in filenames if "__init__.py" not in _]
#print filenames
init_sys_path()
time = datetime.now().strftime('%Y-%m-%d:%H')
for i in xrange(len(filenames)):
f = filenames[i]
try:
module = __import__(os.path.basename(f).split(".py")[0])
except (ImportError, SyntaxError), ex:
print "[x] Failed: import feed file '%s' ('%s')" % (f, ex)
continue
for name, function in inspect.getmembers(module, inspect.isfunction):
if name == "fetch":
print "[o] '%s'" % (module.__url__)
sys.stdout.write("[?] progress: %d/%d (%d%%)\r" % \
(i, len(filenames), i * 100 / len(filenames)))
sys.stdout.flush()
results = function()
for item in results.items():
list_trails.append(
(item[0], item[1][0], item[1][1], time, time))
def update(server=None):
"""
Update trails from feeds
"""
trails = {}
if server:
print "[i] retrieving trails from provided 'UPDATE_SERVER' server..."
_ = retrieve_content(server)
if not _:
print "[!] unable to retrieve data from '%s'" % server
else:
with _fopen_trails("w+b") as f:
f.write(_)
trails = load_trails()
if not trails and ((not os.path.isfile(TRAILS_FILE) or (time.time() - os.stat(TRAILS_FILE).st_mtime) >= config.UPDATE_PERIOD or os.stat(TRAILS_FILE).st_size == 0)):
try:
if not os.path.isdir(USERS_DIR):
os.makedirs(USERS_DIR, 0755)
except Exception, ex:
exit("[!] something went wrong during creation of directory '%s' ('%s')" % (USERS_DIR, ex))
print "[i] updating trails..."
if config.USE_FEED_UPDATES:
sys.path.append(os.path.abspath(os.path.join(ROOT_DIR, "trails", "feeds")))
filenames = glob.glob(os.path.join(sys.path[-1], "*.py"))
else:
filenames = []
sys.path.append(os.path.abspath(os.path.join(ROOT_DIR, "trails")))
filenames += [os.path.join(sys.path[-1], "static")]
filenames += [os.path.join(sys.path[-1], "custom")]
for filename in filenames:
try:
module = __import__(os.path.basename(filename).split(".py")[0])
except (ImportError, SyntaxError), ex:
print "[!] something went wrong during import of feed file '%s' ('%s')" % (filename, ex)
continue
for name, function in inspect.getmembers(module, inspect.isfunction):
if name == "fetch":
print(" [o] '%s'" % module.__url__)
results = function()
for item in results.items():
if not (any(_ in item[1][0] for _ in LOW_PRIORITY_INFO_KEYWORDS) and item[0] in trails):
trails[item[0]] = item[1]
if not results:
print "[!] something went wrong during remote data retrieval ('%s')" % module.__url__
def update_timer():
retries = 0
if not config.no_updates: # 判断是否设置不更新，然后会利用抓取页面检测网络状态
while retries < CHECK_CONNECTION_MAX_RETRIES and not check_connection(
):
sys.stdout.write(
"[!] can't update because of lack of Internet connection (waiting..."
if not retries else '.')
sys.stdout.flush()
log_error(
"[!] can't update because of lack of Internet connection (waiting...",
"Warning")
time.sleep(10)
retries += 1
if retries:
print(")")
# 超出次数，那么使用update_trails的离线模式
if config.no_updates or retries == CHECK_CONNECTION_MAX_RETRIES:
if retries == CHECK_CONNECTION_MAX_RETRIES:
print("[x] going to continue without online update")
log_error("[x] going to continue without online update", "Warning")
_ = update_trails(offline=True)
else: # 正常进入
_ = update_trails()
# update_ipcat()
# 有新的trails
if _:
trails.clear()
trails.update(_)
elif not trails: # load_trails()只是加载trails()进内存
_ = load_trails()
trails.update(_)
_regex = ""
for trail in trails:
if "static" in trails[trail][1]:
if re.search(r"[\].][*+]|\[[a-z0-9_.\-]+\]", trail, re.I):
try:
re.compile(trail)
except:
pass
else:
if re.escape(trail) != trail:
index = _regex.count("(?P<g")
if index < 100: # Reference: https://stackoverflow.com/questions/478458/python-regular-expressions-with-more-than-100-groups
_regex += "|(?P<g%s>%s)" % (index, trail)
trails._regex = _regex.strip('|')
def update_timer():
# 查看trail.csv的上一次修改时间，差1天更新，其实这里的trails已经在初始化阶段加入了trail.csv的内容。
# **只是利用这个函数来读取trail.csv的新内容**
# 因为只是读，所以没有加锁。所以主线程1天更新一次，多线程采用读来同步trails的状态。
if (time.time() -
os.stat(config.TRAILS_FILE).st_mtime) >= config.UPDATE_PERIOD:
_ = None
while True:
_ = load_trails(True)
if _:
trails.clear()
trails.update(_)
break
else:
time.sleep(LOAD_TRAILS_RETRY_SLEEP_TIME)
# 1天后再次执行
threading.Timer(config.UPDATE_PERIOD, update_timer).start()
def cron_job_load_trails():
'''
重新加载trails到内存
:return:
'''
read_config(CONFIG_FILE)
_ = load_trails()
trails.update(_)
_regex = ""
for trail in trails:
if "static" in trails[trail][1]:
if re.search(r"[\].][*+]|\[[a-z0-9_.\-]+\]", trail, re.I):
try:
re.compile(trail)
except:
pass
else:
if re.escape(trail) != trail:
index = _regex.count("(?P<g")
if index < 100: # Reference: https://stackoverflow.com/questions/478458/python-regular-expressions-with-more-than-100-groups
_regex += "|(?P<g%s>%s)" % (index, trail)
trails._regex = _regex.strip('|')
def update_timer():
first = True
while not check_connection():
sys.stdout.write("[!] can't update because of lack of network connection (waiting..." if first else '.')
sys.stdout.flush()
time.sleep(60)
first = False
if not first:
print(")")
_ = update_trails(server=config.UPDATE_SERVER)
update_ipcat()
if _:
trails.clear()
trails.update(_)
elif not trails:
trails.update(load_trails())
thread = threading.Timer(config.UPDATE_PERIOD, update_timer)
thread.daemon = True
thread.start()
except Exception, ex:
exit(
"[!] something went wrong during creation of directory '%s' ('%s')"
% (USERS_DIR, ex))
_chown(USERS_DIR)
if server:
print "[i] retrieving trails from provided 'UPDATE_SERVER' server..."
content = retrieve_content(server)
if not content:
exit("[!] unable to retrieve data from '%s'" % server)
else:
with _fopen(TRAILS_FILE, "w+b") as f:
f.write(content)
trails = load_trails()
trail_files = set()
for dirpath, dirnames, filenames in os.walk(
os.path.abspath(os.path.join(ROOT_DIR, "trails"))):
for filename in filenames:
trail_files.add(os.path.abspath(os.path.join(dirpath, filename)))
if config.CUSTOM_TRAILS_DIR:
for dirpath, dirnames, filenames in os.walk(
os.path.abspath(
os.path.join(ROOT_DIR,
os.path.expanduser(
config.CUSTOM_TRAILS_DIR)))):
for filename in filenames:
trail_files.add(
def update(server=None):
"""
Update trails from feeds
"""
trails = {}
duplicates = {}
if server:
print "[i] retrieving trails from provided 'UPDATE_SERVER' server..."
_ = retrieve_content(server)
if not _:
print "[!] unable to retrieve data from '%s'" % server
else:
with _fopen_trails("w+b") as f:
f.write(_)
trails = load_trails()
trail_files = []
for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(ROOT_DIR, "trails"))) :
for filename in filenames:
trail_files.append(os.path.abspath(os.path.join(dirpath, filename)))
if config.CUSTOM_TRAILS_DIR:
for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(ROOT_DIR, os.path.expanduser(config.CUSTOM_TRAILS_DIR)))) :
for filename in filenames:
trail_files.append(os.path.abspath(os.path.join(dirpath, filename)))
if not trails and ((not os.path.isfile(TRAILS_FILE) or (time.time() - os.stat(TRAILS_FILE).st_mtime) >= config.UPDATE_PERIOD or os.stat(TRAILS_FILE).st_size == 0 or any(os.stat(_).st_mtime > os.stat(TRAILS_FILE).st_mtime for _ in trail_files))):
try:
if not os.path.isdir(USERS_DIR):
os.makedirs(USERS_DIR, 0755)
except Exception, ex:
exit("[!] something went wrong during creation of directory '%s' ('%s')" % (USERS_DIR, ex))
print "[i] updating trails (this might take a while)..."
if config.USE_FEED_UPDATES:
sys.path.append(os.path.abspath(os.path.join(ROOT_DIR, "trails", "feeds")))
filenames = sorted(glob.glob(os.path.join(sys.path[-1], "*.py")))
else:
filenames = []
sys.path.append(os.path.abspath(os.path.join(ROOT_DIR, "trails")))
filenames += [os.path.join(sys.path[-1], "static")]
filenames += [os.path.join(sys.path[-1], "custom")]
for filename in filenames:
try:
module = __import__(os.path.basename(filename).split(".py")[0])
except (ImportError, SyntaxError), ex:
print "[!] something went wrong during import of feed file '%s' ('%s')" % (filename, ex)
continue
for name, function in inspect.getmembers(module, inspect.isfunction):
if name == "fetch":
print(" [o] '%s'" % module.__url__)
try:
results = function()
for item in results.items():
if item[0] in trails:
if item[0] not in duplicates:
duplicates[item[0]] = set((trails[item[0]][1],))
duplicates[item[0]].add(item[1][1])
if not (item[0] in trails and (any(_ in item[1][0] for _ in LOW_PRIORITY_INFO_KEYWORDS) or trails[item[0]][1] in HIGH_PRIORITY_REFERENCES)) or item[1][1] in HIGH_PRIORITY_REFERENCES:
trails[item[0]] = item[1]
if not results and "abuse.ch" not in module.__url__:
print "[!] something went wrong during remote data retrieval ('%s')" % module.__url__
except Exception, ex:
print "[!] something went wrong during processing of feed file '%s' ('%s')" % (filename, ex)
def update_trails():
"""
Update trails from feeds
"""
print datetime.now().strftime('%Y-%m-%d:%H')
trails = TrailsDict()
trails.update(load_trails()) #load trails
'''old=pd.read_csv(TRAILS_CSV,names=['trail', 'info', 'ref'])
old['ftime']=datetime.now().strftime('%Y-%m-%d:%H')
old['ltime']=datetime.now().strftime('%Y-%m-%d:%H')
old.to_csv('old_info.csv',index_label="id")
exit(1)'''
list_trails = []
if not (os.path.isfile('trail_info.csv')
): # create a csv for first time of updating
print "can't find collecting_info.csv!"
old = pd.read_csv(TRAILS_CSV, names=['trail', 'info', 'ref'])
old['ftime'] = datetime.now().strftime('%Y-%m-%d:%H')
old['ltime'] = datetime.now().strftime('%Y-%m-%d:%H')
old.to_csv('trail_info.csv', index_label="id")
else:
old = pd.read_csv('trail_info.csv')
old_trails = set(old.trail)
print "[i] updating trails (this might take a while)..."
filenames = sorted(glob.glob(os.path.join(FEEDS, "*.py")))
filenames += [STATIC] # in static folder, __init__.py has fetch()
filenames += [CUSTOM] # in custom folder, __init__.py has fetch()
# remove __init__.py in feeds folder
filenames = [_ for _ in filenames if "__init__.py" not in _]
init_sys_path()
for i in xrange(len(filenames)):
f = filenames[i]
try:
module = __import__(os.path.basename(f).split(".py")[0])
except (ImportError, SyntaxError), ex:
print "[x] Failed: import feed file '%s' ('%s')" % (f, ex)
continue
for name, function in inspect.getmembers(module, inspect.isfunction):
if name == "fetch":
print "[o] '%s'" % (module.__url__)
sys.stdout.write("[?] progress: %d/%d (%d%%)\r" % \
(i, len(filenames), i * 100 / len(filenames)))
sys.stdout.flush()
try:
results = function()
#print(1)
for item in results.items():
list_trails.append(
(item[0], item[1][0], item[1][1],
datetime.now().strftime('%Y-%m-%d:%H'),
datetime.now().strftime('%Y-%m-%d:%H')))
'''if(item[0] in trails):
tmp=str(old[old.trail == item[0]].ftime)
list_trails.append((item[0], item[1][0],item[1][1],tmp[5:18],time))
#print(1)
else:
list_trails.append((item[0], item[1][0],item[1][1],time,time))
'''
except Exception, ex:
print "[x] Failed: process feed file '%s' ('%s')" % (
filename, ex)
from core.settings import trails
import sys
from core.common import load_trails
trails.update(load_trails())
'''
trails contains blacklisted strings: domains and Ips
e.g.
mqbbsagabardinedazyx.com
dgj4gu1xmithip.net
moegestnessbiophysicalohax.com
rhmxancorml.com
115.28.7.221
jjcwgfwdyqje.pw
'''
for line in sys.stdin:
try:
line = line.strip()
data = line.split(',')
if data[5] in trails or data[9] in trails:
print str(data[6]) + "," + str(100)
else:
print str(data[6]) + "," + str(0)
except:
print line
from core.settings import trails
import sys
from core.common import load_trails
trails.update(load_trails())
'''
trails contains blacklisted strings: domains and Ips
e.g.
mqbbsagabardinedazyx.com
dgj4gu1xmithip.net
moegestnessbiophysicalohax.com
rhmxancorml.com
115.28.7.221
jjcwgfwdyqje.pw
'''
for line in sys.stdin:
try:
line = line.strip()
data = line.split(',')
if data[5] in trails or data[9] in trails:
print str(data[6])+","+str(100)
else:
print str(data[6])+","+str(0)
except:
print line
def update_trails(force=False, offline=False):
"""
Update trails from feeds
"""
success = False
trails = TrailsDict()
duplicates = {}
try:
if not os.path.isdir(USERS_DIR):
os.makedirs(USERS_DIR, 0o755)
except Exception as ex:
exit(
"[!] something went wrong during creation of directory '%s' ('%s')"
% (USERS_DIR, ex))
_chown(USERS_DIR)
if config.UPDATE_SERVER:
print("[i] retrieving trails from provided 'UPDATE_SERVER' server...")
content = retrieve_content(config.UPDATE_SERVER)
if not content or content.count(',') < 2:
print("[x] unable to retrieve data from '%s'" %
config.UPDATE_SERVER)
else:
with _fopen(config.TRAILS_FILE, "w+b" if six.PY2 else "w+",
open if six.PY2 else codecs.open) as f:
f.write(content)
trails = load_trails()
else:
trail_files = set()
for dirpath, dirnames, filenames in os.walk(
os.path.abspath(os.path.join(ROOT_DIR, "trails"))):
for filename in filenames:
trail_files.add(
os.path.abspath(os.path.join(dirpath, filename)))
if config.CUSTOM_TRAILS_DIR:
for dirpath, dirnames, filenames in os.walk(
os.path.abspath(
os.path.join(
ROOT_DIR,
os.path.expanduser(config.CUSTOM_TRAILS_DIR)))):
for filename in filenames:
trail_files.add(
os.path.abspath(os.path.join(dirpath, filename)))
if not trails and (
force or not os.path.isfile(config.TRAILS_FILE) or
(time.time() - os.stat(config.TRAILS_FILE).st_mtime) >=
config.UPDATE_PERIOD
or os.stat(config.TRAILS_FILE).st_size == 0 or any(
os.stat(_).st_mtime > os.stat(config.TRAILS_FILE).st_mtime
for _ in trail_files)):
if not config.no_updates:
print("[i] updating trails (this might take a while)...")
else:
print("[i] checking trails...")
if not offline and (force or config.USE_FEED_UPDATES):
_ = os.path.abspath(os.path.join(ROOT_DIR, "trails", "feeds"))
if _ not in sys.path:
sys.path.append(_)
filenames = sorted(glob.glob(os.path.join(_, "*.py")))
else:
filenames = []
_ = os.path.abspath(os.path.join(ROOT_DIR, "trails"))
if _ not in sys.path:
sys.path.append(_)
filenames += [os.path.join(_, "custom")]
filenames += [
os.path.join(_, "static")
] # Note: higher priority than previous one because of dummy user trails (FE)
filenames = [_ for _ in filenames if "__init__.py" not in _]
if config.DISABLED_FEEDS:
filenames = [
filename for filename in filenames
if os.path.splitext(os.path.split(filename)[-1])[0] not in
re.split(r"[^\w]+", config.DISABLED_FEEDS)
]
for i in xrange(len(filenames)):
filename = filenames[i]
try:
module = __import__(
os.path.basename(filename).split(".py")[0])
except (ImportError, SyntaxError) as ex:
print(
"[x] something went wrong during import of feed file '%s' ('%s')"
% (filename, ex))
continue
for name, function in inspect.getmembers(
module, inspect.isfunction):
if name == "fetch":
url = module.__url__ # Note: to prevent "SyntaxError: can not delete variable 'module' referenced in nested scope"
print(" [o] '%s'%s" %
(url, " " * 20 if len(url) < 20 else ""))
sys.stdout.write(
"[?] progress: %d/%d (%d%%)\r" %
(i, len(filenames), i * 100 // len(filenames)))
sys.stdout.flush()
if config.DISABLED_TRAILS_INFO_REGEX and re.search(
config.DISABLED_TRAILS_INFO_REGEX,
getattr(module, "__info__", "")):
continue
try:
results = function()
for item in results.items():
if item[0].startswith(
"www.") and '/' not in item[0]:
item = [item[0][len("www."):], item[1]]
if item[0] in trails:
if item[0] not in duplicates:
duplicates[item[0]] = set(
(trails[item[0]][1], ))
duplicates[item[0]].add(item[1][1])
if not (
item[0] in trails and
(any(_ in item[1][0]
for _ in LOW_PRIORITY_INFO_KEYWORDS)
or trails[item[0]][1]
in HIGH_PRIORITY_REFERENCES)) or (
item[1][1] in HIGH_PRIORITY_REFERENCES
and "history" not in item[1][0]
) or any(
_ in item[1][0]
for _ in HIGH_PRIORITY_INFO_KEYWORDS):
trails[item[0]] = item[1]
if not results and not any(
_ in url
for _ in ("abuse.ch", "cobaltstrike")):
print(
"[x] something went wrong during remote data retrieval ('%s')"
% url)
except Exception as ex:
print(
"[x] something went wrong during processing of feed file '%s' ('%s')"
% (filename, ex))
try:
sys.modules.pop(module.__name__)
del module
except Exception:
pass
# custom trails from remote location
if config.CUSTOM_TRAILS_URL:
print(" [o] '(remote custom)'%s" % (" " * 20))
for url in re.split(r"[;,]", config.CUSTOM_TRAILS_URL):
url = url.strip()
if not url:
continue
url = ("http://%s" % url) if "//" not in url else url
content = retrieve_content(url)
if not content:
print(
"[x] unable to retrieve data (or empty response) from '%s'"
% url)
else:
__info__ = "blacklisted"
__reference__ = "(remote custom)" # urlparse.urlsplit(url).netloc
for line in content.split('\n'):
line = line.strip()
if not line or line.startswith('#'):
continue
line = re.sub(r"\s*#.*", "", line)
if '://' in line:
line = re.search(r"://(.*)", line).group(1)
line = line.rstrip('/')
if line in trails and any(
_ in trails[line][1]
for _ in ("custom", "static")):
continue
if '/' in line:
trails[line] = (__info__, __reference__)
line = line.split('/')[0]
elif re.search(r"\A\d+\.\d+\.\d+\.\d+\Z", line):
trails[line] = (__info__, __reference__)
else:
trails[line.strip('.')] = (__info__,
__reference__)
for match in re.finditer(r"(\d+\.\d+\.\d+\.\d+)/(\d+)",
content):
prefix, mask = match.groups()
mask = int(mask)
if mask > 32:
continue
start_int = addr_to_int(prefix) & make_mask(mask)
end_int = start_int | ((1 << 32 - mask) - 1)
if 0 <= end_int - start_int <= 1024:
address = start_int
while start_int <= address <= end_int:
trails[int_to_addr(address)] = (
__info__, __reference__)
address += 1
print("[i] post-processing trails (this might take a while)...")
# basic cleanup
for key in list(trails.keys()):
if key not in trails:
continue
if config.DISABLED_TRAILS_INFO_REGEX:
if re.search(config.DISABLED_TRAILS_INFO_REGEX,
trails[key][0]):
del trails[key]
continue
try:
_key = key.decode(UNICODE_ENCODING) if isinstance(
key, bytes) else key
_key = _key.encode("idna")
if six.PY3:
_key = _key.decode(UNICODE_ENCODING)
if _key != key: # for domains with non-ASCII letters (e.g. phishing)
trails[_key] = trails[key]
del trails[key]
key = _key
except:
pass
if not key or re.search(r"(?i)\A\.?[a-z]+\Z", key) and not any(
_ in trails[key][1] for _ in ("custom", "static")):
del trails[key]
continue
if re.search(r"\A\d+\.\d+\.\d+\.\d+\Z", key):
if any(
_ in trails[key][0]
for _ in ("parking site", "sinkhole")
) and key in duplicates: # Note: delete (e.g.) junk custom trails if static trail is a sinkhole
del duplicates[key]
if trails[key][0] == "malware":
trails[key] = ("potential malware site",
trails[key][1])
if config.get("IP_MINIMUM_FEEDS", 3) > 1:
if (key not in duplicates or len(duplicates[key]) <
config.get("IP_MINIMUM_FEEDS", 3)
) and re.search(r"\b(custom|static)\b",
trails[key][1]) is None:
del trails[key]
continue
if any(int(_) > 255 for _ in key.split('.')):
del trails[key]
continue
if trails[key][0] == "ransomware":
trails[key] = ("ransomware (malware)", trails[key][1])
if key.startswith("www.") and '/' not in key:
_ = trails[key]
del trails[key]
key = key[len("www."):]
if key:
trails[key] = _
if '?' in key and not key.startswith('/'):
_ = trails[key]
del trails[key]
key = key.split('?')[0]
if key:
trails[key] = _
if '//' in key:
_ = trails[key]
del trails[key]
key = key.replace('//', '/')
trails[key] = _
if key != key.lower():
_ = trails[key]
del trails[key]
key = key.lower()
trails[key] = _
if key in duplicates:
_ = trails[key]
others = sorted(duplicates[key] - set((_[1], )))
if others and " (+" not in _[1]:
trails[key] = (_[0],
"%s (+%s)" % (_[1], ','.join(others)))
read_whitelist()
for key in list(trails.keys()):
match = re.search(r"\A(\d+\.\d+\.\d+\.\d+)\b", key)
if check_whitelisted(key) or any(
key.startswith(_) for _ in BAD_TRAIL_PREFIXES):
del trails[key]
elif match and (bogon_ip(match.group(1))
or cdn_ip(match.group(1))) and not any(
_ in trails[key][0]
for _ in ("parking", "sinkhole")):
del trails[key]
else:
try:
key.decode("utf8") if hasattr(
key, "decode") else key.encode("utf8")
trails[key][0].decode("utf8") if hasattr(
trails[key][0],
"decode") else trails[key][0].encode("utf8")
trails[key][1].decode("utf8") if hasattr(
trails[key][1],
"decode") else trails[key][1].encode("utf8")
except UnicodeError:
del trails[key]
try:
if trails:
with _fopen(config.TRAILS_FILE, "w+b" if six.PY2 else "w+",
open if six.PY2 else codecs.open) as f:
writer = csv.writer(f,
delimiter=',',
quotechar='\"',
quoting=csv.QUOTE_MINIMAL)
for trail in trails:
row = (trail, trails[trail][0], trails[trail][1])
writer.writerow(row)
success = True
except Exception as ex:
print(
"[x] something went wrong during trails file write '%s' ('%s')"
% (config.TRAILS_FILE, ex))
print("[i] update finished%s" % (40 * " "))
if success:
print("[i] trails stored to '%s'" % config.TRAILS_FILE)
return trails
if not os.path.isdir(USERS_DIR):
os.makedirs(USERS_DIR, 0755)
except Exception, ex:
exit("[!] something went wrong during creation of directory '%s' ('%s')" % (USERS_DIR, ex))
_chown(USERS_DIR)
if config.UPDATE_SERVER:
print "[i] retrieving trails from provided 'UPDATE_SERVER' server..."
content = retrieve_content(config.UPDATE_SERVER)
if not content or content.count(',') < 2:
print "[x] unable to retrieve data from '%s'" % config.UPDATE_SERVER
else:
with _fopen(TRAILS_FILE, "w+b") as f:
f.write(content)
trails = load_trails()
else:
trail_files = set()
for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(ROOT_DIR, "trails"))) :
for filename in filenames:
trail_files.add(os.path.abspath(os.path.join(dirpath, filename)))
if config.CUSTOM_TRAILS_DIR:
for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(ROOT_DIR, os.path.expanduser(config.CUSTOM_TRAILS_DIR)))) :
for filename in filenames:
trail_files.add(os.path.abspath(os.path.join(dirpath, filename)))
if not trails and (force or not os.path.isfile(TRAILS_FILE) or (time.time() - os.stat(TRAILS_FILE).st_mtime) >= config.UPDATE_PERIOD or os.stat(TRAILS_FILE).st_size == 0 or any(os.stat(_).st_mtime > os.stat(TRAILS_FILE).st_mtime for _ in trail_files)):
if not config.no_updates:
print "[i] updating trails (this might take a while)..."