def setUpClass(self):
self.addr = "http://localhost:8200"
self.token = "root"
os.environ['VAULT_ADDR'] = self.addr
os.environ['VAULT_TOKEN'] = self.token
self.v = Vault(self.addr, self.token)
# Setup docker
self.container = docker.from_env().containers.run(
'vault',
'vault server -dev -dev-root-token-id="root" -dev-listen-address="0.0.0.0:8200"',
detach=True,
remove=True,
name='vault',
ports={'8200': '8200'})
# Wait a second for docker
# Change to the provisioner directory and run it
time.sleep(1)
p = subprocess.Popen(
["python", "vault_api_provisioner.py", "--log=Debug"],
cwd="vault-api-provisioner")
p.wait()
def add_stream_entry(stream_key, stream, stream_payload, yara_flagged,
timestamp):
"""
Organized flagging for yara_process.py scanning of
- Streams
- Payload Data
"""
flagged_dict = {}
flag_matches = []
for match in yara_flagged[0].strings:
try:
flag_matches.append(match[2].decode('utf-8'))
except:
flag_matches.append(match[2])
flagged_details = PayloadFlagged(yara_flagged, timestamp, stream_key,
stream, stream_payload,
yara_flagged[0].strings,
yara_flagged[0].rule,
yara_flagged[0].tags)
Organize.packet_counter += 1
flagged_dict[str(Organize.packet_counter)] = flagged_details
Vault.set_flagged(flagged_dict)
logger.info(
f"Payload: {stream_key} --> {yara_flagged[0].rule} [{Thread.name()}]"
)
def main():
"""Entrypoint when used as an executable
:returns: None
"""
# Initialize Logging
logging.basicConfig(level=logging.DEBUG)
args = get_commandline_arguments()
init_logging(args)
vault = Vault(os.environ["VAULT_ADDR"], args.token)
secrets = vault.recursive_list(args.engine, args.vaultpath)
path_depth = 0
ul_count = 0
for secret in secrets:
new_path_depth = secret.count("/")
if not secret.endswith("/"):
new_path_depth = new_path_depth + 1
if new_path_depth > path_depth:
print("<ul>")
ul_count = ul_count + 1
if new_path_depth < path_depth:
diff = path_depth - new_path_depth
for _ in range(diff):
print("</ul>")
ul_count = ul_count - diff
path_depth = new_path_depth
ui_link = vault.path_to_ui_link(args.engine, secret)
print(html_list_element(html_link(secret, ui_link)))
for _ in range(ul_count):
print("</ul>")
def cmd_set(vfname: str, vpass: bytes, salt: bytes, v: vault.Vault) -> None:
"""Create or update vault entry."""
try:
d = {}
d['credname'] = input(f'{"credential:":<20}')
d['username'] = input(f'{"username:"******"password:"******"password:"******"description:":<20}')
except EOFError:
print('\ncancelled')
credname = d['credname']
if credname in v.list():
print(f'\nreplacing: {v.get(credname)}\n')
v.set(**d)
with open(vfname, 'wb') as fp:
util.save_vault(fp, vpass, salt, v)
def carve_stream(k, timestamp, cont_type, cont_length, stream_payload):
"""
Main carving function to carve out files from packet streams' payloads
"""
magic = ""
k = k.replace(" ", "_").replace(":", "-")
bytes_content = BytesIO(stream_payload)
for file_type, magic_bytes in Carver.file_sigs.items():
if file_type == cont_type:
magic = magic_bytes
break
if cont_type and cont_length and magic:
sof = Carver.get_sof(magic, bytes_content)
if sof is None:
return 0
eof = sof + cont_length
view = bytes_content.getbuffer()
carved = view[sof:eof]
with open(
f"{CARVED_DIR}{(fname := Carver.random_str(5))}.{cont_type}",
'ab+') as file_obj:
file_obj.write(carved)
logger.info(
f"File {fname}.{cont_type} carved ({cont_length} bytes) [{Thread.name()}]"
)
Vault.add_carved_file(k, timestamp, f"{fname}.{cont_type}",
cont_length)
return carved
def index():
''' If method == GET , returns HTML Page
If method == POST, returns data for protocl piechart
'''
if request.method != "POST":
return render_template("index.html", status=Vault.get_saving())
global COMMON_PROTOCOLS
protocol_dict = {}
for header in Vault.get_session_headers():
if 'TCP' in header or 'UDP' in header:
header_list = header[4:].replace('_', '-').split('-')
if header_list[1] in COMMON_PROTOCOLS or header_list[
3] in COMMON_PROTOCOLS:
try:
prot = COMMON_PROTOCOLS[header_list[1]]
except Exception:
prot = COMMON_PROTOCOLS[header_list[3]]
protocol_dict[prot] = protocol_dict[
prot] + 1 if prot in protocol_dict else 1
else:
protocol_dict["Other"] = protocol_dict[
"Other"] + 1 if "Other" in protocol_dict else 1
else:
protocol_dict["Other"] = protocol_dict[
"Other"] + 1 if "Other" in protocol_dict else 1
return jsonify(protocol_dict)
def view_arp():
'''returns arp page'''
arp_sessions = [
session for session in Vault.get_session_headers() if 'ARP' in session
]
return render_template("viewarp.html",
arp_sessions=arp_sessions,
status=Vault.get_saving())
def env_file_from_vault(self):
service_envs = Vault().get_service_envs(self.swarm_stack, self.service)
if service_envs:
out_file = open(self.env_filename, 'w')
for key, value in service_envs.items():
out_file.write(f'{key}={value}\n')
out_file.close()
print('[INFO] Creating env file')
def process(self, message, tags, author='rabot32'):
if self.tag_types['message'] in tags:
comms = Comms()
if comms.direct_message(self.message_recipient, message) is True:
# If the message failed to send, then no 'messaged' tag will be added
tags.append(self.tag_types['messaged'])
if self.tag_types['store'] in tags:
vault = Vault()
vault.store(message, tags=['store'], author=author)
def log_on(self):
""" Logs on to a website, using an url.
First checks if the channel requires log on. If so and it's not already
logged on, it should handle the log on. That part should be implemented
by the specific channel.
More arguments can be passed on, but must be handled by custom code.
After a successful log on the self.loggedOn property is set to True and
True is returned.
:return: indication if the login was successful.
:rtype: bool
"""
if self.__idToken:
return True
# check if there is a refresh token
# refresh token: viervijfzes_refresh_token
refresh_token = AddonSettings.get_setting("viervijfzes_refresh_token")
client = AwsIdp("eu-west-1_dViSsKM5Y",
"6s1h851s8uplco5h6mqh1jac8m",
proxy=self.proxy,
logger=Logger.instance())
if refresh_token:
id_token = client.renew_token(refresh_token)
if id_token:
self.__idToken = id_token
return True
else:
Logger.info("Extending token for VierVijfZes failed.")
# username: viervijfzes_username
username = AddonSettings.get_setting("viervijfzes_username")
# password: viervijfzes_password
v = Vault()
password = v.get_setting("viervijfzes_password")
if not username or not password:
XbmcWrapper.show_dialog(
title=None,
lines=LanguageHelper.get_localized_string(
LanguageHelper.MissingCredentials),
)
return False
id_token, refresh_token = client.authenticate(username, password)
if not id_token or not refresh_token:
Logger.error("Error getting a new token. Wrong password?")
return False
self.__idToken = id_token
AddonSettings.set_setting("viervijfzes_refresh_token", refresh_token)
return True
def cmd_remove(vfname: str, vpass: bytes, salt: bytes, v: vault.Vault,
credname: str) -> None:
"""Remove vault entry."""
try:
print(f'\nremoving: {v.get(credname)}\n')
v.remove(credname)
with open(vfname, 'wb') as fp:
util.save_vault(fp, vpass, salt, v)
except KeyError:
print('credential not found')
def test_vault(self):
vault = Vault('password', ':test:')
vault.add_entry('TestArgon2', 'username2', 'password')
print(vault.get_entries())
del vault
vault2 = Vault('password', 'test.db.enc')
vault2.add_entry('Test1', 'username', 'password3')
print(vault2.get_entries())
def get_data():
''' repeatedly send updated packet count data to socket'''
while not thread_stop_event.isSet():
socketio.emit("data", {
"total_packets": Vault.get_total_packet_count(),
"total_streams": len(Vault.get_session_headers()),
"total_flagged": len(Vault.get_flagged())
},
namespace="/socket")
socketio.sleep(0.01)
def cmd_list(v: vault.Vault) -> None:
"""List vault contents."""
print('\nVault contents')
print('--------------')
for k in v.list():
cred = v.get(k)
age = (datetime.datetime.utcnow() -
vault.parse_dt(cred['modified'])).days
print(
f"{k:<20} - u={cred['username']:<30} d={cred['description']:<40} ({age} days)"
)
print('--------------')
def cmd_search(v: vault.Vault, substr: str) -> None:
"""Search vault credential names."""
print('\nSearch results')
print('---------------')
for k in v.search(substr):
cred = v.get(k)
age = (datetime.datetime.utcnow() -
vault.parse_dt(cred['modified'])).days
print(
f"{k:<20} - u={cred['username']:<30} d={cred['description']:<40} ({age} days)"
)
print('---------------')
def main():
"""Entrypoint when used as an executable
:returns: None
"""
# Initialize Logging
logging.basicConfig(level=logging.DEBUG)
args = get_commandline_arguments()
init_logging(args)
vault = Vault(os.environ["VAULT_ADDR"], args.token)
vault.del_userpass_user(args.user)
def main():
"""Entrypoint when used as an executable
:returns: None
"""
# Initialize Logging
logging.basicConfig(level=logging.DEBUG)
args = get_commandline_arguments()
init_logging(args)
vault = Vault(os.environ["VAULT_ADDR"], args.token)
password = vault.add_user(args.firstname, args.lastname)
token = vault.wrap({"password": password})
unwrap = vault.unwrap_str(token)
print(unwrap)
def LogOn(self):
signatureSettings = "mediaan_signature"
signatureSetting = AddonSettings.GetSetting(signatureSettings)
# apiKey = "3_HZ0FtkMW_gOyKlqQzW5_0FHRC7Nd5XpXJZcDdXY4pk5eES2ZWmejRW5egwVm4ug-" # from VTM
apiKey = "3_OEz9nzakKMkhPdUnz41EqSRfhJg5z9JXvS4wUORkqNf2M2c1wS81ilBgCewkot97" # from Stievie
if signatureSetting and "|" not in signatureSetting:
url = "https://accounts.eu1.gigya.com/accounts.getAccountInfo"
data = "APIKey=%s" \
"&sdk=js_7.4.30" \
"&login_token=%s" % (apiKey, signatureSetting, )
logonData = UriHandler.Open(url,
params=data,
proxy=self.proxy,
noCache=True)
if self.__ExtractSessionData(logonData, signatureSettings):
return True
Logger.Warning("Failed to extend the VTM.be session.")
Logger.Info("Logging onto VTM.be")
v = Vault()
password = v.GetSetting("mediaan_password")
username = AddonSettings.GetSetting("mediaan_username")
if not username or not password:
XbmcWrapper.ShowDialog(
title=None,
lines=LanguageHelper.GetLocalizedString(
LanguageHelper.MissingCredentials),
# notificationType=XbmcWrapper.Error,
# displayTime=5000
)
return False
Logger.Debug("Using: %s / %s", username, "*" * len(password))
url = "https://accounts.eu1.gigya.com/accounts.login"
data = "loginID=%s" \
"&password=%s" \
"&targetEnv=jssdk" \
"&APIKey=%s" \
"&includeSSOToken=true" \
"&authMode=cookie" % \
(HtmlEntityHelper.UrlEncode(username), HtmlEntityHelper.UrlEncode(password), apiKey)
logonData = UriHandler.Open(url,
params=data,
proxy=self.proxy,
noCache=True)
return self.__ExtractSessionData(logonData, signatureSettings)
def setUp(self): # Cleanup after any failures from previous runs noexcept(fs.Unmount(self.path()), important=False) noexcept(crypt.Close(self.path()), important=False) noexcept(loopdev.Close(self.path()), important=False) shutil.rmtree(self.dir()) self.v = Vault(self.path())
def view_udp():
'''returns UDP stream page'''
udp_sessions = get_formatted_header('UDP')
payload = None
return render_template("viewudp.html",
udp_sessions=udp_sessions,
payload=payload,
status=Vault.get_saving())
def getSSHKey(config):
try:
vault=Vault(token=config.get('vault').get('token'),type=config.get('vault').get('type'),vault_url=config.get('vault').get('url'))
vault.auth()
out=vault.get(config.get('vault').get('keypath'))
pemfile='/tmp/'+str(uuid.uuid4())
print pemfile
fp=open(pemfile,'w')
print out
for line in out.get('data').get('value').split('\n'):
fp.write(line+'\n')
fp.close()
os.system('chmod 600 '+pemfile)
return pemfile
except Exception,e:
print e.args
logger.info(e)
def view_tcp():
'''returns TCP stream page'''
tcp_sessions = get_formatted_header('TCP')
payload = None
return render_template("viewtcp.html",
tcp_sessions=tcp_sessions,
payload=payload,
status=Vault.get_saving())
def cmd_get(v: vault.Vault, credname: str) -> None:
"""Get vault entry."""
try:
cred = v.get(credname)
print(
f"{credname:<20} - u={cred['username']} p={cred['password']} d={cred['description']}"
)
except KeyError:
print('\ncredential not found\n')
def logs():
''' If method == GET , returns logs page
If method == POST, returns content of selected rule file
'''
if request.method == "POST":
message = Util.tail(LOG_FILE, 20)
return message
return render_template("logs.html", status=Vault.get_saving())
def network():
''' If method == GET , returns network mapping page
If method == POST, returns mapping_dict,ip_list,mal_list for network mapping
'''
if request.method == "POST":
mal_list = []
mapping, ip_list = Vault.get_mapping()
flagged_dict = Vault.get_flagged()
for _, obj in flagged_dict.items():
if 'endpoint' in obj.identifier:
for match in obj.strings:
mal_list.append(match[2].decode('utf-8'))
return jsonify(mapping, ip_list, mal_list)
return render_template("network.html",
status=Vault.get_saving(),
data=Vault.get_mapping())
def bulk_manager(event):
"""
Manages session_yara() and threat()
"""
while not Thread.get_interrupt():
temp_plist = Vault.get_threading_plist()
logger.info(f"{len(temp_plist)} packets processed [{Thread.name()}]")
stream_dict = find_streams(temp_plist)
Vault.add_session(stream_dict)
session_yara(stream_dict)
threat(temp_plist)
all_sessions = Vault.get_session_headers()
logger.info(f"{len(all_sessions)} total sessions [{Thread.name()}]")
event.wait(timeout=BULK_MANAGER_INTERVAL)
logger.info(f"Terminated [{Thread.name()}]")
def main():
"""
Main method to control the program flow
"""
# set runtime name
Vault.set_runtime_name(Util.datetime_to_string())
# create runtime directory
mkdir(f"{SESSION_CACHE_PATH}/{Vault.get_runtime_name()}")
# set up threading
Thread.set_interrupt(False)
event = threading.Event()
# start threads
manager_thread = manager(event)
Escapy.async_sniff(custom_action)
# menu
info_data = [
f"{RED}Sniffer is running but not saving anything locally{RESET}",
f"{GREEN}Sniffer is running saving packets locally{RESET}"
]
option = [
"Type \"start\" to start saving: ", "Type \"stop\" to stop saving: "
]
while True:
print(info_data[0 if not Vault.get_saving() else 1], end="\n")
print(
f"Dashboard: {YELLOW}http://127.0.0.1:8000{RESET} | \'q\' to stop")
user_input = input(option[0 if not Vault.get_saving() else 1])
if not Vault.get_saving() and user_input == "start":
Util.start_saving()
elif Vault.get_saving() and user_input == "stop":
Util.stop_saving()
elif user_input == "q":
break
else:
print(f"{YELLOW}Invalid option{RESET}", end="\n\n")
# SAVE TO FILE IF PROGRAM ENDED AND SAVING IS TRUE
if Vault.get_saving():
Util.stop_saving()
# interrupt threads
Thread.set_interrupt(True)
event.set()
Escapy.stop()
# wait for threads to complete
manager_thread.join()
def LogOn(self):
if self.__idToken:
return True
# check if there is a refresh token
# refresh token: viervijfzes_refresh_token
refreshToken = AddonSettings.GetSetting("viervijfzes_refresh_token")
client = AwsIdp("eu-west-1_dViSsKM5Y",
"6s1h851s8uplco5h6mqh1jac8m",
proxy=self.proxy,
logger=Logger.Instance())
if refreshToken:
idToken = client.RenewToken(refreshToken)
if idToken:
self.__idToken = idToken
return True
else:
Logger.Info("Extending token for VierVijfZes failed.")
# username: viervijfzes_username
username = AddonSettings.GetSetting("viervijfzes_username")
# password: viervijfzes_password
v = Vault()
password = v.GetSetting("viervijfzes_password")
if not username or not password:
XbmcWrapper.ShowDialog(
title=None,
lines=LanguageHelper.GetLocalizedString(
LanguageHelper.MissingCredentials),
)
return False
idToken, refreshToken = client.Authenticate(username, password)
if not idToken or not refreshToken:
Logger.Error("Error getting a new token. Wrong password?")
return False
self.__idToken = idToken
AddonSettings.SetSetting("viervijfzes_refresh_token", refreshToken)
return True
def flagged():
''' If method == GET , returns flagged page
If method == POST, returns payload of flagged packet
'''
if request.method == "POST":
key = request.json["data"].strip()
flagged_dict = Vault.get_flagged()
flagged_obj = flagged_dict[key]
if flagged_obj.identifier == "payload":
return flagged_obj.payload
strings_list = []
for i in range(len(flagged_obj.strings)):
strings_list.append(
(flagged_obj.strings[i][0], flagged_obj.strings[i][0],
flagged_obj.strings[i][2].decode('utf-8')))
return jsonify(strings_list)
return render_template("flagged.html",
flagged_packets=Vault.get_flagged(),
status=Vault.get_saving())
def LogOn(self):
tokenCookie = UriHandler.GetCookie("X-VRT-Token", ".vrt.be")
if tokenCookie is not None:
return True
username = self._GetSetting("username")
if not username:
return None
v = Vault()
password = v.GetChannelSetting(self.guid, "password")
if not password:
Logger.Warning("Found empty password for VRT user")
Logger.Debug("Using: %s / %s", username, "*" * len(password))
url = "https://accounts.eu1.gigya.com/accounts.login"
data = "loginID=%s" \
"&password=%s" \
"&targetEnv=jssdk" \
"&APIKey=3_qhEcPa5JGFROVwu5SWKqJ4mVOIkwlFNMSKwzPDAh8QZOtHqu6L4nD5Q7lk0eXOOG" \
"&includeSSOToken=true" \
"&authMode=cookie" % \
(HtmlEntityHelper.UrlEncode(username), HtmlEntityHelper.UrlEncode(password))
logonData = UriHandler.Open(url, params=data, proxy=self.proxy, noCache=True)
sig, uid, timestamp = self.__ExtractSessionData(logonData)
if sig is None and uid is None and timestamp is None:
return False
url = "https://token.vrt.be/"
tokenData = '{"uid": "%s", ' \
'"uidsig": "%s", ' \
'"ts": "%s", ' \
'"fn": "VRT", "ln": "NU", ' \
'"email": "%s"}' % (uid, sig, timestamp, username)
headers = {"Content-Type": "application/json", "Referer": "https://www.vrt.be/vrtnu/"}
UriHandler.Open(url, params=tokenData, proxy=self.proxy, additionalHeaders=headers)
return True
def savefile():
''' returns viewfile page with all the saved files '''
pcap_files = [
f for f in listdir(CAP_PATH) if isfile(join(CAP_PATH, f))
if f[-4:] == ".cap"
]
carved_files = [
f for f in listdir(CARVED_DIR) if isfile(join(CARVED_DIR, f))
]
return render_template("viewfile.html",
pcap_files=pcap_files,
carved_files=carved_files,
status=Vault.get_saving())
def add_packet_entry(threat_packet, threat_flagged, timestamp):
"""
Organized flagging for threat_intel.py scanning of
- Protocol layers in packets
- Packet details
- Packet contents
"""
flagged_dict = {}
flag_matches = []
for match in threat_flagged[0].strings:
flag_matches.append(match[2].decode('utf-8'))
flagged_details = ThreatFlagged(threat_flagged, timestamp,
threat_packet,
threat_flagged[0].strings,
threat_flagged[0].rule,
threat_flagged[0].tags)
Organize.packet_counter += 1
flagged_dict[str(Organize.packet_counter)] = flagged_details
Vault.set_flagged(flagged_dict)
logger.info(
f"Threat: {flag_matches} --> {threat_flagged[0].rule} [{Thread.name()}]"
)
def get_formatted_header(prot_type):
'''Receives either TCP or UDP to return actual header '''
global COMMON_PROTOCOLS
sessions = {}
for session_header in Vault.get_session_headers():
if prot_type in session_header:
header_list = session_header[4:].replace('_', '-').split('-')
for i in range(1, 4, 2):
if header_list[i] in COMMON_PROTOCOLS:
formatted_header = COMMON_PROTOCOLS[header_list[i]] +\
" " + " ".join(header_list)
sessions[formatted_header] = session_header
break
if i == 3:
sessions[session_header] = session_header
return sessions
class TestVault(unittest.TestCase):
_multiprocess_can_split_ = True
VAULT = "vault_test.vlt"
PASS = "******"
DEFARGS = {
"iter_time": 1,
"use_urandom": True,
"password": PASS,
}
def setUp(self):
# Cleanup after any failures from previous runs
noexcept(fs.Unmount(self.path()), important=False)
noexcept(crypt.Close(self.path()), important=False)
noexcept(loopdev.Close(self.path()), important=False)
shutil.rmtree(self.dir())
self.v = Vault(self.path())
def tearDown(self):
self.v.close()
def id(self):
id = super().id()
return "_".join(id.split('.')[-2:])
def dir(self):
dir = os.path.join("test_data", self.id())
os.makedirs(dir, exist_ok=True)
return dir
def path(self):
return os.path.join(self.dir(), self.VAULT)
def mount(self):
dir = os.path.join(self.dir(), "vault")
return dir
def open(self):
self.v.open(self.mount(), password=self.PASS)
def create(self, randomize=True, **kwargs):
args = dict(**self.DEFARGS)
args.update(**kwargs)
self.v.create(self.mount(), "10m",
randomize=randomize,
**args)
class VaultTest(unittest.TestCase):
def setUp(self):
self.vault = Vault(Options)
self.vault.secret = 'secret'
self.vault.check_crypt()
def testEncrypt(self):
plain = 'hello world'
for x in xrange(0, 10):
tests = ''.join([plain, str(x)])
crypt = self.vault.encrypt(tests)
print tests
print crypt
self.assertEqual(tests, self.vault.decrypt(crypt))
def __del__(self):
self.vault.destroy_vault()
def test_closed(self):
v = Vault(self.c)
v.create(self.s.get(), reinitialize=True)
self.assertEqual(v.save(self.s.secret), VaultStatus.VAULT_CLOSED)
def test_modify(self):
v = Vault(self.c)
v.create(self.s.get(), reinitialize=True)
v.open(self.s.get())
v.data['key'] = 'value'
v.save(self.s.get())
v.close()
v.open(self.s.get())
self.assertEqual(v.data, {'key': 'value'})
def test_open(self):
v = Vault(self.c)
v.create(self.s.get(), reinitialize=True)
self.assertEqual(v.open(self.s.get()), VaultStatus.OK)
self.assertEqual(v.data, {})
def setUp(self):
self.vault = Vault(Options)
self.vault.secret = 'secret'
self.vault.check_crypt()
def process_trends(self, trends, tags, author='rabot32'):
if self.tag_types['store'] in tags:
vault = Vault()
vault.store_trends(trends, ["`store`", "trends"], author=author)
def get_recent_vault_activity(self, limit=10, author='rabot32', author_contains=None):
vault = Vault()
recent_vault_activity = vault.get_recent_activity(
limit=limit, author=author, author_contains=author_contains
)
return recent_vault_activity
