def start_action(self):
"""запуск сканирования"""
log.info("запуск сканирования")
vpath = self.edit_path.text()
vname = self.edit_vname.text()
vicon_index = self.edit_icon.currentIndex()
vicon = VOLUME_TYPE[vicon_index]
vdescription = self.edit_description.toPlainText()
print(vpath, vname, vicon, vdescription)
log.info("каталог для сканирования: " + vpath)
log.info("название: " + vname)
#--- проверка входных данных
self.__update_err("")
result, err = self.__prepare_scan(vpath, vname)
if result is False:
log.warning(err)
self.__update_err(err)
return False
#--- обновляем статус
self.__update_state(is_started=True)
stime_start = datetime.now()
log.info("начало сканирования: " + now_date())
#--- create volume record
self.__create_volume_record()
def user_delete(user_id, user_id_hash):
logging_prefix = logger_prefix + "user_delete({},{}) - ".format(user_id, user_id_hash)
log.info(logging_prefix + "Starting")
redirect_url = "/admin/"
try:
redirect_url = request.args.get("_r")
if not redirect_url:
log.warning(logging_prefix + "redirect_url not set, using default")
redirect_url = "/admin/"
#check user_id against user_id_hash and perform delete if match
if user_id_hash == sha256( SALTS['user'], user_id ):
#now delete the user
conn=get_mysql().cursor(DictCursor)
conn.execute("DELETE FROM users WHERE id=%s", (user_id,))
conn.close()
flash("The user has been deleted", "success")
else:
flash("Unable to delete user", "danger")
except Exception as e:
error = "There was an error completing your request. Details: {}".format(e)
flash(error,'danger')
log.exception(logging_prefix + error)
return redirect(redirect_url)
def move_node_up(self, node_uuid):
"""перемещение ноды вверх
!!!! не реализовано перемещение поддерева !!!!
"""
parent_node = self.find_parent_node(node_uuid)
childrens = self.get_childrens(parent_node)
childrens_uuid = [node.uuid for node in childrens]
index = childrens_uuid.index(node_uuid)
node_a = childrens[index]
if node_a.tree_rk - node_a.tree_lk > 1:
log.warning("невозможно переместить ноду - она имеет потомков!!!")
return False
if index > 0:
node_b = childrens[index - 1]
self.__swap_nodes(node_a, node_b)
return True
return False
def edit_user_permissions(action,value,user_id,user_c):
logging_prefix = logger_prefix + "edit_user_permissions({},{},{},{}) - ".format(action,value,user_id,user_c)
log.info(logging_prefix + "Starting")
action_column_map = {}
action_column_map['approve'] = "approved"
action_column_map['write_perm'] = "write_permission"
action_column_map['delete_perm'] = "delete_permission"
action_column_map['admin_perm'] = "admin"
success = 1
try:
#make sure the value is valid
if value not in ["0","1"]:
raise Exception("Invald value: {}".format(value))
#make sure the action is valid
try:
column = action_column_map[action]
except Exception as f:
log.warning(logging_prefix + "Action '{}' not found in action_column_map".format(action))
raise f
#check the hash
if user_c == sha256(SALTS['user'], str(user_id)):
#if action is approve, emails need to be sent
if action == "approve":
conn = get_mysql().cursor(DictCursor)
conn.execute("SELECT name, email FROM users WHERE id = %s", (user_id,))
user = conn.fetchone()
conn.close()
if value == "1":
log.info(logging_prefix + "Setting approved=1 for user {}".format(user_id))
sendAccountApprovedEmail(user['email'])
else:
log.info(logging_prefix + "Setting approved=0 for user {}".format(user_id))
sendAccountDisapprovedEmail(user['email'])
#now update the desired setting
conn = get_mysql().cursor()
conn.execute("UPDATE users SET "+column+" = %s WHERE id = %s", (value,user_id))
get_mysql().commit()
conn.close()
log.info(logging_prefix + "Successfully update {} to {} for user id {}".format(column,value,user_id))
else:
log.warning(logging_prefix + "Hash mismatch {} {}".format(user_id, user_c))
except Exception as e:
success = 0
error = "There was an error completing your request. Details: {}".format(e)
log.exception(logging_prefix + error)
return jsonify({ "success" : success, "new_value" : value })
def verificaNivel(self, alerta=True):
"""Verifica o nível do usuário logado se é admin ou gerente. Por padrão exibe mensagem
se não tiver permissão e salva no log."""
if current_user.nivel < 1:
if alerta:
log.warning(
'verificaNivel, Não tem nível de acesso. Usuário: ' +
current_user.login)
flash('Não tem nível de acesso!', 'alert-danger')
return False
else:
return True
def fetch_related_choices(t):
logging_prefix = logger_prefix + "fetch_related_choices({}) - ".format(t)
choices = [('_NONE_', 'n/a')]
if t == 'actor':
index = ES_PREFIX + "threat_actors"
doc_type = "actor"
elif t == 'report':
index = ES_PREFIX + "threat_reports"
doc_type = "report"
elif t == 'ttp':
index = ES_PREFIX + "threat_ttps"
doc_type = "ttp"
else:
raise Exception("Invalid type '{}'. Expected 'actor', 'ttp' or 'report'")
es_query = {
"query": {
"match_all": {}
},
"size": 1000,
"fields" : ["name"],
"sort": {
"name": {
"order": "asc"
}
}
}
try:
results = get_es().search(index, doc_type, es_query)
for r in results['hits']['hits']:
choices.append((r['_id'] + ":::" + r['fields']['name'][0],r['fields']['name'][0]))
except TransportError as te:
#if the index was not found, this is most likely becuase theres no data there
if te.status_code == 404:
log.warning("Index '{}' was not found".format(index))
else:
error = "There was an error fetching related {}s. Details: {}".format(t, te)
flash(error,'danger')
log.exception(logging_prefix + error)
except Exception as e:
error = "There was an error fetching related {}s. Details: {}".format(t, e)
flash(error,'danger')
log.exception(logging_prefix + error)
return choices
def comparaNivel(self, id):
"""Verifica se nível do usuário logado é maior que o usuario do id do parametro.
Retorna True se usuário logado for maior, ou se for ele mesmo.
"""
usuario = self.dao.buscarID(Usuario, id)
if (current_user.id
== usuario.id) or (current_user.nivel > usuario.nivel):
return True
else:
log.warning('comparaNivel, Não tem nível de acesso. Usuário: ' +
current_user.login)
flash('Não tem nível de acesso para alterar este usuário!',
'alert-danger')
return False
def PostInit(self):
log.debug('{}.PostInit started'.format(self.name))
_UIM = UIManager()
controller = _UIM.get(self._controller_uid)
parent_controller_uid = _UIM._getparentuid(self._controller_uid)
parent_controller = _UIM.get(parent_controller_uid)
if controller.pos == -1:
# Appending - Not needed to declare pos
controller.pos = parent_controller.view.GetMenuItemCount()
if controller.pos > parent_controller.view.GetMenuItemCount():
# If pos was setted out of range for inserting in parent Menu
msg = 'Invalid menu position for MenuItem with text={}. Position will be setting to {}'.format(
controller.label, parent_controller.view.GetMenuItemCount())
log.warning(msg)
controller.pos = parent_controller.view.GetMenuItemCount()
log.debug('{}.PostInit ended'.format(self.name))
def migrate(self):
log.info("проверка версии базы")
db_version = self.get_version()
log.info("тек. версия: {}, необходима: {}".format(db_version, VERSION))
if db_version == VERSION:
log.info("версия базы подходящая")
return True
log.warning("необходима миграция")
migration_steps = []
if db_version == "1.0":
migration_steps.append(migrations.up1_to_2)
migration_steps.append(migrations.up2_to_3)
migration_steps.append(migrations.up3_to_4)
elif db_version == "2":
migration_steps.append(migrations.up2_to_3)
migration_steps.append(migrations.up3_to_4)
elif db_version == "3":
migration_steps.append(migrations.up3_to_4)
else:
log.warning(
"не найдены инструкции для миграции на новую версию базы")
return False
for action in migration_steps:
action(self.connection)
self.update_version()
self.update_db_timestamp()
self.commit()
db_version = self.get_version()
log.info("тек. версия: {}".format(db_version))
sbus.emit(sbus.DB_MIGRATED)
def verify(email, verification_hash):
logging_prefix = logger_prefix + "verify() - "
log.info(logging_prefix + "Starting")
email = unquote_plus(email)
print(email)
print(verification_hash)
try:
conn=get_mysql().cursor(DictCursor)
conn.execute("SELECT id, email, name, company, justification FROM users WHERE verification_hash = %s", (verification_hash,))
r = conn.fetchone()
if not r:
log.error(logging_prefix + "User with email '{}' was not found".format(email))
flash("Your user was not found, try registering again",'danger')
elif r['email'] == email:
log.info(logging_prefix + "Successful validation of {}".format(email))
flash("Your email address have been verified, you will not be able to log in until you get approved by the Site Admin",'success')
#update the database marking this user active
user_id = r['id']
conn.execute("UPDATE users SET email_verified=1 WHERE id = %s", (user_id,))
get_mysql().commit()
#id, email, name, company
sendNewUserToApproveEmail(r['id'], r['email'],r['name'],r['company'],r['justification'])
else:
log.warning(logging_prefix + "Unsuccessful validation of {}".format(email))
flash("We were unable to verify your account",'danger')
except Exception as e:
error = "There was an error completing your request. Details: {}".format(e)
flash(error,'danger')
log.exception(logging_prefix + error)
finally:
conn.close()
return redirect("/", code=307)
def __start_scan(self):
"""запуск сканирования"""
log.info("запуск сканирования")
self.volume_path = self.scan_path_entry.get()
self.volume_name = self.volume_name_entry.get()
self.volume_description = self.description.get(1.0, tkinter.END)
log.info("каталог для сканирования: " + self.volume_path)
log.info("название: " + self.volume_name)
#--- проверка входных данных
self.__update_err("")
result, err = self.__prepare_scan()
if result is False:
log.warning(err)
self.__update_err(err)
return False
#--- обновляем статус
self.is_started = True
self.__update_state()
self.stime_start = datetime.now()
log.info("начало сканирования: " + now_date())
#--- create volume record
self.__create_volume_record()
self.files_counter = 0
#--- запуск канала чтения потока
self.__start_chan_reader()
# t = threading.Thread(target=scan_dir, args=(self.volume_path, self.chan))
t = threading.Thread(target=scaner.start_scan,
args=(self.volume_path, self.chan))
t.start()
def __on_select_node(self, uuid):
"""select node from tree..."""
if uuid is None:
log.warning("uuid = None")
return
log.debug("on select: " + uuid)
# storage = smanager.get_storage()
#--- show node data
self.current_node = storage.get_node(uuid)
# print(self.current_node)
# shared.set_current_flag(self.current_node)
# self.node_info.update_node(self.current_node)
# self.node_editor.update_node(self.current_node)
# self.node_files.update_node(self.current_node)
#--- update tree node(in project.json)
storage.project.set_current_flag(self.current_node.uuid)
def PostInit(self):
# log.debug('{}.PostInit started'.format(self.name))
UIM = UIManager()
controller = UIM.get(self._controller_uid)
parent_controller_uid = UIM._getparentuid(self._controller_uid)
parent_controller = UIM.get(parent_controller_uid)
#
if isinstance(parent_controller, MenuController):
if controller.pos == -1:
# Appending - Not needed to declare pos
controller.pos = parent_controller.view.GetMenuItemCount()
if controller.pos > parent_controller.view.GetMenuItemCount():
# If pos was setted out of range for inserting in parent Menu
msg = 'Invalid position for Menu with label={}. Position will be setting to {}'.format(
controller.label,
parent_controller.view.GetMenuItemCount())
log.warning(msg)
controller.pos = parent_controller.view.GetMenuCount()
parent_controller.view.Insert(controller.pos, controller.id,
controller.label, self,
controller.help)
elif isinstance(parent_controller, MenuBarController):
if controller.pos == -1:
# Appending - Not needed to declare pos
controller.pos = parent_controller.view.GetMenuCount()
if controller.pos > parent_controller.view.GetMenuCount():
# If pos was setted out of range for inserting in parent Menu
msg = 'Invalid position for Menu with label={}. Position will be setting to {}'.format(
controller.label, parent_controller.view.GetMenuCount())
log.warning(msg)
controller.pos = parent_controller.view.GetMenuCount()
ret_val = parent_controller.view.Insert(controller.pos, self,
controller.label)
if not ret_val:
raise Exception()
else:
raise Exception()
def view_all(t, page=1):
if t == 'favicon.ico':
return jsonify({}), 404
page = int(page)
logging_prefix = logger_prefix + "view_all() - "
log.info(logging_prefix +
"Loading view all page {} for {}".format(page, t))
form = forms.searchForm(request.form)
error = None
page_size = 50
offset = (page - 1) * page_size
url = "/{}/{}/".format(t, page)
search_url = ""
results_text = ""
try:
#this is the default query for actors in ES, i'd imagine this will be recently added/modified actors
es_query = {
"query": {
"match_all": {}
},
"size": page_size,
"from": offset,
"sort": {
"last_updated_s": {
"order": "desc"
}
}
}
#pull the query out of the url
query_string = request.args.get("q")
#someone is searching for something
if request.method == 'POST' and not query_string:
if form.validate():
print("VALID SEARCH OPERATION DETECTED, redirecting...")
#get the value
value = form.query.data
log.info(value)
#redirect to this same page, but setting the query value in the url
return redirect("/{}/1/?q={}".format(t, quote_plus(value)),
code=307)
else:
#if there was an error print the error dictionary to the console
# temporary help, these should also appear under the form field
print(form.errors)
elif query_string:
#now that the query_string is provided as ?q=, perform the search
print("VALID SEARCH OPERATION DETECTED")
#do some searching...
es_query = {
"query": {
"query_string": {
"query": query_string
}
},
"size": page_size,
"from": offset
}
search_url = "?q=" + query_string
#set the form query value to what the user is searching for
form.query.data = query_string
'''
Fetch the data from ES
'''
data = {}
data['hits'] = {}
data['hits']['hits'] = []
if t == 'actor':
index = ES_PREFIX + 'threat_actors'
doc_type = 'actor'
salt = SALTS['actor']
link_prefix = 'actor'
data_header = 'Actors'
field_header = 'Actor Name'
elif t == 'report':
index = ES_PREFIX + 'threat_reports'
doc_type = 'report'
salt = SALTS['report']
link_prefix = 'report'
data_header = 'Reports'
field_header = 'Report Title'
elif t == 'ttp':
index = ES_PREFIX + 'threat_ttps'
doc_type = 'ttp'
salt = SALTS['ttp']
link_prefix = 'ttp'
data_header = 'TTPs'
field_header = 'TTP Name'
else:
raise Exception("Unknown type {}".format(t))
try:
data = get_es().search(index, doc_type, es_query)
num_hits = len(data['hits']['hits'])
#set up previous link
if page == 1:
prev_url = None
else:
prev_url = "/{}/{}/{}".format(t, (page - 1), search_url)
if ((page - 1) * page_size) + num_hits < data['hits']['total']:
next_url = "/{}/{}/{}".format(t, (page + 1), search_url)
else:
next_url = None
url += search_url
for d in data['hits']['hits']:
s = salt + d['_id']
hash_object = hashlib.sha256(s.encode('utf-8'))
hex_dig = hash_object.hexdigest()
d["_source"]['id_hash'] = hex_dig
if num_hits == 0:
results_text = ""
else:
f = ((page - 1) * page_size) + 1
l = f + (num_hits - 1)
results_text = "Showing {} to {} of {} total results".format(
f, l, data['hits']['total'])
except TransportError as te:
#if the index was not found, this is most likely becuase theres no data there
if te.status_code == 404:
log.warning("Index '{}' was not found".format(index))
else:
error = "There was an error fetching {}. Details: {}".format(
t, te)
flash(error, 'danger')
log.exception(logging_prefix + error)
except Exception as e:
error = "The was an error fetching {}. Error: {}".format(t, e)
log.exception(error)
flash(error, "danger")
except Exception as e:
error = "There was an error completing your request. Details: {}".format(
e)
log.exception(error)
flash(error, "danger")
return redirect("/")
return render_template("view_all.html",
page_title="View All",
form=form,
data_header=data_header,
results_text=results_text,
field_header=field_header,
data=data,
link_prefix=link_prefix,
prev_url=prev_url,
next_url=next_url,
url=quote_plus(url))
def index():
logging_prefix = logger_prefix + "index() - "
log.info(logging_prefix + "Loading home page")
form = forms.searchForm(request.form)
error = None
url = "/"
query_string_url = ""
try:
#this is the default query for actors in ES, i'd imagine this will be recently added/modified actors
es_query = {
"query": {
"match_all": {}
},
"size": 10,
"sort": {
"last_updated_s": {
"order": "desc"
}
}
}
#pull the query out of the url
query_string = request.args.get("q")
#someone is searching for something
if request.method == 'POST' and not query_string:
if form.validate():
#get the value
value = form.query.data
#redirect to this same page, but setting the query value in the url
return redirect("/?q={}".format(quote_plus(value)), code=307)
else:
#if there was an error print the error dictionary to the console
# temporary help, these should also appear under the form field
print(form.errors)
elif query_string:
#now that the query_string is provided as ?q=, perform the search
print("VALID SEARCH OPERATION DETECTED")
#do some searching...
es_query = {
"query": {
"query_string": {
"query": query_string
}
},
"size": 10
}
url += "?q=" + query_string
query_string_url = "?q=" + query_string
#set the form query value to what the user is searching for
form.query.data = query_string
'''
Fetch the data from ES
'''
actors = {}
actors['hits'] = {}
actors['hits']['hits'] = []
reports = dict(actors)
ttps = dict(actors)
try:
actors = get_es().search(ES_PREFIX + 'threat_actors', 'actor',
es_query)
except TransportError as te:
#if the index was not found, this is most likely becuase theres no data there
if te.status_code == 404:
log.warning("Index 'threat_actors' was not found")
else:
error = "There was an error fetching actors. Details: {}".format(
te)
flash(error, 'danger')
log.exception(logging_prefix + error)
except Exception as e:
error = "The was an error fetching Actors. Error: {}".format(e)
log.exception(error)
flash(error, "danger")
try:
reports = get_es().search(ES_PREFIX + 'threat_reports', 'report',
es_query)
except TransportError as te:
#if the index was not found, this is most likely becuase theres no data there
if te.status_code == 404:
log.warning("Index 'threat_reports' was not found")
else:
error = "There was an error fetching reports. Details: {}".format(
te)
flash(error, 'danger')
log.exception(logging_prefix + error)
except Exception as e:
error = "The was an error fetching Reports. Error: {}".format(e)
log.exception(error)
flash(error, "danger")
try:
ttps = get_es().search(ES_PREFIX + 'threat_ttps', 'ttp', es_query)
except TransportError as te:
#if the index was not found, this is most likely becuase theres no data there
if te.status_code == 404:
log.warning("Index 'threat_ttps' was not found")
else:
error = "There was an error ttps. Details: {}".format(te)
flash(error, 'danger')
log.exception(logging_prefix + error)
except Exception as e:
error = "The was an error fetching TTPs. Error: {}".format(e)
log.exception(error)
flash(error, "danger")
'''
Modify the data as needed
'''
for actor in actors['hits']['hits']:
s = SALTS['actor'] + actor['_id']
hash_object = hashlib.sha256(s.encode('utf-8'))
hex_dig = hash_object.hexdigest()
actor["_source"]['id_hash'] = hex_dig
for report in reports['hits']['hits']:
s = SALTS['report'] + report['_id']
hash_object = hashlib.sha256(s.encode('utf-8'))
hex_dig = hash_object.hexdigest()
report["_source"]['id_hash'] = hex_dig
for ttp in ttps['hits']['hits']:
s = SALTS['ttp'] + ttp['_id']
hash_object = hashlib.sha256(s.encode('utf-8'))
hex_dig = hash_object.hexdigest()
ttp["_source"]['id_hash'] = hex_dig
except Exception as e:
error = "There was an error completing your request. Details: {}".format(
e)
log.exception(error)
flash(error, "danger")
#render the template, passing the variables we need
# templates live in the templates folder
return render_template("index.html",
page_title="ActorTrackr",
form=form,
query_string_url=query_string_url,
actors=actors,
reports=reports,
ttps=ttps,
url=quote_plus(url))
def check_password(self, supplied_pw):
log.warning(supplied_pw)
return check_password_hash(self.password, supplied_pw)
def login():
logging_prefix = logger_prefix + "login() - "
log.info(logging_prefix + "Starting")
try:
form = forms.loginForm(request.form)
search_form = forms.searchForm()
if request.method == 'POST':
if form.validate():
email = form.user_email.data
password = form.user_password.data
hashed_password = sha256(SALTS['user'], password)
conn=get_mysql().cursor(DictCursor)
#since email is unique
conn.execute("SELECT id, password, name, email_verified, approved, write_permission, delete_permission, admin FROM users WHERE email = %s", (email,))
user = conn.fetchone()
if user:
if user['password'] == hashed_password:
#we have found a valid user
if user['email_verified']==0:
#the user has not verified their email address
flash("Your email address has not been verified. Click here if you did not receive a verification email", "danger")
elif user['approved']==0:
#the user has not been approved, or their access has been disapproved
flash("Your account has been approved yet, you will receive an email when your account has been approved", "danger")
else:
#woohoo successful login, set up session variables
session['logged_in'] = True
session['id'] = user['id']
session['name'] = user['name']
session['email'] = email
session['approved'] = (user['approved'] == 1)
session['write'] = (user['write_permission'] == 1)
session['delete'] = (user['delete_permission'] == 1)
session['admin'] = (user['admin'] == 1)
session['expires'] = math.ceil(time.time()) + SESSION_EXPIRE
#now + 10 minutes of inactivity
#each time this user loads a page
#the expiration time gets now + 10m
#update last login timestamp
conn=get_mysql().cursor(DictCursor)
conn.execute("UPDATE users SET last_login=%s WHERE id = %s", (datetime.now(), user['id']))
get_mysql().commit()
conn.close()
flash("You have been logged in", "success")
if request.args.get("r"):
return redirect(request.args.get("r"))
else:
return redirect("/")
else:
log.warning(logging_prefix + "Invalid login attempt for {}".format(email))
flash("The username or password is incorrect", "danger")
else:
log.warning(logging_prefix + "Invalid login attempt for {}".format(email))
flash("The username or password is incorrect", "danger")
else:
print(form.errors)
except Exception as e:
error = "There was an error completing your request. Details: {}".format(e)
flash(error,'danger')
log.exception(logging_prefix + error)
return render_template("login.html",
page_title="Login",
form=form,
search_form=search_form
)