def unum_file_upload():
"""Return group list API call."""
request_args = request.args
file = request.data
if len(file) < 1:
request_args = request.form
file = request.files['filename']
if '/' == request_args['filename'][-1:]:
# Return 400 BAD REQUEST
abort(400, 'no subdirectories directories allowed')
if file:
file_hash = get_upload_file_hash(file)
duplicate_file = UNUM.query.filter_by(md5_hash=file_hash).first()
if duplicate_file:
error_data = {"error": "This file has already been uploaded.", "error_code": 500}
return jsonify(error_data)
new_upload = UNUM(description=request_args['description'], created_by=g.current_user.id,
classification=request_args['classification'],
group_access=request_args['group_access'],
created_time_stamp=udatetime.utcnow(),
modify_time_stamp=udatetime.utcnow(),
md5_hash=file_hash, file_name=str(request_args['filename']))
db.session.add(new_upload)
db.session.commit()
return jsonify({"file_id": new_upload.id, "md5": file_hash})
error_data = {"error": "This file type is now allowed.", "error_code": 503}
return jsonify(error_data)
def create():
"""Create yara default view."""
group_info = Groups.query.all()
if request.method == 'POST':
group_ids = Group.query.filter_by(username_id=current_user.id).all()
user_groups = []
for user_group in group_ids:
user_groups.append(user_group.groups_id)
form = CreateYara(request.form)
if form.validate():
form.yara_rules = request.form["yara_rules"]
form.yara_list_name = request.form["yara_list_name"]
new_yara = YaraRules(created_by=current_user.id,
group_access=form.group_access.data[0],
last_updated_by=current_user.id,
yara_list_name=form.yara_list_name,
created_time_stamp=udatetime.utcnow(),
modify_time_stamp=udatetime.utcnow(),
yara_rules=form.yara_rules)
db.session.add(new_yara)
db.session.commit()
flash("The yara rule has been created.")
return redirect(url_for('yara.yara_route'))
else:
for error in form.errors:
flash(str(form.errors[error][0]), 'error')
return render_template('yara_create.html',
title='Create A New Yara Ruleset',
form=form,
groups=group_info)
form = CreateYara(request.form)
return render_template('yara_create.html',
title='Create A New Yara Ruleset',
form=form,
groups=group_info)
def create_case_route():
"""Create case default view."""
group_info = Groups.query.all()
# User choices
user_choices = []
page = request.args.get('page', 1, type=int)
for user in User.query.all():
# Just keep admin out of the list
if user.id == 1:
continue
else:
user_choices.append((user.id, user.username))
if request.method == 'POST':
form = CreateCase(request.form)
form.detection_method.choices = AVAILABLE_CHOICES
file_hash = None
if form.validate():
try:
file = request.files['file']
except KeyError:
file = None
if file and allowed_file(file.filename):
secure_filename(file.filename)
file_hash = get_upload_file_hash(file)
detection_method_selection = None
for items in AVAILABLE_CHOICES:
if items[0] == form.detection_method.data[0]:
detection_method_selection = items
new_case = Cases(description=form.description.data,
subject=form.subject.data,
created_by=current_user.id,
case_status=4,
detection_method=detection_method_selection[1],
group_access=form.group_access.data[0],
created_time_stamp=udatetime.utcnow(),
modify_time_stamp=udatetime.utcnow(),
md5_hash=file_hash,
assigned_to=form.assigned_to.data[0])
db.session.add(new_case)
db.session.commit()
flash("The case has been created.")
return redirect(url_for('cases.cases_plugin_route', page=page))
form = CreateCase(request.form)
return render_template('create_case.html',
title='Create Case',
form=form,
groups=group_info,
detection_method=AVAILABLE_CHOICES,
assigned_to=user_choices,
search_url='cases.search')
def unum_file_route():
"""Create upload default view."""
group_info = Groups.query.all()
classification_choices_list = Classification.query.all()
classification_choices = []
assigned_classification_value = None
for items in Classification.query.all():
classification_choices.append((int(items.id), items.classification))
if request.method == 'POST':
form = UploadFile(request.form)
md5_hash = None
if form.validate_on_submit():
form.classification.choices = classification_choices
try:
file = request.files['file']
except KeyError:
file = None
if file and allowed_file(file.filename):
secure_filename(file.filename)
md5_hash = get_upload_file_hash(file)
duplicate_file = UNUM.query.filter_by(md5_hash=md5_hash).first()
classification_choices = classification_choices_list
if duplicate_file:
flash("The upload file has already been uploaded.")
return redirect(url_for('unum.unum_plugin_route'))
new_upload = UNUM(description=form.description.data,
created_by=current_user.id,
classification=form.classification.data[0],
file_name=str(file.filename),
group_access=form.group_access.data[0],
created_time_stamp=udatetime.utcnow(),
modify_time_stamp=udatetime.utcnow(),
md5_hash=md5_hash)
db.session.add(new_upload)
db.session.commit()
flash("The upload file has been created and we started processing it for you.")
return redirect(url_for('unum.unum_plugin_route'))
else:
for error in form.errors:
flash(str(form.errors[error][0]), 'error')
return redirect(url_for('unum.unum_plugin_route'))
form = UploadFile(request.form)
form.classification.choices = classification_choices
return render_template('unum_upload_file.html',
title='Upload New File',
form=form,
groups=group_info,
classification=classification_choices)
def call_back(ch, method, properties, file_hash):
"""Cuckoo Processing call back function."""
app = create_app()
db.init_app(app)
report = submit_file_to_cuckoo(file_hash)
cuckoo_url = os.environ.get('CUCKOO_URL')
report_list_ids = []
url_list = []
for items in report:
url_list.append(str(cuckoo_url + "/analysis/" + str(items)))
report_list_ids.append(str(items))
with app.app_context():
try:
new_cuckoo = CuckooReports(url=str(url_list),
md5_hash=file_hash.decode("utf-8"),
modify_time=udatetime.utcnow(),
report_ids=report_list_ids)
db.session.add(new_cuckoo)
db.session.commit()
message_data = ujson.dumps(new_cuckoo.to_dict(),
indent=2,
sort_keys=True)
match_known_item = UNUM.query.filter_by(
md5_hash=file_hash.decode("utf-8")).first()
if match_known_item:
cuckoo_notification = \
Message(sender_id=1, recipient_id=match_known_item.created_by, body=message_data)
db.session.add(cuckoo_notification)
db.session.commit()
except:
error("Problem creating the cuckoo report")
def utcnow():
"""
Gets the current datetime in UTC format with millisecond precision
:return:
"""
now = udatetime.utcnow().replace(tzinfo=UTC)
return remove_microseconds(now)
def before_request() -> None:
"""Set user last seen time user."""
if current_user.is_authenticated:
current_user.last_seen = udatetime.utcnow().replace(tzinfo=None)
db.session.commit()
g.search_form = SearchForm()
g.locale = str(get_locale())
def call_back(ch, method, properties, report_id):
"""IDS Processing call back function."""
app = create_app()
db.init_app(app)
rules_dir = os.environ.get('IDS_RULE_DIR') or "aucr_app/plugins/IDS_Plugin/rules/"
logs_dir = os.environ.get('IDS_LOGS_DIR') or "aucr_app/plugins/IDS_Plugin/logs/"
with app.app_context():
ids_report = IDSRules.query.filter_by(id=report_id.decode('utf-8')).first()
ids_rules_file = current_app.mongo.db.aucr.find_one({"filename": ids_report.ids_plugin_list_name})
"suricata -v -k none -c suricata.yml -S signatures.rules -r pcap/test.pcap"
with open("aucr_app/plugins/IDS_Plugin/rules/" + str(ids_report.ids_plugin_list_name), 'w') as test_signature:
test_signature.write(ids_rules_file["fileobj"])
args = ["suricata",
"-c", os.environ.get('SURICATA_CONFIG'),
"-k", "none",
"-S", rules_dir + str(ids_report.ids_plugin_list_name),
"-r", os.environ.get('FILE_FOLDER'),
"-l", logs_dir,
]
subprocess.check_call(args)
with open(str(logs_dir + "eve.json"), 'r') as eve_json:
raw_data = eve_json.readlines()
for item in raw_data:
data = ujson.loads(item)
flat_data_dictionary = flatten_dictionary(data)
flat_data_dictionary["report"]["process_time"] = udatetime.utcnow()
# TODO create and use bulk index to ES for better performance.
index_data_to_es("ids_suricata", flat_data_dictionary["report"])
shutil.rmtree(logs_dir)
# TODO upload result data to object storage.
os.mkdir(logs_dir)
def call_back(ch, method, properties, report_id):
"""Yara Processing call back function."""
app = create_app()
db.init_app(app)
with app.app_context():
yara_report = YaraRules.query.filter_by(
id=report_id.decode('utf-8')).first()
yara_rule_file = yara_report.yara_rules
yara_matches = test_yara(yara_report)
for item in yara_matches:
match_known_item = UNUM.query.filter_by(md5_hash=item).first()
if match_known_item:
match_known_classification = Classification.query.filter_by(
id=match_known_item.classification).first()
new_yara_result = YaraRuleResults(
yara_list_id=yara_report.id,
matches=match_known_item.md5_hash,
file_matches=match_known_item.id,
file_string_matches=yara_matches[item]["strings"],
file_classification=match_known_classification.
classification,
run_time=udatetime.utcnow())
db.session.add(new_yara_result)
db.session.commit()
message_data = ujson.dumps(new_yara_result.to_dict(),
indent=2,
sort_keys=True)
yara_notification = \
Message(sender_id=1, recipient_id=yara_report.created_by, body=message_data)
db.session.add(yara_notification)
db.session.commit()
def check_token(token):
"""Check a token against user token."""
now = udatetime.utcnow().replace(tzinfo=None)
user = User.query.filter_by(token=token).first()
if user is None or user.token_expiration < now:
return None
return user
def utcnow():
"""
Gets the current datetime in UTC format with millisecond precision
:return:
"""
now = udatetime.utcnow().replace(tzinfo=UTC)
return datetime(now.year, now.month, now.day, now.hour, now.minute,
now.second, now.microsecond / 1000 * 1000, UTC)
def get_token(self, expires_in=360000):
"""Generate and return a token for user auth."""
now = udatetime.utcnow().replace(tzinfo=None)
if self.token and self.token_expiration > now - timedelta(seconds=60):
return self.token
self.token = base64.b64encode(os.urandom(64)).decode('utf-8')
self.token_expiration = now + timedelta(seconds=expires_in)
db.session.add(self)
return self.token
def create_yara_rule_list():
"""API Update Yara Rule."""
if request.method == "POST":
data = request.form
if 'yara_rule_list' in data and data['yara_rule_list'] != data.yara_rule_list and \
YaraRules.query.filter_by(yara_rule_list=data['yara_rule_list']).first():
return bad_request('Please use a different yara rule list name.')
data_mongo = {
"filename": data["yara_list_name"],
"fileobj": data["yara_rules"]
}
current_app.mongo.db.aucr.insert_one(data_mongo)
new_yara = YaraRules(created_by=int(data["created_by"]),
group_access=int(data["group_access"]),
yara_list_name=str(data["yara_list_name"]),
created_time_stamp=udatetime.utcnow(),
modify_time_stamp=udatetime.utcnow())
db.session.add(new_yara)
db.session.commit()
return jsonify(new_yara.to_dict())
def test_utcnow(self):
dt_now = datetime.utcnow()
now = udatetime.utcnow()
self.assertIsInstance(now, datetime)
self.assertEqual(now.year, dt_now.year)
self.assertEqual(now.month, dt_now.month)
self.assertEqual(now.day, dt_now.day)
self.assertEqual(now.hour, dt_now.hour)
self.assertEqual(now.minute, dt_now.minute)
self.assertEqual(now.second, dt_now.second)
def test_utcnow(self):
dt_now = datetime.utcnow()
now = udatetime.utcnow()
self.assertIsInstance(now, datetime)
self.assertEqual(now.year, dt_now.year)
self.assertEqual(now.month, dt_now.month)
self.assertEqual(now.day, dt_now.day)
self.assertEqual(now.hour, dt_now.hour)
self.assertEqual(now.minute, dt_now.minute)
self.assertEqual(now.second, dt_now.second)
def test_utcnow(self):
dt_now = datetime.utcnow()
now = udatetime.utcnow()
self.assertIsInstance(now, datetime)
self.assertEqual(now.year, dt_now.year)
self.assertEqual(now.month, dt_now.month)
self.assertEqual(now.day, dt_now.day)
self.assertEqual(now.hour, dt_now.hour)
self.assertEqual(now.minute, dt_now.minute)
self.assertEqual(now.second, dt_now.second)
# self.assertEqual(now.microsecond, dt_now.microsecond)
self.assertEqual(now.utcoffset(), timedelta(0))
self.assertEqual(now.dst(), NO_DST)
def create_ids_rule_list():
"""API Update IDS Rule."""
if request.method == "POST":
data = request.form
if 'ids_plugin_list_name' in data and data['ids_plugin_list_name'] != data.ids_plugin_list_name and \
IDSRules.query.filter_by(ids_plugin_list_name=data['ids_plugin_list_name']).first():
return bad_request('Please use a different IDS_Plugin rule list name.')
data_mongo = {"filename": data["ids_plugin_list_name"], "fileobj": data["ids_rules"]}
current_app.mongo.db.aucr.insert_one(data_mongo)
new_ids_rule_list = IDSRules(created_by=int(data["created_by"]), group_access=int(data["group_access"]),
ids_plugin_list_name=str(data["ids_plugin_list_name"]),
created_time_stamp=udatetime.utcnow(), modify_time_stamp=udatetime.utcnow())
db.session.add(new_ids_rule_list)
db.session.commit()
return jsonify(new_ids_rule_list.to_dict())
def run(self, results):
"""Index the Cuckoo report into ElasticSearch.
@param results: analysis results dictionary.
@raise CuckooReportError: if the connection or reporting failed.
"""
# Gets the time which will be used for indexing the document into ES
# ES needs epoch time in seconds per the mapping
self.report_time = udatetime.utcnow_to_string()
# Get the index time option and set the dated index accordingly
date_index = udatetime.utcnow().strftime({
"yearly": "%Y",
"monthly": "%Y-%m",
"daily": "%Y-%m-%d",
}[elastic.index_time_pattern])
self.dated_index = "%s-%s" % (elastic.index, date_index)
# Index target information, the behavioral summary, and
# VirusTotal results.
# index elements that are not empty ES should not index blank fields
doc = self.process_info(results)
doc["cuckoo_node"] = elastic.cuckoo_node
signatures = results.get("signatures")
if signatures:
doc["signatures"] = self.process_signatures(signatures)
dropped = results.get("dropped")
if dropped:
doc["dropped"] = dropped
procmemory = results.get("procmemory")
if procmemory:
doc["procmemory"] = procmemory
self.do_index(doc)
# Index the API calls.
if elastic.calls:
self.process_behavior(results)
def messages():
"""Return user message flask app blueprint route."""
current_user.last_message_read_time = \
udatetime.utcnow().replace(tzinfo=None)
current_user.add_notification('unread_message_count', 0)
db.session.commit()
page = request.args.get('page', 1, type=int)
messages_list = current_user.messages_received.order_by(
Message.timestamp.desc()).paginate(
page, int(current_app.config['POSTS_PER_PAGE']), False)
next_url = url_for(
'auth.messages',
page=messages_list.next_num) if messages_list.has_next else None
prev_url = url_for(
'auth.messages',
page=messages_list.prev_num) if messages_list.has_prev else None
return render_template('messages.html',
messages=messages_list.items,
next_url=next_url,
page=page,
prev_url=prev_url)
def get_task_status(tid):
rconn = get_redis_conn()
rtkey = DOWNLOADER_TASK_PREFIX + tid
try:
tdata = rconn.hgetall(rtkey)
if len(tdata) == 0:
return None
tdata["query"] = json.loads(tdata["query"])
tdata["status_url"] = url_for(".status", tid=tid, _external=True)
ttl = rconn.ttl(rtkey)
if ttl != -1:
ttl = timedelta(seconds=ttl)
tdata["expires"] = udatetime.to_string(udatetime.utcnow() + ttl)
if tdata.get('task_status'):
# `task_status` isn't set in redis while pending, so if
# set then it is complete
tdata["complete"] = True
else:
ar = downloader.AsyncResult(tid)
tdata['task_status'] = ar.status
tdata["complete"] = ar.ready()
if ar.ready():
rconn.hset(rtkey, "task_status", ar.status)
if ar.successful():
tdata["download_url"] = ar.result
rconn.hset(rtkey, "download_url", ar.result)
elif ar.failed():
tdata["error"] = str(ar.result)
rconn.hset(rtkey, "error", str(ar.result))
gevent.spawn(dissociate_query_hash, tid, tdata)
except Exception as e:
logger.exception("Failed getting status of download %s", tid)
tdata = {
"complete": False,
"task_status": "UNKNOWN",
"status_url": url_for(".status", tid=tid, _external=True),
"error": str(e)
}
return tdata
def udatetime_utcnow():
udatetime.utcnow()
def _udatetime():
return udatetime.utcnow()
def revoke_token(self):
"""Check and expire user token if expiration time is True."""
self.token_expiration = udatetime.utcnow().replace(
tzinfo=None) - timedelta(seconds=1)
def udatetime_utcnow():
udatetime.utcnow()
def yara_rule_edit():
"""Edit yara view."""
group_info = Groups.query.all()
submitted_yara_id = request.args.get("id")
group_ids = Group.query.filter_by(username_id=current_user.id).all()
user_groups = []
for user_group in group_ids:
user_groups.append(user_group.groups_id)
yara_value = YaraRules.query.filter_by(id=submitted_yara_id)
yara_value = yara_value.filter(
or_(YaraRules.id == submitted_yara_id,
YaraRules.group_access.in_(user_groups))).first()
if request.method == 'POST':
if yara_value:
yara_value.group_access = yara_value.group_access
form = EditYara(request.form)
if form.validate_on_submit():
yara_value.group_access = yara_value.group_access
rabbit_mq_server_ip = current_app.config['RABBITMQ_SERVER']
yara_value.yara_rules = request.form["yara_rules"]
yara_value.yara_list_name = request.form["yara_list_name"]
mq_config_dict = get_mq_yaml_configs()
files_config_dict = mq_config_dict["reports"]
yara_value.modify_time_stamp = udatetime.utcnow()
for item in files_config_dict:
if "yara" in item:
logging.info("Adding " + str(yara_value.id) + " " +
str(item["yara"][0]) + " to MQ")
index_mq_aucr_report(str(yara_value.id),
str(rabbit_mq_server_ip),
item["yara"][0])
db.session.commit()
flash("The Yara Rule " + str(yara_value.yara_list_name) +
" has been updated and the rule is running.")
else:
for error in form.errors:
flash(str(form.errors[error][0]), 'error')
return render_template('yara_edit.html', form=form)
return redirect(url_for('yara.yara_route'))
if request.method == "GET":
if yara_value:
form = EditYara(yara_value)
yara_list_results = YaraRuleResults.query.filter_by(
yara_list_id=yara_value.id)
yara_results_dict = {}
for item in yara_list_results:
if item.run_time > yara_value.modify_time_stamp:
item_dict = {
"id": item.file_matches,
"MD5 Hash": item.matches,
"Classification": item.file_classification
}
yara_results_dict[str(item.file_matches)] = item_dict
yara_dict = {
"id": yara_value.id,
"yara_rules": yara_value.yara_rules,
"yara_list_name": yara_value.yara_list_name,
"length": yara_value.yara_rules.count('\n') + 2
}
return render_template('yara_edit.html',
title='Edit Yara Ruleset',
form=form,
groups=group_info,
table_dict=yara_dict,
yara_results=yara_results_dict)
else:
return yara_route()
