def unum_file_upload(): """Return group list API call.""" request_args = request.args file = request.data if len(file) < 1: request_args = request.form file = request.files['filename'] if '/' == request_args['filename'][-1:]: # Return 400 BAD REQUEST abort(400, 'no subdirectories directories allowed') if file: file_hash = get_upload_file_hash(file) duplicate_file = UNUM.query.filter_by(md5_hash=file_hash).first() if duplicate_file: error_data = {"error": "This file has already been uploaded.", "error_code": 500} return jsonify(error_data) new_upload = UNUM(description=request_args['description'], created_by=g.current_user.id, classification=request_args['classification'], group_access=request_args['group_access'], created_time_stamp=udatetime.utcnow(), modify_time_stamp=udatetime.utcnow(), md5_hash=file_hash, file_name=str(request_args['filename'])) db.session.add(new_upload) db.session.commit() return jsonify({"file_id": new_upload.id, "md5": file_hash}) error_data = {"error": "This file type is now allowed.", "error_code": 503} return jsonify(error_data)
def create(): """Create yara default view.""" group_info = Groups.query.all() if request.method == 'POST': group_ids = Group.query.filter_by(username_id=current_user.id).all() user_groups = [] for user_group in group_ids: user_groups.append(user_group.groups_id) form = CreateYara(request.form) if form.validate(): form.yara_rules = request.form["yara_rules"] form.yara_list_name = request.form["yara_list_name"] new_yara = YaraRules(created_by=current_user.id, group_access=form.group_access.data[0], last_updated_by=current_user.id, yara_list_name=form.yara_list_name, created_time_stamp=udatetime.utcnow(), modify_time_stamp=udatetime.utcnow(), yara_rules=form.yara_rules) db.session.add(new_yara) db.session.commit() flash("The yara rule has been created.") return redirect(url_for('yara.yara_route')) else: for error in form.errors: flash(str(form.errors[error][0]), 'error') return render_template('yara_create.html', title='Create A New Yara Ruleset', form=form, groups=group_info) form = CreateYara(request.form) return render_template('yara_create.html', title='Create A New Yara Ruleset', form=form, groups=group_info)
def create_case_route(): """Create case default view.""" group_info = Groups.query.all() # User choices user_choices = [] page = request.args.get('page', 1, type=int) for user in User.query.all(): # Just keep admin out of the list if user.id == 1: continue else: user_choices.append((user.id, user.username)) if request.method == 'POST': form = CreateCase(request.form) form.detection_method.choices = AVAILABLE_CHOICES file_hash = None if form.validate(): try: file = request.files['file'] except KeyError: file = None if file and allowed_file(file.filename): secure_filename(file.filename) file_hash = get_upload_file_hash(file) detection_method_selection = None for items in AVAILABLE_CHOICES: if items[0] == form.detection_method.data[0]: detection_method_selection = items new_case = Cases(description=form.description.data, subject=form.subject.data, created_by=current_user.id, case_status=4, detection_method=detection_method_selection[1], group_access=form.group_access.data[0], created_time_stamp=udatetime.utcnow(), modify_time_stamp=udatetime.utcnow(), md5_hash=file_hash, assigned_to=form.assigned_to.data[0]) db.session.add(new_case) db.session.commit() flash("The case has been created.") return redirect(url_for('cases.cases_plugin_route', page=page)) form = CreateCase(request.form) return render_template('create_case.html', title='Create Case', form=form, groups=group_info, detection_method=AVAILABLE_CHOICES, assigned_to=user_choices, search_url='cases.search')
def unum_file_route(): """Create upload default view.""" group_info = Groups.query.all() classification_choices_list = Classification.query.all() classification_choices = [] assigned_classification_value = None for items in Classification.query.all(): classification_choices.append((int(items.id), items.classification)) if request.method == 'POST': form = UploadFile(request.form) md5_hash = None if form.validate_on_submit(): form.classification.choices = classification_choices try: file = request.files['file'] except KeyError: file = None if file and allowed_file(file.filename): secure_filename(file.filename) md5_hash = get_upload_file_hash(file) duplicate_file = UNUM.query.filter_by(md5_hash=md5_hash).first() classification_choices = classification_choices_list if duplicate_file: flash("The upload file has already been uploaded.") return redirect(url_for('unum.unum_plugin_route')) new_upload = UNUM(description=form.description.data, created_by=current_user.id, classification=form.classification.data[0], file_name=str(file.filename), group_access=form.group_access.data[0], created_time_stamp=udatetime.utcnow(), modify_time_stamp=udatetime.utcnow(), md5_hash=md5_hash) db.session.add(new_upload) db.session.commit() flash("The upload file has been created and we started processing it for you.") return redirect(url_for('unum.unum_plugin_route')) else: for error in form.errors: flash(str(form.errors[error][0]), 'error') return redirect(url_for('unum.unum_plugin_route')) form = UploadFile(request.form) form.classification.choices = classification_choices return render_template('unum_upload_file.html', title='Upload New File', form=form, groups=group_info, classification=classification_choices)
def call_back(ch, method, properties, file_hash): """Cuckoo Processing call back function.""" app = create_app() db.init_app(app) report = submit_file_to_cuckoo(file_hash) cuckoo_url = os.environ.get('CUCKOO_URL') report_list_ids = [] url_list = [] for items in report: url_list.append(str(cuckoo_url + "/analysis/" + str(items))) report_list_ids.append(str(items)) with app.app_context(): try: new_cuckoo = CuckooReports(url=str(url_list), md5_hash=file_hash.decode("utf-8"), modify_time=udatetime.utcnow(), report_ids=report_list_ids) db.session.add(new_cuckoo) db.session.commit() message_data = ujson.dumps(new_cuckoo.to_dict(), indent=2, sort_keys=True) match_known_item = UNUM.query.filter_by( md5_hash=file_hash.decode("utf-8")).first() if match_known_item: cuckoo_notification = \ Message(sender_id=1, recipient_id=match_known_item.created_by, body=message_data) db.session.add(cuckoo_notification) db.session.commit() except: error("Problem creating the cuckoo report")
def utcnow(): """ Gets the current datetime in UTC format with millisecond precision :return: """ now = udatetime.utcnow().replace(tzinfo=UTC) return remove_microseconds(now)
def before_request() -> None: """Set user last seen time user.""" if current_user.is_authenticated: current_user.last_seen = udatetime.utcnow().replace(tzinfo=None) db.session.commit() g.search_form = SearchForm() g.locale = str(get_locale())
def call_back(ch, method, properties, report_id): """IDS Processing call back function.""" app = create_app() db.init_app(app) rules_dir = os.environ.get('IDS_RULE_DIR') or "aucr_app/plugins/IDS_Plugin/rules/" logs_dir = os.environ.get('IDS_LOGS_DIR') or "aucr_app/plugins/IDS_Plugin/logs/" with app.app_context(): ids_report = IDSRules.query.filter_by(id=report_id.decode('utf-8')).first() ids_rules_file = current_app.mongo.db.aucr.find_one({"filename": ids_report.ids_plugin_list_name}) "suricata -v -k none -c suricata.yml -S signatures.rules -r pcap/test.pcap" with open("aucr_app/plugins/IDS_Plugin/rules/" + str(ids_report.ids_plugin_list_name), 'w') as test_signature: test_signature.write(ids_rules_file["fileobj"]) args = ["suricata", "-c", os.environ.get('SURICATA_CONFIG'), "-k", "none", "-S", rules_dir + str(ids_report.ids_plugin_list_name), "-r", os.environ.get('FILE_FOLDER'), "-l", logs_dir, ] subprocess.check_call(args) with open(str(logs_dir + "eve.json"), 'r') as eve_json: raw_data = eve_json.readlines() for item in raw_data: data = ujson.loads(item) flat_data_dictionary = flatten_dictionary(data) flat_data_dictionary["report"]["process_time"] = udatetime.utcnow() # TODO create and use bulk index to ES for better performance. index_data_to_es("ids_suricata", flat_data_dictionary["report"]) shutil.rmtree(logs_dir) # TODO upload result data to object storage. os.mkdir(logs_dir)
def call_back(ch, method, properties, report_id): """Yara Processing call back function.""" app = create_app() db.init_app(app) with app.app_context(): yara_report = YaraRules.query.filter_by( id=report_id.decode('utf-8')).first() yara_rule_file = yara_report.yara_rules yara_matches = test_yara(yara_report) for item in yara_matches: match_known_item = UNUM.query.filter_by(md5_hash=item).first() if match_known_item: match_known_classification = Classification.query.filter_by( id=match_known_item.classification).first() new_yara_result = YaraRuleResults( yara_list_id=yara_report.id, matches=match_known_item.md5_hash, file_matches=match_known_item.id, file_string_matches=yara_matches[item]["strings"], file_classification=match_known_classification. classification, run_time=udatetime.utcnow()) db.session.add(new_yara_result) db.session.commit() message_data = ujson.dumps(new_yara_result.to_dict(), indent=2, sort_keys=True) yara_notification = \ Message(sender_id=1, recipient_id=yara_report.created_by, body=message_data) db.session.add(yara_notification) db.session.commit()
def check_token(token): """Check a token against user token.""" now = udatetime.utcnow().replace(tzinfo=None) user = User.query.filter_by(token=token).first() if user is None or user.token_expiration < now: return None return user
def utcnow(): """ Gets the current datetime in UTC format with millisecond precision :return: """ now = udatetime.utcnow().replace(tzinfo=UTC) return datetime(now.year, now.month, now.day, now.hour, now.minute, now.second, now.microsecond / 1000 * 1000, UTC)
def get_token(self, expires_in=360000): """Generate and return a token for user auth.""" now = udatetime.utcnow().replace(tzinfo=None) if self.token and self.token_expiration > now - timedelta(seconds=60): return self.token self.token = base64.b64encode(os.urandom(64)).decode('utf-8') self.token_expiration = now + timedelta(seconds=expires_in) db.session.add(self) return self.token
def create_yara_rule_list(): """API Update Yara Rule.""" if request.method == "POST": data = request.form if 'yara_rule_list' in data and data['yara_rule_list'] != data.yara_rule_list and \ YaraRules.query.filter_by(yara_rule_list=data['yara_rule_list']).first(): return bad_request('Please use a different yara rule list name.') data_mongo = { "filename": data["yara_list_name"], "fileobj": data["yara_rules"] } current_app.mongo.db.aucr.insert_one(data_mongo) new_yara = YaraRules(created_by=int(data["created_by"]), group_access=int(data["group_access"]), yara_list_name=str(data["yara_list_name"]), created_time_stamp=udatetime.utcnow(), modify_time_stamp=udatetime.utcnow()) db.session.add(new_yara) db.session.commit() return jsonify(new_yara.to_dict())
def test_utcnow(self): dt_now = datetime.utcnow() now = udatetime.utcnow() self.assertIsInstance(now, datetime) self.assertEqual(now.year, dt_now.year) self.assertEqual(now.month, dt_now.month) self.assertEqual(now.day, dt_now.day) self.assertEqual(now.hour, dt_now.hour) self.assertEqual(now.minute, dt_now.minute) self.assertEqual(now.second, dt_now.second)
def test_utcnow(self): dt_now = datetime.utcnow() now = udatetime.utcnow() self.assertIsInstance(now, datetime) self.assertEqual(now.year, dt_now.year) self.assertEqual(now.month, dt_now.month) self.assertEqual(now.day, dt_now.day) self.assertEqual(now.hour, dt_now.hour) self.assertEqual(now.minute, dt_now.minute) self.assertEqual(now.second, dt_now.second) # self.assertEqual(now.microsecond, dt_now.microsecond) self.assertEqual(now.utcoffset(), timedelta(0)) self.assertEqual(now.dst(), NO_DST)
def create_ids_rule_list(): """API Update IDS Rule.""" if request.method == "POST": data = request.form if 'ids_plugin_list_name' in data and data['ids_plugin_list_name'] != data.ids_plugin_list_name and \ IDSRules.query.filter_by(ids_plugin_list_name=data['ids_plugin_list_name']).first(): return bad_request('Please use a different IDS_Plugin rule list name.') data_mongo = {"filename": data["ids_plugin_list_name"], "fileobj": data["ids_rules"]} current_app.mongo.db.aucr.insert_one(data_mongo) new_ids_rule_list = IDSRules(created_by=int(data["created_by"]), group_access=int(data["group_access"]), ids_plugin_list_name=str(data["ids_plugin_list_name"]), created_time_stamp=udatetime.utcnow(), modify_time_stamp=udatetime.utcnow()) db.session.add(new_ids_rule_list) db.session.commit() return jsonify(new_ids_rule_list.to_dict())
def run(self, results): """Index the Cuckoo report into ElasticSearch. @param results: analysis results dictionary. @raise CuckooReportError: if the connection or reporting failed. """ # Gets the time which will be used for indexing the document into ES # ES needs epoch time in seconds per the mapping self.report_time = udatetime.utcnow_to_string() # Get the index time option and set the dated index accordingly date_index = udatetime.utcnow().strftime({ "yearly": "%Y", "monthly": "%Y-%m", "daily": "%Y-%m-%d", }[elastic.index_time_pattern]) self.dated_index = "%s-%s" % (elastic.index, date_index) # Index target information, the behavioral summary, and # VirusTotal results. # index elements that are not empty ES should not index blank fields doc = self.process_info(results) doc["cuckoo_node"] = elastic.cuckoo_node signatures = results.get("signatures") if signatures: doc["signatures"] = self.process_signatures(signatures) dropped = results.get("dropped") if dropped: doc["dropped"] = dropped procmemory = results.get("procmemory") if procmemory: doc["procmemory"] = procmemory self.do_index(doc) # Index the API calls. if elastic.calls: self.process_behavior(results)
def messages(): """Return user message flask app blueprint route.""" current_user.last_message_read_time = \ udatetime.utcnow().replace(tzinfo=None) current_user.add_notification('unread_message_count', 0) db.session.commit() page = request.args.get('page', 1, type=int) messages_list = current_user.messages_received.order_by( Message.timestamp.desc()).paginate( page, int(current_app.config['POSTS_PER_PAGE']), False) next_url = url_for( 'auth.messages', page=messages_list.next_num) if messages_list.has_next else None prev_url = url_for( 'auth.messages', page=messages_list.prev_num) if messages_list.has_prev else None return render_template('messages.html', messages=messages_list.items, next_url=next_url, page=page, prev_url=prev_url)
def get_task_status(tid): rconn = get_redis_conn() rtkey = DOWNLOADER_TASK_PREFIX + tid try: tdata = rconn.hgetall(rtkey) if len(tdata) == 0: return None tdata["query"] = json.loads(tdata["query"]) tdata["status_url"] = url_for(".status", tid=tid, _external=True) ttl = rconn.ttl(rtkey) if ttl != -1: ttl = timedelta(seconds=ttl) tdata["expires"] = udatetime.to_string(udatetime.utcnow() + ttl) if tdata.get('task_status'): # `task_status` isn't set in redis while pending, so if # set then it is complete tdata["complete"] = True else: ar = downloader.AsyncResult(tid) tdata['task_status'] = ar.status tdata["complete"] = ar.ready() if ar.ready(): rconn.hset(rtkey, "task_status", ar.status) if ar.successful(): tdata["download_url"] = ar.result rconn.hset(rtkey, "download_url", ar.result) elif ar.failed(): tdata["error"] = str(ar.result) rconn.hset(rtkey, "error", str(ar.result)) gevent.spawn(dissociate_query_hash, tid, tdata) except Exception as e: logger.exception("Failed getting status of download %s", tid) tdata = { "complete": False, "task_status": "UNKNOWN", "status_url": url_for(".status", tid=tid, _external=True), "error": str(e) } return tdata
def udatetime_utcnow(): udatetime.utcnow()
def _udatetime(): return udatetime.utcnow()
def revoke_token(self): """Check and expire user token if expiration time is True.""" self.token_expiration = udatetime.utcnow().replace( tzinfo=None) - timedelta(seconds=1)
def yara_rule_edit(): """Edit yara view.""" group_info = Groups.query.all() submitted_yara_id = request.args.get("id") group_ids = Group.query.filter_by(username_id=current_user.id).all() user_groups = [] for user_group in group_ids: user_groups.append(user_group.groups_id) yara_value = YaraRules.query.filter_by(id=submitted_yara_id) yara_value = yara_value.filter( or_(YaraRules.id == submitted_yara_id, YaraRules.group_access.in_(user_groups))).first() if request.method == 'POST': if yara_value: yara_value.group_access = yara_value.group_access form = EditYara(request.form) if form.validate_on_submit(): yara_value.group_access = yara_value.group_access rabbit_mq_server_ip = current_app.config['RABBITMQ_SERVER'] yara_value.yara_rules = request.form["yara_rules"] yara_value.yara_list_name = request.form["yara_list_name"] mq_config_dict = get_mq_yaml_configs() files_config_dict = mq_config_dict["reports"] yara_value.modify_time_stamp = udatetime.utcnow() for item in files_config_dict: if "yara" in item: logging.info("Adding " + str(yara_value.id) + " " + str(item["yara"][0]) + " to MQ") index_mq_aucr_report(str(yara_value.id), str(rabbit_mq_server_ip), item["yara"][0]) db.session.commit() flash("The Yara Rule " + str(yara_value.yara_list_name) + " has been updated and the rule is running.") else: for error in form.errors: flash(str(form.errors[error][0]), 'error') return render_template('yara_edit.html', form=form) return redirect(url_for('yara.yara_route')) if request.method == "GET": if yara_value: form = EditYara(yara_value) yara_list_results = YaraRuleResults.query.filter_by( yara_list_id=yara_value.id) yara_results_dict = {} for item in yara_list_results: if item.run_time > yara_value.modify_time_stamp: item_dict = { "id": item.file_matches, "MD5 Hash": item.matches, "Classification": item.file_classification } yara_results_dict[str(item.file_matches)] = item_dict yara_dict = { "id": yara_value.id, "yara_rules": yara_value.yara_rules, "yara_list_name": yara_value.yara_list_name, "length": yara_value.yara_rules.count('\n') + 2 } return render_template('yara_edit.html', title='Edit Yara Ruleset', form=form, groups=group_info, table_dict=yara_dict, yara_results=yara_results_dict) else: return yara_route()