def update_indicator(indicator_id, report_id): print('{} {}'.format(indicator_id, report_id)) form_data = request.form report_indicator = Links.query.filter_by(report=report_id).filter_by( indicator=indicator_id).first() if report_indicator: print('link found') if form_data.get('kill_chain'): print('kill chain {}'.format(form_data.get('kill_chain'))) report_indicator.kill_chain = form_data.get('kill_chain') if form_data.get('confidence'): print('confidence {}'.format(form_data.get('confidence'))) report_indicator.confidence = form_data.get('confidence') if form_data.get('diamond_model'): print('diamond_model {}'.format(form_data.get('diamond_model'))) report_indicator.diamond_model = form_data.get('diamond_model') db.session.commit() db.session.flush() else: report_indicator = Links(report=report_id, indicator=indicator_id) report_indicator.kill_chain = form_data.get('kill_chain', 'Unknown') report_indicator.confidence = form_data.get('confidence', 'Low') report_indicator.diamond_model = form_data.get('diamond_model', 'Unknown') add_db_entry(report_indicator) return 'success'
def user_profile(): form = LinksForm() if form.validate_on_submit(): links = Links(form.youtube.data, form.twitter.data, form.instagram.data, form.facebook.data) current_user.links = links current_user.save() return render_template('user_profile.html', form=form)
def add_url_to_db(url=None,sentiment=None): if url is None or sentiment is None: return None Session = sessionmaker(bind=engine) session = Session() url_id = get_salted_hash(url) url_text = url record = session.query(Links).filter_by(url_id=url_id).first() if record: record.sentiment = sentiment else: cc_url = Links(url_id=url_id,url_text=url_text,sentiment=sentiment) session.add(cc_url) session.commit() session.close()
def home(): if request.method == 'GET': return render_template('index.html') elif request.method == 'POST': if 'link' in request.form: link = Links.query.filter_by(original=request.form['link']).first() if link: pass else: link = Links(request.form['link']) db.session.add(link) db.session.commit() return render_template('index.html', link_name=str(link.id)) else: return "WHAT" else: return "GET OUTTA HERE"
def get_pagination(self): """ function parser the pagination xml file """ for pagination in self.meta: total = pagination.find('total').text count = pagination.find('count').text per_page = pagination.find('per_page').text current_page = pagination.find('current_page').text total_pages = pagination.find('total_pages').text links = [] for child in pagination: if child.tag == 'links': next = child.find('next').text links.append(Links(next)) return Pagination(total, count, per_page, current_page, total_pages, links)
def dashboard(unique_id): selected_link_info = None name = get_user_name(Users.query.filter_by(unique_id=unique_id).first()) if UserDashboard.query.filter_by(unique_id=unique_id).first() is not None: selected_link = "" obj = UserDashboard.query.filter_by(unique_id=unique_id).all() for o in obj: selected_link = o.short_url return redirect(url_for("dashboard_with_links", unique_id=unique_id, selected_link=selected_link, selected_link_info=selected_link_info, name=name)) check_create_button = False if request.method == "POST": check_create_button = True user = UserDashboard() user.unique_id = unique_id user.original_url = request.form["long_url"] user.title = request.form["long_url"] short_url = generate_short_link() user.short_url = short_url redirect_link = Links() redirect_link.original_url = user.original_url redirect_link.short_url = user.short_url db.session.add_all([redirect_link, user]) db.session.commit() return render_template("dashboard.html", flag=check_create_button, short_url=short_url, unique_id=unique_id, selected_link_info=selected_link_info, name=name) return render_template("dashboard.html", flag=check_create_button, unique_id=unique_id, selected_link_info=selected_link_info, name=name)
def save_to_db(self,const): # store in db, uses self.data Extract objects, iterate through and generate the appropriate injections for the db if const is "search_term": s_db = Search(date=timezone.now(),term=self.data[0].search_term) print "Adding %s data into db."% s_db s_db.save() for q in self.data: print q # save data around Search term for each Extract object in self.data # each Extract object has multiple links, get them all and associate to the created search term try: for url in q.job_urls: l_db = Links(search=s_db, link=url) l_db.save() # each Extract object has a single location, get it and associate it to search term if q.loc != "": loc_db = Location(city=q.city,state=q.state) loc_db.save() # each Extract object has a summary attribute that has all the data, modify the data pool to fit the parameters specified by user # and store the data in a Results table associated to its Search table summary = q.pool_summary(pos=self.pos, with_filter=self.with_filter, lower=self.lower, with_bigrams=self.with_bigrams) data = summary[('Word', 'Word_Count', 'POS_Tag')] for tup in data: w = str(tup[0]) c = tup[1] try: p = str(tup[2]) except IndexError: p = "" r_db = Results(search=s_db,location=loc_db,word=w,count=c,pos=p,is_bigram=self.with_bigrams) r_db.save() except: if q.loc != "": loc_db = Location(city=q.city,state=q.state) loc_db.save() r_db = Results(search=s_db,location=loc_db,word="N/A",count=0,pos="",is_bigram=False) r_db.save()
def getConcepts(category_id): """ This function renders the Concepts Template It lists concepts linked to the Category the user has clicked on """ categories = session.query(Category).filter_by( id=category_id).one_or_none() concepts = session.query(Concept).filter_by(category_id=category_id) links = session.query(Links).filter_by(category_id=category_id) if request.method == 'POST': newLink = Links(name=request.form['name'], link=request.form['link'], category_id=category_id) session.add(newLink) flash('New Link "%s" Successfully Added' % newLink.name) session.commit() return redirect(url_for('getConcepts', category_id=category_id)) else: return render_template('concepts.html', categories=categories, concepts=concepts, links=links, login_session=login_session)
def add_link(): # getting ip address of the user try: ip = request.environ['HTTP_X_FORWARDED_FOR'] # if behind a proxy except KeyError: ip = request.environ['REMOTE_ADDR'] ip_addr = IpAddresses.query.filter_by(ipAddress=ip).first() # if IP is in database then increment the count else add the IP in the database if ip_addr is not None: if ip_addr.count < 6: ip_addr.count += 1 else: ip_addr = IpAddresses(ip) ip_addr.count = 1 db.session.add(ip_addr) db.session.commit() # only if URL shortener requests are less than 4, run the shortener if ip_addr.count < 6: original_url = request.form['url'] # getting url from the form # adding https:// to the url if "https://" not in original_url and "http://" not in original_url: original_url = "https://" + original_url link = Links(original_url=original_url) db.session.add(link) db.session.commit() # sending success response to javascript file return jsonify({'result': 'success', 'new_link': link.short_url, 'long_link': link.original_url}) else: return redirect(url_for('index'))
#ADD STOPS TO DATABASE with open("stops.csv", "r") as file_to_read: csv_to_read = csv.reader(file_to_read) for row in csv_to_read: current_stop = Stops.query.filter_by(id=row[1]).first() if current_stop != None: continue new_stop = Stops(row[1], row[0], row[3], row[2]) db.session.add(new_stop) db.session.commit() #ADD LINKS TO DATABASE with open("stops.csv", "r") as file_to_read: csv_to_read = csv.reader(file_to_read) for row in csv_to_read: new_link = Links(generate_random_string(10, 20), row[1], row[5]) db.session.add(new_link) db.session.commit() #ADD EDGES TO DATABASE with open("edges.csv", "r") as file_to_read: csv_to_read = csv.reader(file_to_read) for row in csv_to_read: new_edge = Edges(generate_random_string(10, 20), row[0], row[1], int(row[2])) db.session.add(new_edge) db.session.commit() print("Finished")
def parse_indicators(summary, report_id, queue): ips = re.findall(IP_REGEX,summary) domains = re.findall(DOMAIN_REGEX,summary) attacks = re.findall(ATTACK_REGEX,summary) sha256 = re.findall(SHA_REGEX,summary) emails = re.findall(EMAIL_REGEX, summary) md5 = re.findall(MD5_REGEX, summary) sha512 = re.findall(SHA512_REGEX, summary) urls = re.findall(URL_REGEX, summary) cve = re.findall(CVE_REGEX, summary) org_id=User.query.filter_by(id=current_user.id).first().organization #get a list of indicator ids to link to report indicator_ids={} for match in ips: match_check = Indicators.query.filter_by(indicator=match).first() if match_check: indicator_ids[match_check.id]=match else: # Indicator doesn't exist, create new indicator and link new_indicator = Indicators(indicator=match,indicator_type='IP') add_db_entry(new_indicator) indicator_ids[new_indicator.id]=match for match in urls: match_check = Indicators.query.filter_by(indicator=match).first() if match_check: indicator_ids[match_check.id]=match else: # Indicator doesn't exist, create new indicator and link new_indicator = Indicators(indicator=match,indicator_type='URL') add_db_entry(new_indicator) indicator_ids[new_indicator.id]=match for match in md5: match_check = Indicators.query.filter_by(indicator=match).first() if match_check: indicator_ids[match_check.id]=match else: # Indicator doesn't exist, create new indicator and link new_indicator = Indicators(indicator=match,indicator_type='MD5 Hash') add_db_entry(new_indicator) indicator_ids[new_indicator.id]=match for match in domains: #so don't do as a domain if an email address if match not in emails + ips + urls: match_check = Indicators.query.filter_by(indicator=match).first() if match_check: indicator_ids[match_check.id]=match else: # Indicator doesn't exist, create new indicator and link new_indicator = Indicators(indicator=match,indicator_type='Domain') add_db_entry(new_indicator) indicator_ids[new_indicator.id]=match for match in attacks: match_check = Indicators.query.filter_by(indicator=match).first() if match_check: indicator_ids[match_check.id]=match else: # Indicator doesn't exist, create new indicator and link new_indicator = Indicators(indicator=match,indicator_type='MITRE ATT&CK Technique') add_db_entry(new_indicator) indicator_ids[new_indicator.id]=match for match in sha256: match_check = Indicators.query.filter_by(indicator=match).first() if match_check: indicator_ids[match_check.id]=match else: # Indicator doesn't exist, create new indicator and link new_indicator = Indicators(indicator=match,indicator_type='SHA256 Hash') add_db_entry(new_indicator) indicator_ids[new_indicator.id]=match for match in sha512: match_check = Indicators.query.filter_by(indicator=match).first() if match_check: indicator_ids[match_check.id]=match else: # Indicator doesn't exist, create new indicator and link new_indicator = Indicators(indicator=match,indicator_type='SHA512 Hash') add_db_entry(new_indicator) indicator_ids[new_indicator.id]=match for match in emails: match_check = Indicators.query.filter_by(indicator=match).first() if match_check: indicator_ids[match_check.id]=match else: # Indicator doesn't exist, create new indicator and link new_indicator = Indicators(indicator=match,indicator_type='Email') add_db_entry(new_indicator) indicator_ids[new_indicator.id]=match for match in cve: match_check = Indicators.query.filter_by(indicator=match).first() if match_check: indicator_ids[match_check.id]=match else: # Indicator doesn't exist, create new indicator and link new_indicator = Indicators(indicator=match,indicator_type='CVE') add_db_entry(new_indicator) indicator_ids[new_indicator.id]=match #consolidated all this down here for id in indicator_ids: match=indicator_ids[id] # Kickoff a task to enrich the new indicator job = queue.enqueue('main.enrich_pipeline', json.dumps({'indicator': str(match),'organization':org_id})) new_link=Links(indicator=id, report=report_id,kill_chain='Unknown',diamond_model='Unknown',confidence='Low') if not link_exists(new_link): add_db_entry(new_link)
def dashboard_with_links(unique_id, selected_link): # Getting all the info from the database name = get_user_name(Users.query.filter_by(unique_id=unique_id).first()) user_info = UserDashboard().query.filter_by(unique_id=unique_id).all() total_visits = 0 max_visits = -1 max_country_name = "" total_links = 0 labels = [] bar_chart_data = [] count_labels = [] # Getting info for creating chart for user in user_info: total_visits += user.visits total_links += 1 date = user.date_created.strftime("%d") month = user.date_created.strftime("%b") out = month + " " + date count_labels.append(out) if out not in labels: labels.append(out) if user.max_country_visit > max_visits: max_visits = user.max_country_visit max_country_name = user.max_country_visit_name count_labels_dict = Counter(count_labels) for label in count_labels_dict: bar_chart_data.append(count_labels_dict[label]) background_color = ['rgba(215, 146, 104, 1)'] * total_links # setting background color selected_link_info = UserDashboard().query.filter_by(short_url=selected_link).first() flag = request.args.get("flag") # if flag is true the it is request for editing the link if request.method == "POST" and flag == "True": try: user = UserDashboard.query.filter_by(short_url=selected_link).first() update_link = Links.query.filter_by(short_url=selected_link).first() customized_link = (request.form["short_url_customized"])[25:] if request.form["short_url_title"] != "": user.title = request.form["short_url_title"] user.short_url = customized_link update_link.short_url = customized_link db.session.commit() return redirect(url_for("dashboard", unique_id=unique_id, flag=False, selected_link_info=selected_link_info, bar_chart_data=bar_chart_data, labels=json.dumps(labels), background_color=background_color, name=name, total_visits=total_visits, max_visits=max_visits, max_country_name=max_country_name)) except BadRequestKeyError: return redirect(url_for("dashboard", unique_id=unique_id, flag=False, selected_link_info=selected_link_info, bar_chart_data=bar_chart_data, labels=json.dumps(labels), background_color=background_color, name=name, total_visits=total_visits, max_visits=max_visits, max_country_name=max_country_name)) check_create_button = False # Creating a new link if request.method == "POST" and flag != "True": check_create_button = True user = UserDashboard() user.unique_id = unique_id user.original_url = request.form["long_url"] user.title = request.form["long_url"] short_url = generate_short_link() user.short_url = short_url redirect_link = Links() redirect_link.original_url = user.original_url redirect_link.short_url = user.short_url db.session.add_all([redirect_link, user]) db.session.commit() return render_template("dashboard.html", flag=check_create_button, short_url=short_url, selected_link_info=selected_link_info, bar_chart_data=bar_chart_data, labels=json.dumps(labels), background_color=background_color, unique_id=unique_id, name=name, total_visits=total_visits, max_visits=max_visits, max_country_name=max_country_name) return render_template("dashboard.html", flag=check_create_button, unique_id=unique_id, user_info=user_info, selected_link_info=selected_link_info, bar_chart_data=bar_chart_data, labels=json.dumps(labels), background_color=background_color, name=name, total_visits=total_visits, max_visits=max_visits, max_country_name=max_country_name)