def search(): query = request.form['query'] q = [] q.append(query) r = [] #complete path c = [] #preview of the paste content paste_date = [] paste_size = [] num_elem_to_get = 50 # Search filename for path in r_serv_pasteName.smembers(q[0]): r.append(path) paste = Paste.Paste(path) content = paste.get_p_content().decode('utf8', 'ignore') content_range = max_preview_char if len( content) > max_preview_char else len(content) - 1 c.append(content[0:content_range]) curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4] + '/' + curr_date[4:6] + '/' + curr_date[6:] paste_date.append(curr_date) paste_size.append(paste._get_p_size()) # Search full line from whoosh import index from whoosh.fields import Schema, TEXT, ID schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) indexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path")) ix = index.open_dir(indexpath) from whoosh.qparser import QueryParser with ix.searcher() as searcher: query = QueryParser("content", ix.schema).parse(" ".join(q)) results = searcher.search_page(query, 1, pagelen=num_elem_to_get) for x in results: r.append(x.items()[0][1]) paste = Paste.Paste(x.items()[0][1]) content = paste.get_p_content().decode('utf8', 'ignore') content_range = max_preview_char if len( content) > max_preview_char else len(content) - 1 c.append(content[0:content_range]) curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4] + '/' + curr_date[ 4:6] + '/' + curr_date[6:] paste_date.append(curr_date) paste_size.append(paste._get_p_size()) results = searcher.search(query) num_res = len(results) return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal, num_res=num_res)
def search(): query = request.form['query'] q = [] q.append(query) r = [] #complete path c = [] #preview of the paste content paste_date = [] paste_size = [] index_name = request.form['index_name'] num_elem_to_get = 50 # select correct index if index_name is None or index_name == "0": selected_index = get_current_index() else: selected_index = os.path.join(baseindexpath, index_name) # Search filename for path in r_serv_pasteName.smembers(q[0]): r.append(path) paste = Paste.Paste(path) content = paste.get_p_content().decode('utf8', 'ignore') content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 c.append(content[0:content_range]) curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] paste_date.append(curr_date) paste_size.append(paste._get_p_size()) # Search full line schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) ix = index.open_dir(selected_index) with ix.searcher() as searcher: query = QueryParser("content", ix.schema).parse(" ".join(q)) results = searcher.search_page(query, 1, pagelen=num_elem_to_get) for x in results: r.append(x.items()[0][1]) paste = Paste.Paste(x.items()[0][1]) content = paste.get_p_content().decode('utf8', 'ignore') content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 c.append(content[0:content_range]) curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] paste_date.append(curr_date) paste_size.append(paste._get_p_size()) results = searcher.search(query) num_res = len(results) index_min = 1 index_max = len(get_index_list()) return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal, num_res=num_res, index_min=index_min, index_max=index_max, index_list=get_index_list(selected_index) )
def showDiff(): s1 = request.args.get('s1', '') s2 = request.args.get('s2', '') p1 = Paste.Paste(s1) p2 = Paste.Paste(s2) maxLengthLine1 = p1.get_lines_info()[1] maxLengthLine2 = p2.get_lines_info()[1] if maxLengthLine1 > DiffMaxLineLength or maxLengthLine2 > DiffMaxLineLength: return "Can't make the difference as the lines are too long." htmlD = difflib.HtmlDiff() lines1 = p1.get_p_content().splitlines() lines2 = p2.get_p_content().splitlines() the_html = htmlD.make_file(lines1, lines2) return the_html
def post(self): key = self.request.get("key") if key != APP_KEY: self.error(404) return content = self.request.get("content") if content in (None,""): self.error(400) return title = self.request.get("title") mime = self.request.get("mime") or "application/octet-stream" keyName = str(uuid4()) #check if paste is a code if self.request.get("type") == "code": data = {"code":content} path = os.path.join(os.path.dirname(__file__), 'html/pasteShow.html') content = render(path, {'DATA':data}) mime = "text/html" Paste.Paste(title=title, content=db.Blob(str(content)),key_name=keyName, mime=mime).put() url = self.request.url[:self.request.url.index("/paste")] + "/" + keyName + "/paste" #Add a possible extension to the link ext = mimetypes.guess_extension(mime) if ext: url += ext self.response.out.write(json.dumps({"result":"success", "url":ShortLink.getShortUrl(url)})) self.response.out.write("\r\n")
def importantPasteByModule(): module_name = request.args.get('moduleName') all_content = [] paste_date = [] paste_linenum = [] all_path = [] for path in getPastebyType(r_serv_db, module_name): all_path.append(path) paste = Paste.Paste(path) content = paste.get_p_content().decode('utf8', 'ignore') content_range = max_preview_char if len( content) > max_preview_char else len(content) - 1 all_content.append(content[0:content_range]) curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4] + '/' + curr_date[4:6] + '/' + curr_date[6:] paste_date.append(curr_date) paste_linenum.append(paste.get_lines_info()[0]) return render_template("important_paste_by_module.html", all_path=all_path, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal)
def terms_management_query_paste(): term = request.args.get('term') paste_info = [] # check if regex or not if term.startswith('/') and term.endswith('/'): set_paste_name = "regex_" + term track_list_path = r_serv_term.smembers(set_paste_name) elif term.startswith('\\') and term.endswith('\\'): set_paste_name = "set_" + term track_list_path = r_serv_term.smembers(set_paste_name) else: set_paste_name = "tracked_" + term track_list_path = r_serv_term.smembers(set_paste_name) for path in track_list_path: paste = Paste.Paste(path) p_date = str(paste._get_p_date()) p_date = p_date[0:4]+'/'+p_date[4:6]+'/'+p_date[6:8] p_source = paste.p_source p_size = paste.p_size p_mime = paste.p_mime p_lineinfo = paste.get_lines_info() p_content = paste.get_p_content() if p_content != 0: p_content = p_content[0:400] paste_info.append({"path": path, "date": p_date, "source": p_source, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content}) return jsonify(paste_info)
def terms_management_query_paste(): term = request.args.get('term') TrackedTermsSet_Name = "TrackedSetTermSet" paste_info = [] set_paste_name = "tracked_" + term track_list_path = r_serv_term.smembers(set_paste_name) for path in track_list_path: paste = Paste.Paste(path) p_date = str(paste._get_p_date()) p_date = p_date[6:] + '/' + p_date[4:6] + '/' + p_date[0:4] p_source = paste.p_source p_encoding = paste._get_p_encoding() p_size = paste.p_size p_mime = paste.p_mime p_lineinfo = paste.get_lines_info() p_content = paste.get_p_content().decode('utf-8', 'ignore') if p_content != 0: p_content = p_content[0:400] paste_info.append({ "path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content }) return jsonify(paste_info)
def SuperMainCode(): while True: OperationChoice = input("Выберите операцию. Если вы не прочитали инструкции, введите \'--help \'. Если хотите выйти, введите \'exit\':\t\t") if OperationChoice == "--help": Functions.Info() elif OperationChoice == "cls": Functions.ClearConsole() elif OperationChoice == "exit": Functions.Exit() break elif OperationChoice == "1" or OperationChoice == "copy": Paste.CopyPaste() elif OperationChoice == "2" or OperationChoice == "create": CreateNew.CreateNewFile() elif OperationChoice == "3" or OperationChoice == "edit": EditFile.EditFile() elif OperationChoice == "4" or OperationChoice == "rename": RenameFile.RenameFile() elif OperationChoice == "5" or OperationChoice == "remove": RemoveFile.RemoveFile()
def credentials_management_query_paste(): cred = request.args.get('cred') allPath = request.json['allPath'] paste_info = [] for pathNum in allPath: path = r_serv_cred.hget(REDIS_KEY_ALL_PATH_SET_REV, pathNum) paste = Paste.Paste(path) p_date = str(paste._get_p_date()) p_date = p_date[6:] + '/' + p_date[4:6] + '/' + p_date[0:4] p_source = paste.p_source p_encoding = paste._get_p_encoding() p_size = paste.p_size p_mime = paste.p_mime p_lineinfo = paste.get_lines_info() p_content = paste.get_p_content().decode('utf-8', 'ignore') if p_content != 0: p_content = p_content[0:400] paste_info.append({ "path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content }) return jsonify(paste_info)
def importantPasteByModule(): module_name = request.args.get('moduleName') all_content = [] paste_date = [] paste_linenum = [] all_path = [] allPastes = getPastebyType(r_serv_db, module_name) for path in allPastes[0:10]: all_path.append(path) paste = Paste.Paste(path) content = paste.get_p_content().decode('utf8', 'ignore') content_range = max_preview_char if len( content) > max_preview_char else len(content) - 1 all_content.append(content[0:content_range].replace( "\"", "\'").replace("\r", " ").replace("\n", " ")) curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4] + '/' + curr_date[4:6] + '/' + curr_date[6:] paste_date.append(curr_date) paste_linenum.append(paste.get_lines_info()[0]) if len(allPastes) > 10: finished = False else: finished = True return render_template("important_paste_by_module.html", moduleName=module_name, all_path=all_path, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal, finished=finished)
def get_more_search_result(): query = request.form['query'] q = [] q.append(query) page_offset = int(request.form['page_offset']) index_name = request.form['index_name'] num_elem_to_get = 50 # select correct index if index_name is None or index_name == "0": selected_index = get_current_index() else: selected_index = os.path.join(baseindexpath, index_name) path_array = [] preview_array = [] date_array = [] size_array = [] schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) ix = index.open_dir(selected_index) with ix.searcher() as searcher: query = QueryParser("content", ix.schema).parse(" ".join(q)) results = searcher.search_page(query, page_offset, num_elem_to_get) for x in results: path_array.append(x.items()[0][1]) paste = Paste.Paste(x.items()[0][1]) content = paste.get_p_content().decode('utf8', 'ignore') content_range = max_preview_char if len( content) > max_preview_char else len(content) - 1 preview_array.append(content[0:content_range]) curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4] + '/' + curr_date[ 4:6] + '/' + curr_date[6:] date_array.append(curr_date) size_array.append(paste._get_p_size()) to_return = {} to_return["path_array"] = path_array to_return["preview_array"] = preview_array to_return["date_array"] = date_array to_return["size_array"] = size_array if len(path_array) < num_elem_to_get: #pagelength to_return["moreData"] = False else: to_return["moreData"] = True return jsonify(to_return)
def get_more_search_result(): query = request.form['query'] q = [] q.append(query) page_offset = int(request.form['page_offset']) num_elem_to_get = 50 path_array = [] preview_array = [] date_array = [] size_array = [] from whoosh import index from whoosh.fields import Schema, TEXT, ID schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) indexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path")) ix = index.open_dir(indexpath) from whoosh.qparser import QueryParser with ix.searcher() as searcher: query = QueryParser("content", ix.schema).parse(" ".join(q)) results = searcher.search_page(query, page_offset, num_elem_to_get) for x in results: path_array.append(x.items()[0][1]) paste = Paste.Paste(x.items()[0][1]) content = paste.get_p_content().decode('utf8', 'ignore') content_range = max_preview_char if len( content) > max_preview_char else len(content) - 1 preview_array.append(content[0:content_range]) curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4] + '/' + curr_date[ 4:6] + '/' + curr_date[6:] date_array.append(curr_date) size_array.append(paste._get_p_size()) to_return = {} to_return["path_array"] = path_array to_return["preview_array"] = preview_array to_return["date_array"] = date_array to_return["size_array"] = size_array print "len(path_array)=" + str(len(path_array)) if len(path_array) < num_elem_to_get: #pagelength to_return["moreData"] = False else: to_return["moreData"] = True return jsonify(to_return)
def event_stream_getImportantPasteByModule(module_name, year): index = 0 all_pastes_list = getPastebyType(r_serv_db[year], module_name) paste_tags = [] for path in all_pastes_list: index += 1 paste = Paste.Paste(path) content = paste.get_p_content() content_range = max_preview_char if len( content) > max_preview_char else len(content) - 1 curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4] + '/' + curr_date[4:6] + '/' + curr_date[6:] p_tags = r_serv_metadata.smembers('tag:' + path) l_tags = [] for tag in p_tags: complete_tag = tag.replace('"', '"') tag = tag.split('=') if len(tag) > 1: if tag[1] != '': tag = tag[1][1:-1] # no value else: tag = tag[0][1:-1] # use for custom tags else: tag = tag[0] l_tags.append((tag, complete_tag)) data = {} data["module"] = module_name data["index"] = index data["path"] = path data["content"] = content[0:content_range] data["linenum"] = paste.get_lines_info()[0] data["date"] = curr_date data["l_tags"] = l_tags data["bootstrap_label"] = bootstrap_label data["char_to_display"] = max_preview_modal data["finished"] = True if index == len(all_pastes_list) else False yield 'retry: 100000\ndata: %s\n\n' % json.dumps( data) #retry to avoid reconnection of the browser
def importantPasteByModule(): module_name = request.args.get('moduleName') # # TODO: VERIFY YEAR VALIDITY try: currentSelectYear = int(request.args.get('year')) except: print('Invalid year input') currentSelectYear = int(datetime.now().year) all_content = [] paste_date = [] paste_linenum = [] all_path = [] allPastes = getPastebyType(r_serv_db[currentSelectYear], module_name) for path in allPastes[0:10]: all_path.append(path) paste = Paste.Paste(path) content = paste.get_p_content() content_range = max_preview_char if len( content) > max_preview_char else len(content) - 1 all_content.append(content[0:content_range].replace( "\"", "\'").replace("\r", " ").replace("\n", " ")) curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4] + '/' + curr_date[4:6] + '/' + curr_date[6:] paste_date.append(curr_date) paste_linenum.append(paste.get_lines_info()[0]) if len(allPastes) > 10: finished = False else: finished = True return render_template("important_paste_by_module.html", moduleName=module_name, year=currentSelectYear, all_path=all_path, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal, finished=finished)
def event_stream_getImportantPasteByModule(module_name, year): index = 0 all_pastes_list = getPastebyType(r_serv_db[year], module_name) for path in all_pastes_list: index += 1 paste = Paste.Paste(path) content = paste.get_p_content().decode('utf8', 'ignore') content_range = max_preview_char if len( content) > max_preview_char else len(content) - 1 curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4] + '/' + curr_date[4:6] + '/' + curr_date[6:] data = {} data["module"] = module_name data["index"] = index data["path"] = path data["content"] = content[0:content_range] data["linenum"] = paste.get_lines_info()[0] data["date"] = curr_date data["char_to_display"] = max_preview_modal data["finished"] = True if index == len(all_pastes_list) else False yield 'retry: 100000\ndata: %s\n\n' % json.dumps( data) #retry to avoid reconnection of the browser
def Tags_page(): date_from = request.args.get('date_from') date_to = request.args.get('date_to') tags = request.args.get('ltags') if tags is None: dates = get_all_dates_range(date_from, date_to) return render_template("Tags.html", date_from=dates['date_from'], date_to=dates['date_to']) # unpack tags list_tags = tags.split(',') list_tag = [] for tag in list_tags: list_tag.append(tag.replace('"', '\"')) #no search by date, use last_seen for date_from/date_to if date_from is None and date_to is None and tags is not None: date_from = get_last_seen_from_tags_list(list_tags) date_to = date_from # TODO verify input dates = get_all_dates_range(date_from, date_to) if (type(list_tags) is list): # no tag if list_tags is False: print('empty') # 1 tag elif len(list_tags) < 2: tagged_pastes = [] for date in dates['date_range']: tagged_pastes.extend( r_serv_tags.smembers('{}:{}'.format(list_tags[0], date))) # 2 tags or more else: tagged_pastes = [] for date in dates['date_range']: tag_keys = [] for tag in list_tags: tag_keys.append('{}:{}'.format(tag, date)) if len(tag_keys) > 1: daily_items = r_serv_tags.sinter(tag_keys[0], *tag_keys[1:]) else: daily_items = r_serv_tags.sinter(tag_keys[0]) tagged_pastes.extend(daily_items) else: return 'INCORRECT INPUT' all_content = [] paste_date = [] paste_linenum = [] all_path = [] allPastes = list(tagged_pastes) paste_tags = [] try: page = int(request.args.get('page')) except: page = 1 if page <= 0: page = 1 nb_page_max = len(tagged_pastes) / (max_tags_result) if not nb_page_max.is_integer(): nb_page_max = int(nb_page_max) + 1 else: nb_page_max = int(nb_page_max) if page > nb_page_max: page = nb_page_max start = max_tags_result * (page - 1) stop = max_tags_result * page for path in allPastes[start:stop]: all_path.append(path) paste = Paste.Paste(path) content = paste.get_p_content() content_range = max_preview_char if len( content) > max_preview_char else len(content) - 1 all_content.append(content[0:content_range].replace( "\"", "\'").replace("\r", " ").replace("\n", " ")) curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4] + '/' + curr_date[4:6] + '/' + curr_date[6:] paste_date.append(curr_date) paste_linenum.append(paste.get_lines_info()[0]) p_tags = r_serv_metadata.smembers('tag:' + path) complete_tags = [] l_tags = [] for tag in p_tags: complete_tag = tag tag = tag.split('=') if len(tag) > 1: if tag[1] != '': tag = tag[1][1:-1] # no value else: tag = tag[0][1:-1] # use for custom tags else: tag = tag[0] l_tags.append((tag, complete_tag)) paste_tags.append(l_tags) if len(allPastes) > 10: finished = False else: finished = True if len(list_tag) == 1: tag_nav = tags.replace('"', '').replace('=', '').replace(':', '') else: tag_nav = 'empty' return render_template("Tags.html", all_path=all_path, tags=tags, tag_nav=tag_nav, list_tag=list_tag, date_from=dates['date_from'], date_to=dates['date_to'], page=page, nb_page_max=nb_page_max, paste_tags=paste_tags, bootstrap_label=bootstrap_label, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal, finished=finished)
def getmoredata(): requested_path = request.args.get('paste', '') paste = Paste.Paste(requested_path) p_content = paste.get_p_content() to_return = p_content[max_preview_modal - 1:] return to_return
def showsavedrawpaste(): requested_path = request.args.get('paste', '') paste = Paste.Paste(requested_path) content = paste.get_p_content() return Response(content, mimetype='text/plain')
def show_item_min(requested_path, content_range=0): relative_path = None if PASTES_FOLDER not in requested_path: relative_path = requested_path requested_path = os.path.join(PASTES_FOLDER, requested_path) else: relative_path = requested_path.replace(PASTES_FOLDER, '', 1) # remove old full path #requested_path = requested_path.replace(PASTES_FOLDER, '') # escape directory transversal if os.path.commonprefix( (os.path.realpath(requested_path), PASTES_FOLDER)) != PASTES_FOLDER: return 'path transversal detected' item_info = {} paste = Paste.Paste(requested_path) item_basic_info = get_item_basic_info(paste) item_info['nb_duplictates'] = paste.get_nb_duplicate() ## TODO: use this for fix ? item_content = paste.get_p_content() char_to_display = len(item_content) if content_range != 0: item_content = item_content[0:content_range] vt_enabled = Flask_config.vt_enabled p_hashtype_list = [] print(requested_path) l_tags = r_serv_metadata.smembers('tag:' + relative_path) if relative_path is not None: l_tags.union(r_serv_metadata.smembers('tag:' + relative_path)) item_info['tags'] = l_tags item_info['name'] = relative_path.replace('/', ' / ') l_64 = [] # load hash files if r_serv_metadata.scard('hash_paste:' + relative_path) > 0: set_b64 = r_serv_metadata.smembers('hash_paste:' + relative_path) for hash in set_b64: nb_in_file = r_serv_metadata.zscore('nb_seen_hash:' + hash, relative_path) # item list not updated if nb_in_file is None: l_pastes = r_serv_metadata.zrange('nb_seen_hash:' + hash, 0, -1) for paste_name in l_pastes: # dynamic update if PASTES_FOLDER in paste_name: score = r_serv_metadata.zscore( 'nb_seen_hash:{}'.format(hash), paste_name) r_serv_metadata.zrem('nb_seen_hash:{}'.format(hash), paste_name) paste_name = paste_name.replace(PASTES_FOLDER, '', 1) r_serv_metadata.zadd('nb_seen_hash:{}'.format(hash), score, paste_name) nb_in_file = r_serv_metadata.zscore( 'nb_seen_hash:{}'.format(hash), relative_path) nb_in_file = int(nb_in_file) estimated_type = r_serv_metadata.hget('metadata_hash:' + hash, 'estimated_type') file_type = estimated_type.split('/')[0] # set file icon if file_type == 'application': file_icon = 'fa-file ' elif file_type == 'audio': file_icon = 'fa-file-video ' elif file_type == 'image': file_icon = 'fa-file-image' elif file_type == 'text': file_icon = 'fa-file-alt' else: file_icon = 'fa-file' saved_path = r_serv_metadata.hget('metadata_hash:' + hash, 'saved_path') if r_serv_metadata.hexists('metadata_hash:' + hash, 'vt_link'): b64_vt = True b64_vt_link = r_serv_metadata.hget('metadata_hash:' + hash, 'vt_link') b64_vt_report = r_serv_metadata.hget('metadata_hash:' + hash, 'vt_report') else: b64_vt = False b64_vt_link = '' b64_vt_report = r_serv_metadata.hget('metadata_hash:' + hash, 'vt_report') # hash never refreshed if b64_vt_report is None: b64_vt_report = '' l_64.append((file_icon, estimated_type, hash, saved_path, nb_in_file, b64_vt, b64_vt_link, b64_vt_report)) crawler_metadata = {} if 'infoleak:submission="crawler"' in l_tags: crawler_metadata['get_metadata'] = True crawler_metadata['domain'] = r_serv_metadata.hget( 'paste_metadata:' + relative_path, 'domain') crawler_metadata['paste_father'] = r_serv_metadata.hget( 'paste_metadata:' + relative_path, 'father') crawler_metadata['real_link'] = r_serv_metadata.hget( 'paste_metadata:' + relative_path, 'real_link') crawler_metadata['screenshot'] = get_item_screenshot_path( relative_path) else: crawler_metadata['get_metadata'] = False misp_event = r_serv_metadata.get('misp_events:' + requested_path) if misp_event is None: misp_eventid = False misp_url = '' else: misp_eventid = True misp_url = misp_event_url + misp_event hive_case = r_serv_metadata.get('hive_cases:' + requested_path) if hive_case is None: hive_caseid = False hive_url = '' else: hive_caseid = True hive_url = hive_case_url.replace('id_here', hive_case) return render_template("show_saved_item_min.html", bootstrap_label=bootstrap_label, content=item_content, item_basic_info=item_basic_info, item_info=item_info, initsize=len(item_content), hashtype_list=p_hashtype_list, crawler_metadata=crawler_metadata, l_64=l_64, vt_enabled=vt_enabled, misp_eventid=misp_eventid, misp_url=misp_url, hive_caseid=hive_caseid, hive_url=hive_url)
def get_more_search_result(): query = request.form['query'] q = [] q.append(query) page_offset = int(request.form['page_offset']) index_name = request.form['index_name'] num_elem_to_get = 50 # select correct index if index_name is None or index_name == "0": selected_index = get_current_index() else: selected_index = os.path.join(baseindexpath, index_name) path_array = [] preview_array = [] date_array = [] size_array = [] list_tags = [] schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) ix = index.open_dir(selected_index) with ix.searcher() as searcher: query = QueryParser("content", ix.schema).parse(" ".join(q)) results = searcher.search_page(query, page_offset, num_elem_to_get) for x in results: path = x.items()[0][1] path = path.replace(PASTES_FOLDER, '', 1) path_array.append(path) paste = Paste.Paste(path) content = paste.get_p_content() content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 preview_array.append(content[0:content_range]) curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] date_array.append(curr_date) size_array.append(paste._get_p_size()) p_tags = r_serv_metadata.smembers('tag:'+path) l_tags = [] for tag in p_tags: complete_tag = tag tag = tag.split('=') if len(tag) > 1: if tag[1] != '': tag = tag[1][1:-1] # no value else: tag = tag[0][1:-1] # use for custom tags else: tag = tag[0] l_tags.append( (tag, complete_tag) ) list_tags.append(l_tags) to_return = {} to_return["path_array"] = path_array to_return["preview_array"] = preview_array to_return["date_array"] = date_array to_return["size_array"] = size_array to_return["list_tags"] = list_tags to_return["bootstrap_label"] = bootstrap_label if len(path_array) < num_elem_to_get: #pagelength to_return["moreData"] = False else: to_return["moreData"] = True return jsonify(to_return)
def search(): query = request.form['query'] q = [] q.append(query) r = [] #complete path c = [] #preview of the paste content paste_date = [] paste_size = [] paste_tags = [] index_name = request.form['index_name'] num_elem_to_get = 50 # select correct index if index_name is None or index_name == "0": selected_index = get_current_index() else: selected_index = os.path.join(baseindexpath, index_name) ''' temporary disabled # Search filename for path in r_serv_pasteName.smembers(q[0]): r.append(path) paste = Paste.Paste(path) content = paste.get_p_content() content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 c.append(content[0:content_range]) curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] paste_date.append(curr_date) paste_size.append(paste._get_p_size()) ''' # Search full line schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) ix = index.open_dir(selected_index) with ix.searcher() as searcher: query = QueryParser("content", ix.schema).parse("".join(q)) results = searcher.search_page(query, 1, pagelen=num_elem_to_get) for x in results: r.append(x.items()[0][1].replace(PASTES_FOLDER, '', 1)) path = x.items()[0][1].replace(PASTES_FOLDER, '', 1) paste = Paste.Paste(path) content = paste.get_p_content() content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 c.append(content[0:content_range]) curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] paste_date.append(curr_date) paste_size.append(paste._get_p_size()) p_tags = r_serv_metadata.smembers('tag:'+path) l_tags = [] for tag in p_tags: complete_tag = tag tag = tag.split('=') if len(tag) > 1: if tag[1] != '': tag = tag[1][1:-1] # no value else: tag = tag[0][1:-1] # use for custom tags else: tag = tag[0] l_tags.append( (tag, complete_tag) ) paste_tags.append(l_tags) results = searcher.search(query) num_res = len(results) index_list = get_index_list() index_min = 1 index_max = len(index_list) return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal, num_res=num_res, index_min=index_min, index_max=index_max, bootstrap_label=bootstrap_label, paste_tags=paste_tags, index_list=index_list )
def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, publish, path): paste = Paste.Paste(path) source = path.split('/')[-6:] source = '/'.join(source)[:-3] ail_uuid = r_serv_db.get('ail:uuid') pseudofile = BytesIO(paste.get_p_content().encode()) temp = paste._get_p_duplicate() #beautifier if not temp: temp = '' p_duplicate_number = len(temp) if len(temp) >= 0 else 0 to_ret = "" for dup in temp[:10]: dup = dup.replace('\'', '\"').replace('(', '[').replace(')', ']') dup = json.loads(dup) algo = dup[0] path = dup[1].split('/')[-6:] path = '/'.join(path)[:-3] # -3 removes .gz if algo == 'tlsh': perc = 100 - int(dup[2]) else: perc = dup[2] to_ret += "{}: {} [{}%]\n".format(path, algo, perc) p_duplicate = to_ret today = datetime.date.today() # [0-3] if publish == 'True': published = True else: published = False org_id = None orgc_id = None sharing_group_id = None date = today event = pymisp.new_event(distribution, threat_level_id, analysis, info, date, published, orgc_id, org_id, sharing_group_id) eventUuid = event['Event']['uuid'] eventid = event['Event']['id'] r_serv_metadata.set('misp_events:' + path, eventid) # add tags for tag in l_tags: pymisp.tag(eventUuid, tag) # create attributes obj_name = 'ail-leak' leak_obj = MISPObject(obj_name) leak_obj.add_attribute('sensor', value=ail_uuid, type="text") leak_obj.add_attribute('origin', value=source, type='text') leak_obj.add_attribute('last-seen', value=date_to_str(paste.p_date), type='datetime') leak_obj.add_attribute('raw-data', value=source, data=pseudofile, type="attachment") if p_duplicate_number > 0: leak_obj.add_attribute('duplicate', value=p_duplicate, type='text') leak_obj.add_attribute('duplicate_number', value=p_duplicate_number, type='counter') try: templateID = [ x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list()['response'] if x['ObjectTemplate']['name'] == obj_name ][0] except IndexError: valid_types = ", ".join([ x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list() ]) print("Template for type {} not found! Valid types are: {%s}".format( obj_name, valid_types)) r = pymisp.add_object(eventid, templateID, leak_obj) if 'errors' in r: print(r) return False else: event_url = misp_event_url + eventid return event_url
def importantPasteByModule(): module_name = request.args.get('moduleName') # # TODO: VERIFY YEAR VALIDITY try: currentSelectYear = int(request.args.get('year')) except: print('Invalid year input') currentSelectYear = int(datetime.now().year) all_content = [] paste_date = [] paste_linenum = [] all_path = [] paste_tags = [] allPastes = getPastebyType(r_serv_db[currentSelectYear], module_name) for path in allPastes[0:10]: all_path.append(path) paste = Paste.Paste(path) content = paste.get_p_content() content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " ")) curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] paste_date.append(curr_date) paste_linenum.append(paste.get_lines_info()[0]) p_tags = r_serv_metadata.smembers('tag:'+path) l_tags = [] for tag in p_tags: complete_tag = tag tag = tag.split('=') if len(tag) > 1: if tag[1] != '': tag = tag[1][1:-1] # no value else: tag = tag[0][1:-1] # use for custom tags else: tag = tag[0] l_tags.append( (tag, complete_tag) ) paste_tags.append(l_tags) if len(allPastes) > 10: finished = False else: finished = True return render_template("important_paste_by_module.html", moduleName=module_name, year=currentSelectYear, all_path=all_path, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal, paste_tags=paste_tags, bootstrap_label=bootstrap_label, finished=finished)
def showpaste(content_range, requested_path): if PASTES_FOLDER not in requested_path: # remove full path requested_path_full = os.path.join(requested_path, PASTES_FOLDER) else: requested_path_full = requested_path requested_path = requested_path.replace(PASTES_FOLDER, '', 1) # escape directory transversal if os.path.commonprefix( (requested_path_full, PASTES_FOLDER)) != PASTES_FOLDER: return 'path transversal detected' vt_enabled = Flask_config.vt_enabled paste = Paste.Paste(requested_path) p_date = str(paste._get_p_date()) p_date = p_date[6:] + '/' + p_date[4:6] + '/' + p_date[0:4] p_source = paste.p_source p_encoding = paste._get_p_encoding() p_language = 'None' p_size = paste.p_size p_mime = paste.p_mime p_lineinfo = paste.get_lines_info() p_content = paste.get_p_content() p_duplicate_str_full_list = paste._get_p_duplicate() p_duplicate_full_list = [] p_duplicate_list = [] p_simil_list = [] p_date_list = [] p_hashtype_list = [] for dup_list in p_duplicate_str_full_list: dup_list = dup_list[1:-1].replace('\'', '').replace(' ', '').split(',') if dup_list[0] == "tlsh": dup_list[2] = 100 - int(dup_list[2]) else: dup_list[2] = int(dup_list[2]) p_duplicate_full_list.append(dup_list) #p_duplicate_full_list.sort(lambda x,y: cmp(x[2], y[2]), reverse=True) # Combine multiple duplicate paste name and format for display new_dup_list = [] dup_list_removed = [] for dup_list_index in range(0, len(p_duplicate_full_list)): if dup_list_index in dup_list_removed: continue indices = [ i for i, x in enumerate(p_duplicate_full_list) if x[1] == p_duplicate_full_list[dup_list_index][1] ] hash_types = [] comp_vals = [] for i in indices: hash_types.append(p_duplicate_full_list[i][0]) comp_vals.append(p_duplicate_full_list[i][2]) dup_list_removed.append(i) #hash_types = str(hash_types).replace("[","").replace("]","") if len(hash_types)==1 else str(hash_types) #comp_vals = str(comp_vals).replace("[","").replace("]","") if len(comp_vals)==1 else str(comp_vals) if len(p_duplicate_full_list[dup_list_index]) > 3: try: date_paste = str(int(p_duplicate_full_list[dup_list_index][3])) date_paste = date_paste[0:4] + "-" + date_paste[ 4:6] + "-" + date_paste[6:8] except ValueError: date_paste = str(p_duplicate_full_list[dup_list_index][3]) else: date_paste = "No date available" new_dup_list.append([ hash_types, p_duplicate_full_list[dup_list_index][1], comp_vals, date_paste ]) # Create the list to pass to the webpage for dup_list in new_dup_list: hash_type, path, simil_percent, date_paste = dup_list p_duplicate_list.append(path) p_simil_list.append(simil_percent) p_hashtype_list.append(hash_type) p_date_list.append(date_paste) if content_range != 0: p_content = p_content[0:content_range] #active taxonomies active_taxonomies = r_serv_tags.smembers('active_taxonomies') l_tags = r_serv_metadata.smembers('tag:' + requested_path) #active galaxies active_galaxies = r_serv_tags.smembers('active_galaxies') list_tags = [] for tag in l_tags: if (tag[9:28] == 'automatic-detection'): automatic = True else: automatic = False if r_serv_statistics.sismember('tp:' + tag, requested_path): tag_status_tp = True else: tag_status_tp = False if r_serv_statistics.sismember('fp:' + tag, requested_path): tag_status_fp = True else: tag_status_fp = False list_tags.append((tag, automatic, tag_status_tp, tag_status_fp)) l_64 = [] # load hash files if r_serv_metadata.scard('hash_paste:' + requested_path) > 0: set_b64 = r_serv_metadata.smembers('hash_paste:' + requested_path) for hash in set_b64: nb_in_file = r_serv_metadata.zscore('nb_seen_hash:' + hash, requested_path) # item list not updated if nb_in_file is None: l_pastes = r_serv_metadata.zrange('nb_seen_hash:' + hash, 0, -1) for paste_name in l_pastes: # dynamic update if PASTES_FOLDER in paste_name: score = r_serv_metadata.zscore( 'nb_seen_hash:{}'.format(hash), paste_name) r_serv_metadata.zrem('nb_seen_hash:{}'.format(hash), paste_name) paste_name = paste_name.replace(PASTES_FOLDER, '', 1) r_serv_metadata.zadd('nb_seen_hash:{}'.format(hash), score, paste_name) nb_in_file = r_serv_metadata.zscore('nb_seen_hash:' + hash, requested_path) nb_in_file = int(nb_in_file) estimated_type = r_serv_metadata.hget('metadata_hash:' + hash, 'estimated_type') file_type = estimated_type.split('/')[0] # set file icon if file_type == 'application': file_icon = 'fa-file-o ' elif file_type == 'audio': file_icon = 'fa-file-video-o ' elif file_type == 'image': file_icon = 'fa-file-image-o' elif file_type == 'text': file_icon = 'fa-file-text-o' else: file_icon = 'fa-file' saved_path = r_serv_metadata.hget('metadata_hash:' + hash, 'saved_path') if r_serv_metadata.hexists('metadata_hash:' + hash, 'vt_link'): b64_vt = True b64_vt_link = r_serv_metadata.hget('metadata_hash:' + hash, 'vt_link') b64_vt_report = r_serv_metadata.hget('metadata_hash:' + hash, 'vt_report') else: b64_vt = False b64_vt_link = '' b64_vt_report = r_serv_metadata.hget('metadata_hash:' + hash, 'vt_report') # hash never refreshed if b64_vt_report is None: b64_vt_report = '' l_64.append((file_icon, estimated_type, hash, saved_path, nb_in_file, b64_vt, b64_vt_link, b64_vt_report)) crawler_metadata = {} if 'infoleak:submission="crawler"' in l_tags: crawler_metadata['get_metadata'] = True crawler_metadata['domain'] = r_serv_metadata.hget( 'paste_metadata:' + requested_path, 'domain') crawler_metadata['paste_father'] = r_serv_metadata.hget( 'paste_metadata:' + requested_path, 'father') crawler_metadata['real_link'] = r_serv_metadata.hget( 'paste_metadata:' + requested_path, 'real_link') crawler_metadata['screenshot'] = get_item_screenshot_path( requested_path) else: crawler_metadata['get_metadata'] = False if Flask_config.pymisp is False: misp = False else: misp = True if Flask_config.HiveApi is False: hive = False else: hive = True misp_event = r_serv_metadata.get('misp_events:' + requested_path) if misp_event is None: misp_eventid = False misp_url = '' else: misp_eventid = True misp_url = misp_event_url + misp_event hive_case = r_serv_metadata.get('hive_cases:' + requested_path) if hive_case is None: hive_caseid = False hive_url = '' else: hive_caseid = True hive_url = hive_case_url.replace('id_here', hive_case) return render_template("show_saved_paste.html", date=p_date, bootstrap_label=bootstrap_label, active_taxonomies=active_taxonomies, active_galaxies=active_galaxies, list_tags=list_tags, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list=p_duplicate_list, simil_list=p_simil_list, hashtype_list=p_hashtype_list, date_list=p_date_list, crawler_metadata=crawler_metadata, l_64=l_64, vt_enabled=vt_enabled, misp=misp, hive=hive, misp_eventid=misp_eventid, misp_url=misp_url, hive_caseid=hive_caseid, hive_url=hive_url)
def showpaste(content_range): requested_path = request.args.get('paste', '') paste = Paste.Paste(requested_path) p_date = str(paste._get_p_date()) p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4] p_source = paste.p_source p_encoding = paste._get_p_encoding() p_language = paste._get_p_language() p_size = paste.p_size p_mime = paste.p_mime p_lineinfo = paste.get_lines_info() p_content = paste.get_p_content() p_duplicate_str_full_list = paste._get_p_duplicate() p_duplicate_full_list = [] p_duplicate_list = [] p_simil_list = [] p_date_list = [] p_hashtype_list = [] for dup_list in p_duplicate_str_full_list: dup_list = dup_list[1:-1].replace('\'', '').replace(' ', '').split(',') if dup_list[0] == "tlsh": dup_list[2] = 100 - int(dup_list[2]) else: dup_list[2] = int(dup_list[2]) p_duplicate_full_list.append(dup_list) #p_duplicate_full_list.sort(lambda x,y: cmp(x[2], y[2]), reverse=True) # Combine multiple duplicate paste name and format for display new_dup_list = [] dup_list_removed = [] for dup_list_index in range(0, len(p_duplicate_full_list)): if dup_list_index in dup_list_removed: continue indices = [i for i, x in enumerate(p_duplicate_full_list) if x[1] == p_duplicate_full_list[dup_list_index][1]] hash_types = [] comp_vals = [] for i in indices: hash_types.append(p_duplicate_full_list[i][0]) comp_vals.append(p_duplicate_full_list[i][2]) dup_list_removed.append(i) #hash_types = str(hash_types).replace("[","").replace("]","") if len(hash_types)==1 else str(hash_types) #comp_vals = str(comp_vals).replace("[","").replace("]","") if len(comp_vals)==1 else str(comp_vals) if len(p_duplicate_full_list[dup_list_index]) > 3: try: date_paste = str(int(p_duplicate_full_list[dup_list_index][3])) date_paste = date_paste[0:4]+"-"+date_paste[4:6]+"-"+date_paste[6:8] except ValueError: date_paste = str(p_duplicate_full_list[dup_list_index][3]) else: date_paste = "No date available" new_dup_list.append([hash_types, p_duplicate_full_list[dup_list_index][1], comp_vals, date_paste]) # Create the list to pass to the webpage for dup_list in new_dup_list: hash_type, path, simil_percent, date_paste = dup_list p_duplicate_list.append(path) p_simil_list.append(simil_percent) p_hashtype_list.append(hash_type) p_date_list.append(date_paste) if content_range != 0: p_content = p_content[0:content_range] #active taxonomies active_taxonomies = r_serv_tags.smembers('active_taxonomies') l_tags = r_serv_metadata.smembers('tag:'+requested_path) #active galaxies active_galaxies = r_serv_tags.smembers('active_galaxies') list_tags = [] for tag in l_tags: if(tag[9:28] == 'automatic-detection'): automatic = True else: automatic = False tag_hash = ssdeep.hash(tag) if r_serv_statistics.sismember('tp:'+tag, requested_path): tag_status_tp = True else: tag_status_tp = False if r_serv_statistics.sismember('fp:'+tag, requested_path): tag_status_fp = True else: tag_status_fp = False list_tags.append( (tag, automatic, tag_status_tp, tag_status_fp) ) if Flask_config.pymisp is False: misp = False else: misp = True if Flask_config.HiveApi is False: hive = False else: hive = True misp_event = r_serv_metadata.get('misp_events:' + requested_path) if misp_event is None: misp_eventid = False misp_url = '' else: misp_eventid = True misp_url = misp_event_url + misp_event hive_case = r_serv_metadata.get('hive_cases:' + requested_path) if hive_case is None: hive_caseid = False hive_url = '' else: hive_caseid = True hive_url = hive_case_url.replace('id_here', hive_case) return render_template("show_saved_paste.html", date=p_date, bootstrap_label=bootstrap_label, active_taxonomies=active_taxonomies, active_galaxies=active_galaxies, list_tags=list_tags, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list, misp=misp, hive=hive, misp_eventid=misp_eventid, misp_url=misp_url, hive_caseid=hive_caseid, hive_url=hive_url)
def showsavedrawpaste(): requested_path = request.args.get('paste', '') paste = Paste.Paste(requested_path) content = paste.get_p_content() return content, 200, {'Content-Type': 'text/plain'}
def showpaste(content_range): requested_path = request.args.get('paste', '') paste = Paste.Paste(requested_path) p_date = str(paste._get_p_date()) p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4] p_source = paste.p_source p_encoding = paste._get_p_encoding() p_language = paste._get_p_language() p_size = paste.p_size p_mime = paste.p_mime p_lineinfo = paste.get_lines_info() p_content = paste.get_p_content().decode('utf-8', 'ignore') p_duplicate_full_list = json.loads(paste._get_p_duplicate()) p_duplicate_list = [] p_simil_list = [] p_date_list = [] p_hashtype_list = [] for dup_list in p_duplicate_full_list: if dup_list[0] == "tlsh": dup_list[2] = int(((tlsh_to_percent - float(dup_list[2])) / tlsh_to_percent)*100) else: dup_list[2] = int(dup_list[2]) p_duplicate_full_list.sort(lambda x,y: cmp(x[2], y[2]), reverse=True) # Combine multiple duplicate paste name and format for display new_dup_list = [] dup_list_removed = [] for dup_list_index in range(0, len(p_duplicate_full_list)): if dup_list_index in dup_list_removed: continue indices = [i for i, x in enumerate(p_duplicate_full_list) if x[1] == p_duplicate_full_list[dup_list_index][1]] hash_types = [] comp_vals = [] for i in indices: hash_types.append(p_duplicate_full_list[i][0].encode('utf8')) comp_vals.append(p_duplicate_full_list[i][2]) dup_list_removed.append(i) hash_types = str(hash_types).replace("[","").replace("]","") if len(hash_types)==1 else str(hash_types) comp_vals = str(comp_vals).replace("[","").replace("]","") if len(comp_vals)==1 else str(comp_vals) if len(p_duplicate_full_list[dup_list_index]) > 3: try: date_paste = str(int(p_duplicate_full_list[dup_list_index][3])) date_paste = date_paste[0:4]+"-"+date_paste[4:6]+"-"+date_paste[6:8] except ValueError: date_paste = str(p_duplicate_full_list[dup_list_index][3]) else: date_paste = "No date available" new_dup_list.append([hash_types.replace("'", ""), p_duplicate_full_list[dup_list_index][1], comp_vals, date_paste]) # Create the list to pass to the webpage for dup_list in new_dup_list: hash_type, path, simil_percent, date_paste = dup_list p_duplicate_list.append(path) p_simil_list.append(simil_percent) p_hashtype_list.append(hash_type) p_date_list.append(date_paste) if content_range != 0: p_content = p_content[0:content_range] return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list)
def get_tagged_paste(): tags = request.args.get('ltags') list_tags = tags.split(',') list_tag = [] for tag in list_tags: list_tag.append(tag.replace('"', '\"')) # TODO verify input if (type(list_tags) is list): # no tag if list_tags is False: print('empty') # 1 tag elif len(list_tags) < 2: tagged_pastes = r_serv_tags.smembers(list_tags[0]) # 2 tags or more else: tagged_pastes = r_serv_tags.sinter(list_tags[0], *list_tags[1:]) else: return 'INCORRECT INPUT' #TODO FIXME currentSelectYear = int(datetime.now().year) all_content = [] paste_date = [] paste_linenum = [] all_path = [] allPastes = list(tagged_pastes) paste_tags = [] for path in allPastes[0:50]: ######################moduleName all_path.append(path) paste = Paste.Paste(path) content = paste.get_p_content() content_range = max_preview_char if len( content) > max_preview_char else len(content) - 1 all_content.append(content[0:content_range].replace( "\"", "\'").replace("\r", " ").replace("\n", " ")) curr_date = str(paste._get_p_date()) curr_date = curr_date[0:4] + '/' + curr_date[4:6] + '/' + curr_date[6:] paste_date.append(curr_date) paste_linenum.append(paste.get_lines_info()[0]) p_tags = r_serv_metadata.smembers('tag:' + path) complete_tags = [] l_tags = [] for tag in p_tags: complete_tag = tag tag = tag.split('=') if len(tag) > 1: if tag[1] != '': tag = tag[1][1:-1] # no value else: tag = tag[0][1:-1] # use for custom tags else: tag = tag[0] l_tags.append((tag, complete_tag)) paste_tags.append(l_tags) if len(allPastes) > 10: finished = False else: finished = True return render_template("tagged.html", year=currentSelectYear, all_path=all_path, tags=tags, list_tag=list_tag, paste_tags=paste_tags, bootstrap_label=bootstrap_label, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal, finished=finished)