def distance(): data = request.args.get('data', '') title1 = request.args.get('title1', '') title2 = request.args.get('title2', '') data = eval(data) print data res = {'title1': title1, 'title2': title2} nums = data['ids'].values() for num in nums: res[u'原差距'] = mongo_spider.sim.distance(data['num'], num) title_num1 = Simhash(title1).value - sys.maxint title_num2 = Simhash(title2).value - sys.maxint res[u'标题差距'] = mongo_spider.sim.distance(title_num1, title_num2) article = article_mongo.find_one({'_id': data['_id']}, { 'content': 1, 'title': 1 }) content_num1 = Simhash(article['content']).value - sys.maxint text_num = Simhash(article['title'] + default + html2text(article['content'])).value - sys.maxint res['content1'] = article['content'] cursors = article_mongo.find({"_id": { "$in": data['ids'].keys() }}, { 'content': 1, 'title': 1 }) for cursor in cursors: content_num2 = Simhash(cursor['content']).value - sys.maxint text_num2 = Simhash(cursor['title'] + default + html2text(cursor['content'])).value - sys.maxint res[u'正文差距'] = mongo_spider.sim.distance(content_num1, content_num2) res[u'新差距'] = mongo_spider.sim.distance(text_num2, text_num) res['content2'] = cursor['content'] d = render_html_diff(res['content1'], res['content2']) # result = list(d.compare(res['content1'], res['content2'])) # res['res'] = ''.join(result.spilt('+')) # from pprint import pprint as _pprint # _pprint(result) res['d'] = d return render_template('test/detail.html', data=res)
def article(self): score = len(html2text(self.content)) + len(self.imgs) * 50 if len(self.title) < 4 or score < 100: return None return { 'title':self.title, 'content':self.content, 'src_name':self.xsource['text'], 'src_link':self.xsource['link'], 'pubtime':self.pubtime, 'pages':self.pages, 'imgs':self.imgs, }
def article(self): score = len(html2text(self.content)) + len(self.imgs) * 50 if len(self.title) < 4 or score < 100: return None return { 'title': self.title, 'content': self.content, 'src_name': self.xsource['text'], 'src_link': self.xsource['link'], 'pubtime': self.pubtime, 'pages': self.pages, 'imgs': self.imgs, }
def tests(): data = request.args.get('data', '') assert data is not None res, data = {}, eval(data) article = article_mongo.find_one({'_id': data['_id']}, { 'content': 1, 'title': 1 }) cursors = article_mongo.find({"_id": { "$in": data['ids'].keys() }}, { 'content': 1, 'title': 1 }) for cursor in cursors: for i in xrange(1, 10): text_num1 = Simhash(default * i + article['title'] + html2text( article['content'])).value - sys.maxint text_num2 = Simhash(default * i + cursor['title'] + html2text( cursor['content'])).value - sys.maxint text_num3 = Simhash(article['title'] + default * i + html2text( article['content'])).value - sys.maxint text_num4 = Simhash(cursor['title'] + default * i + html2text( cursor['content'])).value - sys.maxint text_num5 = Simhash(article['title'] + html2text(article['content']) + default * i).value - sys.maxint text_num6 = Simhash(cursor['title'] + html2text(cursor['content']) + default * i).value - sys.maxint text_num7 = Simhash(default * i + article['title'] + default * i + html2text(article['content']) + default * i).value - sys.maxint text_num8 = Simhash(default * i + cursor['title'] + default * i + html2text(cursor['content']) + default * i).value - sys.maxint distance = mongo_spider.sim.distance res[i] = [ distance(text_num1, text_num2), distance(text_num3, text_num4), distance(text_num5, text_num6), distance(text_num7, text_num8) ] return render_template('test/tests.html', data=res)
def distance(): data = request.args.get('data', '') title1 = request.args.get('title1', '') title2 = request.args.get('title2', '') data = eval(data) print data res = {'title1':title1, 'title2':title2} nums = data['ids'].values() for num in nums: res[u'原差距'] = mongo_spider.sim.distance(data['num'], num) title_num1 = Simhash(title1).value - sys.maxint title_num2 = Simhash(title2).value - sys.maxint res[u'标题差距'] = mongo_spider.sim.distance(title_num1, title_num2) article = article_mongo.find_one({'_id':data['_id']}, {'content':1, 'title':1}) content_num1 = Simhash(article['content']).value - sys.maxint text_num = Simhash(article['title'] + default + html2text(article['content']) ).value - sys.maxint res['content1'] = article['content'] cursors = article_mongo.find({"_id":{"$in" : data['ids'].keys()}}, {'content':1, 'title':1}) for cursor in cursors: content_num2 = Simhash(cursor['content']).value - sys.maxint text_num2 = Simhash(cursor['title'] + default + html2text(cursor['content'])).value - sys.maxint res[u'正文差距'] = mongo_spider.sim.distance(content_num1, content_num2) res[u'新差距'] = mongo_spider.sim.distance(text_num2, text_num ) res['content2'] = cursor['content'] d = render_html_diff(res['content1'], res['content2']) # result = list(d.compare(res['content1'], res['content2'])) # res['res'] = ''.join(result.spilt('+')) # from pprint import pprint as _pprint # _pprint(result) res['d'] = d return render_template('test/detail.html', data=res)
def calc(): res = {'code': 1} id1 = request.args.get('_id1', '') id2 = request.args.get('_id2', '') if not id1 or not id2: res['msg'] = 'article id was null' return testjsonify(res) article1 = mongo_iweb.spider_article.find_one({'_id':id1}) article2 = mongo_iweb.spider_article.find_one({'_id':id2}) if not article1 or not article2: res['msg'] = 'article id was error' return testjsonify(res) try: text1 = article1['title'] + html2text(article1['content']) num1 = Simhash(text1).value - sys.maxint text2 = article2['title'] + html2text(article2['content']) num2 = Simhash(text2).value - sys.maxint distance = mongo_iweb.sim.distance(num1, num2) except Exception, e: res['msg'] = str(e) return testjsonify(res)
def sim(handler, key, article, ext): mongo = handler.mongo mongo.sim.unset(article['_id'], article['sim']) if 'content' not in ext or not ext['content']: ext['content'] = handler.text_file.get(article['content']) if not ext['content']: raise ValueError('load content failed.') ext['content'] = content.decode('utf-8') text = article['title'] + html2text(ext['content']) num = Simhash(text).value - sys.maxint near = mongo.sim.near(num) key = mongo.sim.save(article['_id'], num, near) return {'sim':key}
def segment(handler, key, article, ext): if 'content' not in ext or not ext['content']: ext['content'] = handler.text_file.get(article['content']) if not ext['content']: raise ValueError('load content failed.') ext['content'] = content.decode('utf-8') words = segmentor.seg(article['title'], html2text(ext['content'])) save_words(handler, article, words, ext) return { 'pubdate': article['pubdate'], 'id': article['id'], 'words': article['words'], 'tags': article['tags'], }
def segment(handler, key, article, ext): if 'content' not in ext or not ext['content']: ext['content'] = handler.text_file.get(article['content']) if not ext['content']: raise ValueError('load content failed.') ext['content'] = content.decode('utf-8') words = segmentor.seg(article['title'], html2text(ext['content'])) save_words(handler, article, words, ext) return { 'pubdate': article['pubdate'], 'id': article['id'], 'words': article['words'], 'tags': article['tags'], }
def sim(handler, key, article, ext): mongo = handler.mongo mongo.sim.unset(article['_id'], article['sim']) if 'content' not in ext or not ext['content']: ext['content'] = handler.text_file.get(article['content']) if not ext['content']: raise ValueError('load content failed.') ext['content'] = content.decode('utf-8') text = article['title'] + html2text(ext['content']) num = Simhash(text).value - sys.maxint near = mongo.sim.near(num) key = mongo.sim.save(article['_id'], num, near) return {'sim': key}
def article(self): try: score = len(html2text(self.content)) + len(self.imgs) * 50 except lxml.etree.ParserError: score = 0 if len(self.title) < 4 or score < 50: return None return { 'title': self.title, 'content': self.content, 'src_name': self.xsource['text'], 'src_link': self.xsource['link'], 'pubtime': self.pubtime, 'pages': self.pages, 'imgs': self.imgs, }
def article(self): try: score = len(html2text(self.content)) + len(self.imgs) * 50 except lxml.etree.ParserError: score = 0 if len(self.title) < 4 or score < 50: return None return { 'title':self.title, 'content':self.content, 'src_name':self.xsource['text'], 'src_link':self.xsource['link'], 'pubtime':self.pubtime, 'pages':self.pages, 'imgs':self.imgs, }
def get_page(self) -> Page: raw_page = self.get_raw_page() raw_posts = raw_page.find('article') if not raw_posts: logger.warning( "No raw posts (<article> elements) were found in this page.") if logger.isEnabledFor(logging.DEBUG): content = textwrap.indent( utils.html2text(raw_page.html), prefix='| ', predicate=lambda _: True, ) sep = '+' + '-' * 60 logger.debug("The page url is: %s", self.response.url) logger.debug("The page content is:\n%s\n%s%s\n", sep, content, sep) return raw_posts
def sim(article, content): while True: try: # if 'sim' in article and article['sim'] != False: # iweb.sim.unset(article['long'], article['sim']) content = html2text(content) if len(content) < 20: text = article['title'] + content else: text = content num = Simhash(text).value - sys.maxint near = iweb.sim.near(num) key = iweb.sim.save(article['long'], num, near) return key except (pymongo.errors.OperationFailure, IndexError), e: print str(e) gevent.sleep(1)
def sim(article, content): while True: try: # if 'sim' in article and article['sim'] != False: # iweb.sim.unset(article['long'], article['sim']) content = html2text(content) if len(content) < 20: text = article['title'] + content else: text = content num = Simhash(text).value - sys.maxint near = iweb.sim.near(num) key = iweb.sim.save(article['long'], num, near) return key except (pymongo.errors.OperationFailure, IndexError), e: print str(e) gevent.sleep(1)
def test_segment(url): url = url.split('#')[0].split('?')[0] if not url.startswith('http://'): return 'url is not startswith http://' add_test_url(url) html = get_or_cache(url) extractor = ArticleExtractor(html, url) content = extractor.content if extractor.pages: content = ArticleMerger( url, extractor.title, fetch_urls(extractor.pages, handle=get_or_cache), **extractor.selector ).content return json.dumps({ 'url':url, 'title': extractor.title, 'words': segmentor.seg(extractor.title, html2text(content)) })
def test_segment_all(): urls = get_test_urls() res = [] for url in urls: html = get_or_cache(url) extractor = ArticleExtractor(html, url) content = extractor.content if extractor.pages: content = ArticleMerger( url, extractor.title, fetch_urls(extractor.pages, handle=get_or_cache), **extractor.selector ).content res.append({ 'url':url, 'title': extractor.title, 'words': segmentor.seg(extractor.title, html2text(content, code=False)) }) return json.dumps(res)
def test_segment(url): url = url.split('#')[0].split('?')[0] if not url.startswith('http://'): return 'url is not startswith http://' add_test_url(url) html = get_or_cache(url) extractor = ArticleExtractor(html, url) content = extractor.content if extractor.pages: content = ArticleMerger( url, extractor.title, fetch_urls(extractor.pages, handle=get_or_cache), **extractor.selector).content return json.dumps({ 'url': url, 'title': extractor.title, 'words': segmentor.seg(extractor.title, html2text(content)) })
def test_segment_all(): urls = get_test_urls() res = [] for url in urls: html = get_or_cache(url) extractor = ArticleExtractor(html, url) content = extractor.content if extractor.pages: content = ArticleMerger( url, extractor.title, fetch_urls(extractor.pages, handle=get_or_cache), **extractor.selector).content res.append({ 'url': url, 'title': extractor.title, 'words': segmentor.seg(extractor.title, html2text(content, code=False)) }) return json.dumps(res)
def tests(): data = request.args.get('data', '') assert data is not None res, data = {}, eval(data) article = article_mongo.find_one({'_id':data['_id']}, {'content':1, 'title':1}) cursors = article_mongo.find({"_id":{"$in" : data['ids'].keys()}}, {'content':1, 'title':1}) for cursor in cursors: for i in xrange(1, 10): text_num1 = Simhash( default * i + article['title'] + html2text(article['content']) ).value - sys.maxint text_num2 = Simhash( default * i + cursor['title'] + html2text(cursor['content'])).value - sys.maxint text_num3 = Simhash(article['title'] + default * i + html2text(article['content']) ).value - sys.maxint text_num4 = Simhash(cursor['title'] + default * i + html2text(cursor['content'])).value - sys.maxint text_num5 = Simhash(article['title'] + html2text(article['content']) + default * i).value - sys.maxint text_num6 = Simhash(cursor['title'] + html2text(cursor['content']) + default * i).value - sys.maxint text_num7 = Simhash( default * i + article['title'] + default * i + html2text(article['content']) + default * i).value - sys.maxint text_num8 = Simhash( default * i + cursor['title'] + default * i + html2text(cursor['content']) + default * i).value - sys.maxint distance = mongo_spider.sim.distance res[i] = [distance(text_num1, text_num2), distance(text_num3, text_num4), distance(text_num5, text_num6), distance(text_num7, text_num8)] return render_template('test/tests.html', data=res)
def sendIssue(self, event): """ This sends selected issues(>=1) to DefectDojo be they selected from the DefectDojo Tab or the Context Menu in the Target Tab . Due to the current limitations in DefectDojo API request/response pairs cannot be added *yet* . """ checkMessage = self.checkSelection("test") if checkMessage: JOptionPane.showMessageDialog(self.getUiComponent().parent, checkMessage, "Error", JOptionPane.WARNING_MESSAGE) return if hasattr(self.ddui, 'userID'): pass else: self.getUserId() if event.getActionCommand() == 'Send To DefectDojo (Existing Test)': lgt = len(self.contextMenu._invoker.getSelectedIssues()) issues = self.contextMenu._invoker.getSelectedIssues() elif event.getActionCommand() == 'Send Issue': lgt = len(self.ddui._listTargetIss.getSelectedIndices()) issues = self.ddui._listTargetIss.getSelectedIndices() for i in range(lgt): ureqresp = [] if event.getActionCommand( ) == 'Send To DefectDojo (Existing Test)': title = issues[i].getIssueName() description = issues[i].getIssueDetail() if issues[ i].getIssueDetail() else issues[i].getIssueBackground() severity = issues[i].getSeverity() if severity == 'Information' or severity == 'Informational': severity = "Info" impact = issues[i].getIssueBackground() if issues[i].getRemediationBackground(): mitigation = issues[i].getRemediationBackground() + '\n' if issues[i].getRemediationDetail(): mitigation += issues[i].getRemediationDetail() else: mitigation = str(issues[i].getIssueType()) for mess in issues[i].getHttpMessages(): ureqresp.append({ "req": self._helpers.bytesToString(mess.getRequest()), "resp": self._helpers.bytesToString(mess.getResponse()) }) url = str(issues[i].getUrl()) description = html2text(description) impact = html2text(impact) mitigation = html2text(mitigation) try: json.loads(description) except: description = description.replace("\'", "") try: json.loads(impact) except: impact = impact.replace("\'", "") try: json.loads(mitigation) except: mitigation = mitigation.replace("\'", "") data = { 'title': title, 'description': description, 'severity': severity, 'product': '/api/v1/products/' + self._helpers.urlEncode(self.ddui.productID.getText()) + '/', 'engagement': '/api/v1/engagements/' + self._helpers.urlEncode(self.ddui.engagementID.getText()) + '/', 'reporter': '/api/v1/users/' + self._helpers.urlEncode(str(self.ddui.userID)) + '/', 'test': '/api/v1/tests/' + self._helpers.urlEncode(self.ddui.testID.getText()) + '/', 'impact': impact, 'active': True, 'verified': True, 'mitigation': mitigation, 'static_finding': False, 'dynamic_finding': False, 'file_path': url # 'steps_to_reproduce': ureqresp } data = json.dumps(data) self.checkUpdateSender() start_new_thread(self.sender.makeRequest, ('POST', '/api/v1/findings/', data)) message = str("Successfully imported (" + str(i + 1) + ") selected issue(s). Access Test : " + self.ddui.testID.getText()) link = str(self.ddui.defectDojoURL.getText() + "test/" + self.ddui.testID.getText()) linkDialog(message, link, JOptionPane, self.getUiComponent().parent)