class FetchContentSF: ''' 获取漏洞内容 包括 漏洞名称 CVE号 类型 最早公开时间 受影响的软件或系统 漏洞起因 安全建议 原始信息来源 ''' def __init__(self, vuls, proxy=False): self.vuls = vuls self.size = 6 self.proxy = proxy self.check = False self.check = Check() self.load() self.lock = Lock() self.pool = ThreadPool(self.size) def load(self): error_times = 0 LOG.pprint('+', 'load cncert cookie', GREEN) while error_times < 3: code = self.check.load() if code: break LOG.pprint('-', 'load cncert cookie failed, load again', RED) LOG.pprint('+', 'load cncert cookie success', GREEN) def setThreadSize(self, size): self.size = size def setCookie(self, cookie): self.cookie = cookie self.check = True def fetch(self): res = self.pool.map(self.fetchALL, self.vuls) self.pool.close() self.pool.join() return res def fetchALL(self, url): info = self._fetchInfo(url) info['url'] = url flag = info.has_key('CVE') and 'N-A' not in info['CVE'] code = -1000 if flag: if self.lock.acquire(1): code = self.check.check(info['CVE']) self.lock.release() if code == -1: LOG.pprint('-', '无法获取CNCERT cookie,请查看是否可以正常访问网站', RED) self.pool.terminate() sys.exit(0) try: if flag and info['CVE'] in CVEDB: LOG.pprint("-", "已存在CVE漏洞库中 " + url + ", 不载入,请人工确认", RED) elif flag and code > 0: LOG.pprint("-", "已存在CNCERT漏洞库中 " + url + ", 不载入", RED) elif flag and code == 0: discuss = self._fetchDiscussion(url) exploit = self._fetchExploit(url) solution = self._fetchSolution(url) company = self._fetchReferences(url) references = {'references': url} LOG.pprint("+", "fetch " + url + " details done", GREEN) return dict( dict( dict(dict(dict(info, **discuss), **exploit), **solution), **references), **company) else: LOG.pprint("-", "不存在CVE " + url + ", 不载入,请人工确认", RED) except KeyboardInterrupt: LOG.pprint('+', 'user stop', GREEN) self.pool.terminate() sys.exit(0) def _fetchInfo(self, url): url = url + "/info" HTTPCONTAINER.setHeaders(HEADERS) r = HTTPCONTAINER.get(url, self.proxy) pattern = re.compile( r'\s*<td>\s*<span class="label">([\s\S]+?):</span>\s*</td>\s*<td>\s*([\s\S]*?)\s*</td>' ) results = pattern.findall(r.content) res = {} for line in results: if line[0] == 'CVE': if line[1] == "": res['CVE'] = "N-A-" + str(int(random.random() * 1000)) else: res['CVE'] = line[1][:13] elif line[0] == 'Vulnerable': res['Vulnerable'] = '\t'.join(line[1].split('\t')) res['Vulnerable'] = '\n'.join(res['Vulnerable'].split('\n')) res['Vulnerable'] = (' '.join( res['Vulnerable'].split())).replace('<br/>', '\n') res['Vulnerable'] = re.sub('<[^<]+?>', '', res['Vulnerable']) pattern = re.compile(r'\s*([\S\s]+?)\s[0-9]{1}[\s\S]*?\n') product = list(set(pattern.findall(res['Vulnerable']))) res['product'] = "##".join(product).replace('+', '').strip() # print res['Vulnerable'] else: res[line[0]] = re.sub('<[^<]+?>', '', line[1]) return res def _fetchReferences(self, url): ''' 获取解决方案中的url 获取厂商名 :param url: :return: ''' url += "/references" HTTPCONTAINER.setHeaders(HEADERS) r = HTTPCONTAINER.get(url, self.proxy) content = r.content[r.content.index('<li class="here"><a href='):] #正则匹配 pattern = re.compile( r'<li><a href="([\s\S]+?)">([\s\S]+?)</a>\s*\([\s\S]+?\)\s*<br/></li>' ) results = pattern.findall(content) data = {"company": "", "solutionUrl": "", "companyUrl": ""} max = 0 for result in results: if "Homepage" in result[1] or "Home Page" in result[1]: data['company'] = result[1].strip("Homepage").strip( "Home Page").strip() data['companyUrl'] = result[0] else: if max < len(result[1].strip()): data['solutionUrl'] = result[0] max = len(result[1].strip()) return data def _fetchDiscussion(self, url): url = url + "/discuss" HTTPCONTAINER.setHeaders(HEADERS) r = HTTPCONTAINER.get(url, self.proxy) pattern = re.compile( r'<div id="vulnerability">\s*<span class="title">([\s\S]*?)</span><br/><br/>\s*([\s\S]*?)\s*</div>' ) results = pattern.findall(r.content)[0] temp = results[1].replace('<br/>', '') temp = ' '.join(temp.split()) return {"title": results[0], "discuss": temp} def _fetchExploit(self, url): url = url + "/exploit" HTTPCONTAINER.setHeaders(HEADERS) r = HTTPCONTAINER.get(url, self.proxy) pattern = re.compile( r'<div id="vulnerability">\s*<span class="title">[\s\S]*?</span><br/><br/>\s*([\s\S]*?)\s*</div>' ) res = pattern.findall(r.content) temp = res[0].replace('<br/>', '') temp = ' '.join(temp.split()) return {"exploit": temp} def _fetchSolution(self, url): url = url + "/solution" HTTPCONTAINER.setHeaders(HEADERS) r = HTTPCONTAINER.get(url, self.proxy) pattern = re.compile(r'<b>Solution:</b><br/>\s*([\s\S]*?)\s*</div>') res = pattern.findall(r.content) temp = res[0].replace('<br/>', '') temp = ' '.join(temp.split()) if "": pass return {"solution": temp}