def search(query, page, g, github_username): mail_notice_list = [] webhook_notice_list = [] logger.info('开始抓取: tag is {} keyword is {}, page is {}'.format( query.get('tag'), query.get('keyword'), page + 1)) try: repos = g.search_code(query=query.get('keyword'), sort="indexed", order="desc") github_col.update_one({'username': github_username}, { '$set': { 'rate_remaining': int(g.get_rate_limit().search.remaining) } }) except Exception as error: logger.critical(error) logger.critical("触发限制啦") return try: for repo in repos.get_page(page): setting_col.update_one({'key': 'task'}, { '$set': { 'key': 'task', 'pid': os.getpid(), 'last': timestamp() } }, upsert=True) if not result_col.count({'_id': repo.sha}): try: code = str(repo.content).replace('\n', '') except: code = '' leakage = { 'link': repo.html_url, 'project': repo.repository.full_name, 'project_url': repo.repository.html_url, '_id': repo.sha, 'language': repo.repository.language, 'username': repo.repository.owner.login, 'avatar_url': repo.repository.owner.avatar_url, 'filepath': repo.path, 'filename': repo.name, 'security': 0, 'ignore': 0, 'tag': query.get('tag'), 'code': code, } try: leakage['affect'] = get_affect_assets(repo.decoded_content) except Exception as error: logger.critical('{} {}'.format(error, leakage.get('link'))) leakage['affect'] = [] if int(repo.raw_headers.get('x-ratelimit-remaining')) == 0: logger.critical('剩余使用次数: {}'.format( repo.raw_headers.get('x-ratelimit-remaining'))) return last_modified = datetime.datetime.strptime( repo.last_modified, '%a, %d %b %Y %H:%M:%S %Z') leakage['datetime'] = last_modified leakage['timestamp'] = last_modified.timestamp() in_blacklist = False for blacklist in blacklist_col.find({}): if blacklist.get('text').lower() in leakage.get( 'link').lower(): logger.warning('{} 包含白名单中的 {}'.format( leakage.get('link'), blacklist.get('text'))) in_blacklist = True if in_blacklist: continue if result_col.count({ "project": leakage.get('project'), "ignore": 1 }): continue if not result_col.count({ "project": leakage.get('project'), "filepath": leakage.get("filepath"), "security": 0 }): mail_notice_list.append( '上传时间:{} 地址: <a href={}>{}/{}</a>'.format( leakage.get('datetime'), leakage.get('link'), leakage.get('project'), leakage.get('filename'))) webhook_notice_list.append('[{}/{}]({}) 上传于 {}'.format( leakage.get('project').split('.')[-1], leakage.get('filename'), leakage.get('link'), leakage.get('datetime'))) try: result_col.insert_one(leakage) logger.info(leakage.get('project')) except errors.DuplicateKeyError: logger.info('已存在') logger.info('抓取关键字:{} {}'.format(query.get('tag'), leakage.get('link'))) except Exception as error: if 'Not Found' not in error.data: g, github_username = new_github() search.schedule(args=(query, page, g, github_username), delay=huey.pending_count() + huey.scheduled_count()) logger.critical(error) logger.error('抓取: tag is {} keyword is {}, page is {} 失败'.format( query.get('tag'), query.get('keyword'), page + 1)) return logger.info('抓取: tag is {} keyword is {}, page is {} 成功'.format( query.get('tag'), query.get('keyword'), page + 1)) query_col.update_one({'tag': query.get('tag')}, { '$set': { 'last': int(time.time()), 'status': 1, 'reason': '抓取第{}页成功'.format(page), 'api_total': repos.totalCount, 'found_total': result_col.count({'tag': query.get('tag')}) } }) if setting_col.count({ 'key': 'mail', 'enabled': True }) and len(mail_notice_list): main_content = '<h2>规则名称: {}</h2><br>{}'.format( query.get('tag'), '<br>'.join(mail_notice_list)) send_mail(main_content) logger.info(len(webhook_notice_list)) webhook_notice(query.get('tag'), webhook_notice_list)
def crawl(query, page): mail_notice_list = [] webhook_notice_list = [] search_url = 'https://search.gitee.com/?skin=rec&type=code&q={1}&sort=last_indexed' \ '&pageno={0}' session = gitee_login() logger.info('Gitee开始抓取: tag is {} keyword is {}, page is {}'.format( query.get('tag'), query.get('keyword'), page + 1)) totalCount = 0 for page in range(page + 1, page + 2): try: logger.info("Gitee ------ 启动抓取: {}".format( search_url.format(page, query.get('keyword')))) resp = session.get(search_url.format(page, query.get('keyword'))) logger.info("Gitee 启动抓取: {}".format( search_url.format(page, query.get('keyword')))) tree = etree.HTML(resp.text) nodes = tree.xpath('//*[@id="hits-list"]/div[@class="item"]') for node in nodes: logger.info("Gitee 开始抓取节点") totalCount += 1 # i = nodes.index(node) + 1 leakage = {} leakage['affect'] = [] datetime_ = node.xpath(Gitee.DATETIME)[0].text # print(datetime) datetime_match = re.match("[^\d]*(?P<Date>\d+.*)", datetime_) if not datetime_match: leakage['datetime'] = _format_time( datetime.datetime.now().date()) else: leakage['datetime'] = _format_time( datetime_match.groups("Date")[0]) leakage['timestamp'] = leakage.get('datetime').timestamp() leakage['link'] = cut_tail( node.xpath(Gitee.LINK)[0].attrib['href']) leakage['filepath'] = node.xpath(Gitee.LINK)[0].text leakage['filename'] = leakage.get("filepath").split("/")[-1] # leakage['link'] = 'https://gitee.com' + realative_link leakage['_id'] = _md5(leakage['link']) logger.info("Gitee ****** 开始抓取节点 {}".format( leakage['datetime'])) project_username = node.xpath(Gitee.USERNAME)[0].text leakage["vendor"] = "GITEE" leakage['username'] = project_username.split("/")[0] leakage['project'] = project_username leakage['project_url'] = cut_tail( node.xpath(Gitee.USERNAME)[0].attrib['href']) logger.info("Gitee 抓取到 {}".format(leakage.get("project_url"))) if result_col.find_one({"link": leakage['link'], "datetime": leakage['datetime']}) or \ result_col.find_one({'_id': leakage['_id']}): continue in_blacklist = False for blacklist in blacklist_col.find({}): if blacklist.get('text').lower() in leakage.get( 'link').lower(): logger.warning('{} 包含白名单中的 {}'.format( leakage.get('link'), blacklist.get('text'))) in_blacklist = True if in_blacklist: continue if result_col.count({ "project": leakage.get('project'), "ignore": 1 }): continue #gitee中可以只有项目,没有代码 leakage['avatar_url'] = 'https://gitee.com/logo-black.svg' raw_code = gitee_raw_code(leakage['link']) leakage['code'] = base64.b64encode( raw_code.encode("utf-8")).decode("utf-8") try: leakage['affect'] = get_affect_assets(raw_code) except Exception as error: logger.critical('{} {}'.format(error, leakage.get('link'))) leakage['affect'] = [] leakage['tag'] = query['tag'] # leakage['detail'] = etree.tostring(node,encoding='unicode').replace('{{', '<<').\ # replace('}}', '>>') language_node = node.xpath(Gitee.LANGUAGE) if language_node: leakage['language'] = language_node[0].text.strip() else: leakage['language'] = 'Unknow' leakage['security'] = 0 leakage['ignore'] = 0 if not result_col.count({ "project": leakage.get('project'), "filepath": leakage.get("filepath"), "security": 0 }): mail_notice_list.append( '上传时间:{} 地址: <a href={}>{}/{}</a>'.format( leakage.get('datetime'), leakage.get('link'), leakage.get('project'), leakage.get('filename'))) webhook_notice_list.append('[{}/{}]({}) 上传于 {}'.format( leakage.get('project').split('/')[-1], leakage.get('filename'), leakage.get('link'), leakage.get('datetime'))) result_col.insert_one(leakage) logger.info("Gitee 抓取到的结果: {}".format( leakage.get("project_url"))) except Exception as e: raise (e) print(e) logger.error("Gitee error is {}".format(e)) return logger.info('Gitee抓取: tag is {} keyword is {}, page is {} 成功'.format( query.get('tag'), query.get('keyword'), page + 1)) query_col.update_one({'tag': query.get('tag')}, { '$set': { 'last': int(time.time()), 'status': 1, 'reason': '抓取第{}页成功'.format(page), 'api_total': totalCount, 'found_total': result_col.count({'tag': query.get('tag')}) } }) if setting_col.count({ 'key': 'mail', 'enabled': True }) and len(mail_notice_list): main_content = '<h2>规则名称: {}</h2><br>{}'.format( query.get('tag'), '<br>'.join(mail_notice_list)) send_mail(main_content) logger.info(len(webhook_notice_list)) webhook_notice(query.get('tag'), webhook_notice_list)