def dingtalk(tag, results): """ :param tag: :param results: :return: """ if len(results): hostname = setting_col.find_one({ 'key': 'dingtalk', 'enabled': True }).get('domain') webhook = setting_col.find_one({ 'key': 'dingtalk', 'enabled': True }).get('webhook') __content = { "msgtype": "markdown", "markdown": { "title": "GitHub泄露", "text": '#### [规则名称: {}]({}/view/tag/{})\n\n- {}'.format( tag, hostname, tag, '\n- '.join(results)) }, "at": { "atMobiles": [], "isAtAll": False } } requests.post(webhook, json=__content)
def post(self): parser = reqparse.RequestParser() parser.add_argument('page', type=int, default=1, help='') parser.add_argument('minute', type=int, default=10, help='') args = parser.parse_args() page = args.get('page') minute = args.get('minute') setting_col.update_many( {'key': 'task'}, {'$set': { 'key': 'task', 'page': page, 'minute': minute }}, upsert=True) try: os.kill( setting_col.find_one({ 'key': 'task' }).get('pid'), signal.SIGHUP) except ProcessLookupError: pass result = list(setting_col.find({}, {'_id': 0})) data = {'status': 201, 'msg': '设置成功', 'result': result} return jsonify(data)
def post(self): parser = reqparse.RequestParser() parser.add_argument('from', type=str, help='From (sender email)') parser.add_argument('host', type=str, help='SMTPServer Host') parser.add_argument('port', type=int, help='SMTPServer Port') parser.add_argument('tls', type=inputs.boolean, default=False, help='Force TLS') parser.add_argument('username', type=str, help='Username') parser.add_argument('password', type=str, help='Password') parser.add_argument('domain', type=str, help='System URL Host') parser.add_argument('enabled', type=inputs.boolean, default=False, help='Enabled Mail Notice') parser.add_argument('test', type=inputs.boolean, default=False, help='Test Mail Notice') args = parser.parse_args() __setting = args setting_col.update_many({'key': 'mail'}, {'$set': dict({'key': 'mail'}, **__setting)}, upsert=True) result = setting_col.find_one({'key': 'mail'}, {'_id': 0}) data = {'status': 201, 'msg': '设置成功', 'result': result} return jsonify(data)
def get(self): result = setting_col.find_one({'key': 'task'}, {'_id': 0}) if result: data = {'status': 200, 'msg': '获取信息成功', 'result': result} else: data = {'status': 400, 'msg': '请配置查询页数和周期', 'result': result} return jsonify(data)
def check(): setting_col.update_one({'key': 'task'}, {'$set': {'key': 'task', 'pid': os.getpid()}}, upsert=True) query_count = query_col.count({'enabled': True}) logger.info('需要处理的关键词总数: {}'.format(query_count)) if query_count: logger.info('需要处理的关键词总数: {}'.format(query_count)) else: logger.warning('请添加关键词') return if github_col.count({'rate_remaining': {'$gt': 5}}): pass else: logger.error('请配置github账号') return if setting_col.count({'key': 'task', 'page': {'$exists': True}}): setting_col.update_one({'key': 'task'}, {'$set': {'pid': os.getpid()}}) page = int(setting_col.find_one({'key': 'task'}).get('page')) for p in range(0, page): for query in query_col.find({'enabled': True}).sort('last', ASCENDING): github_account = random.choice( list(github_col.find({"rate_limit": {"$gt": 5}}).sort('rate_remaining', DESCENDING))) github_username = github_account.get('username') github_password = github_account.get('password') github_token = github_account.get('token') rate_remaining = github_account.get('rate_remaining') logger.info(github_username) logger.info(rate_remaining) g = Github(github_username, github_token, user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36') search.schedule(args=(query, p, g, github_username), delay=huey.pending_count() + huey.scheduled_count()) else: logger.error('请在页面上配置任务参数')
def send_mail(content): smtp_config = setting_col.find_one({'key': 'mail'}) receivers = [data.get('mail') for data in notice_col.find({})] try: if mail_notice(smtp_config, receivers, content): logger.info('邮件发送成功') else: logger.critical('Error: 无法发送邮件') except smtplib.SMTPException as error: logger.critical('Error: 无法发送邮件 {}'.format(error))
def post(self): parser = reqparse.RequestParser() parser.add_argument('webhook_url', type=str, help='webhook_url') parser.add_argument('enabled', type=inputs.boolean, default=False, help='enabled Notice') parser.add_argument('test', type=inputs.boolean, default=False, help='test') parser.add_argument('www_host', type=str, help='Hostname (for webhook notice link)') args = parser.parse_args() if args.get('test'): if not args.get('webhook_url'): data = {'status': 404, 'msg': '错误的webhook地址', 'result': []} return jsonify(data) test_content = { "msgtype": "markdown", "markdown": { "title": "GitHub泄露", "text": '### 规则名称: [钉钉告警测试]()' }, "at": { "atMobiles": [], "isAtAll": False } } response = requests.post(args.get('webhook_url'), json=test_content) print(response.json()) print(response.text()) print(response.ok) data = {'status': 201, 'msg': '已发送,请查收', 'result': []} return jsonify(data) webhook_setting = args setting_col.update_many( {'key': 'webhook'}, {'$set': dict({'key': 'webhook'}, **webhook_setting)}, upsert=True) print(webhook_setting) result = setting_col.find_one({'key': 'webhook'}, {'_id': 0}) data = {'status': 201, 'msg': '设置成功', 'result': result} return jsonify(data)
def send_mail(content): smtp_config = setting_col.find_one({'key': 'mail'}) receivers = [data.get('mail') for data in notice_col.find({})] elementcmd = "kmg imu zengshuai \'[GitHub警告]<br />" + content + "\'" try: if mail_notice(smtp_config, receivers, content): logger.info('邮件发送成功') else: os.system("kmg imu zengshuai 'haweye email Error'") logger.critical('Error: 无法发送邮件') except smtplib.SMTPException as error: logger.critical('Error: 无法发送邮件 {}'.format(error)) try: if os.system(elementcmd): logger.info('element 信息发送成功') else: os.system("kmg imu zengshuai 'haweye element Error'") loger.critical('Error: element信息发送失败')
def post(self): parser = reqparse.RequestParser() parser.add_argument('webhook', type=str, help='DingTalk WebHook URL') parser.add_argument('domain', type=str, help='System URL Host') parser.add_argument('enabled', type=inputs.boolean, default=False, help='Enabled DingTalk Notice') parser.add_argument('test', type=inputs.boolean, default=False, help='Test DingTalk Notice') args = parser.parse_args() __setting = args __setting['webhook'] = str(args.get('webhook')) if not (urlparse(__setting.get('webhook')).netloc == 'oapi.dingtalk.com' and urlparse( __setting.get('webhook')).scheme == 'https'): data = {'status': 400, 'msg': '错误的webhook地址', 'result': []} return jsonify(data) if args.get('test'): test_content = { "msgtype": "markdown", "markdown": {"title": "GitHub泄露", "text": '### 规则名称: [钉钉告警测试]()' }, "at": { "atMobiles": [ ], "isAtAll": False } } response = requests.post( args.get('webhook'), json=test_content) if response.ok: if response.json().get('errmsg') == 'ok': data = {'status': 201, 'msg': '已发送,请前往钉钉群查看', 'result': []} else: data = {'status': 400, 'msg': '发送失败,请检查webhook地址', 'result': []} return jsonify(data) else: data = {'status': 400, 'msg': '发送失败,请检查服务器网络', 'result': []} return jsonify(data) setting_col.update_many({'key': 'dingtalk'}, {'$set': dict({'key': 'dingtalk'}, **__setting)}, upsert=True) result = setting_col.find_one({'key': 'dingtalk'}, {'_id': 0}) data = {'status': 201, 'msg': '设置成功', 'result': result} return jsonify(data)
from utils.notice import mail_notice huey = RedisHuey('hawkeye', host=REDIS_HOST, port=int(REDIS_PORT)) base_path = os.path.split(os.path.realpath(__file__))[0] extract = tldextract.TLDExtract(cache_file='{}/.tld_set'.format(base_path)) if setting_col.count({ 'key': 'task', 'minute': { '$exists': True }, 'page': { '$exists': True } }): minute = int(setting_col.find_one({'key': 'task'}).get('minute')) setting_col.update_one( {'key': 'task'}, {'$set': { 'key': 'task', 'pid': os.getpid(), 'last': timestamp() }}, upsert=True) else: minute = 10 setting_col.update_one({'key': 'task'}, { '$set': { 'key': 'task', 'pid': os.getpid(),
def get(self): result = setting_col.find_one({'key': 'webhook'}, {'_id': 0}) data = {'status': 200, 'msg': '获取信息成功', 'result': result} return jsonify(data)
def get(self): parser = reqparse.RequestParser() parser.add_argument('tag', type=str, help='') args = parser.parse_args() tag = args.get('tag') if tag: total = { 'total': result_col.count({'tag': tag}), 'ignore': result_col.count({ 'tag': tag, 'security': 1 }), 'risk': result_col.count({ 'tag': tag, 'security': 0, "desc": { "$exists": True } }) } today = { 'total': result_col.count({ 'tag': tag, 'timestamp': { '$gte': today_start() } }), 'ignore': result_col.count({ 'tag': tag, 'timestamp': { '$gte': today_start() }, 'security': 1 }), 'risk': result_col.count({ 'tag': tag, 'timestamp': { '$gte': today_start() }, 'security': 0, "desc": { "$exists": True } }), } else: total = { 'total': result_col.count(), 'ignore': result_col.count({'security': 1}), 'risk': result_col.count({ 'security': 0, "desc": { "$exists": True } }) } today = { 'total': result_col.count({'timestamp': { '$gte': today_start() }}), 'ignore': result_col.count({ 'timestamp': { '$gte': today_start() }, 'security': 1 }), 'risk': result_col.count({ 'timestamp': { '$gte': today_start() }, 'security': 0, "desc": { "$exists": True } }), } if setting_col.count({'key': 'task'}): status = psutil.pid_exists( int(setting_col.find_one({ 'key': 'task' }).get('pid'))) last = setting_col.find_one({'key': 'task'}).get('last') else: status = False last = 0 engine = { 'status': status, 'last': last, } result = {'all': total, 'today': today, 'engine': engine} data = {'status': 200, 'msg': '获取信息成功', 'result': result} return jsonify(data)
def run(): # setting_col.update_one({'key': 'task'}, {'$set': {'key': 'task', 'pid': os.getpid()}}, upsert=True) query_count = query_col.count({'enabled': True}) logger.info('需要处理的关键词总数: {}'.format(query_count)) if query_count: logger.info('需要处理的关键词总数: {}'.format(query_count)) else: logger.warning('请添加关键词') return if github_col.count({'rate_remaining': {'$gt': 5}}): pass else: logger.error('请配置github账号') return if setting_col.count({'key': 'task', 'page': {'$exists': True}}): setting_col.update_one({'key': 'task'}, {'$set': {'pid': os.getpid()}}) page = int(setting_col.find_one({'key': 'task'}).get('page')) for p in range(0, page): for query in query_col.find({'enabled': True}).sort('last', 1): github_account = random.choice( list( github_col.find({ "rate_limit": { "$gt": 5 } }).sort('rate_remaining', -1))) github_username = github_account.get('username') github_password = github_account.get('password') rate_remaining = github_account.get('rate_remaining') logger.info(github_username) logger.info(rate_remaining) g = Github( github_username, github_password, per_page=PER_PAGE, user_agent= 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36' ) # total = query.get('total') # if total is None: # repos = g.search_code(query=query.get('keyword'), # sort="indexed", order="desc") # total = repos.totalCount api_total = query.get('api_total') if api_total: total = api_total else: repos = g.search_code(query=query.get('keyword'), sort="indexed", order="desc") total = repos.totalCount if total > 1000: total = 1000 page_pre = int(query.get('page_pre')) if query.get( 'page_pre') is not None else -1 page_all = math.ceil(total / 30) if page_all == 0: continue if page_pre + 1 >= page_all: page_pre = -1 page_now = page_pre + 1 search(query, page_now, g, github_username) else: logger.error('请在页面上配置任务参数')