def dingtalk(tag, results):
"""
:param tag:
:param results:
:return:
"""
if len(results):
hostname = setting_col.find_one({
'key': 'dingtalk',
'enabled': True
}).get('domain')
webhook = setting_col.find_one({
'key': 'dingtalk',
'enabled': True
}).get('webhook')
__content = {
"msgtype": "markdown",
"markdown": {
"title":
"GitHub泄露",
"text":
'#### [规则名称: {}]({}/view/tag/{})\n\n- {}'.format(
tag, hostname, tag, '\n- '.join(results))
},
"at": {
"atMobiles": [],
"isAtAll": False
}
}
requests.post(webhook, json=__content)
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('page', type=int, default=1, help='')
parser.add_argument('minute', type=int, default=10, help='')
args = parser.parse_args()
page = args.get('page')
minute = args.get('minute')
setting_col.update_many(
{'key': 'task'},
{'$set': {
'key': 'task',
'page': page,
'minute': minute
}},
upsert=True)
try:
os.kill(
setting_col.find_one({
'key': 'task'
}).get('pid'), signal.SIGHUP)
except ProcessLookupError:
pass
result = list(setting_col.find({}, {'_id': 0}))
data = {'status': 201, 'msg': '设置成功', 'result': result}
return jsonify(data)
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('from', type=str, help='From (sender email)')
parser.add_argument('host', type=str, help='SMTPServer Host')
parser.add_argument('port', type=int, help='SMTPServer Port')
parser.add_argument('tls',
type=inputs.boolean,
default=False,
help='Force TLS')
parser.add_argument('username', type=str, help='Username')
parser.add_argument('password', type=str, help='Password')
parser.add_argument('domain', type=str, help='System URL Host')
parser.add_argument('enabled',
type=inputs.boolean,
default=False,
help='Enabled Mail Notice')
parser.add_argument('test',
type=inputs.boolean,
default=False,
help='Test Mail Notice')
args = parser.parse_args()
__setting = args
setting_col.update_many({'key': 'mail'},
{'$set': dict({'key': 'mail'}, **__setting)},
upsert=True)
result = setting_col.find_one({'key': 'mail'}, {'_id': 0})
data = {'status': 201, 'msg': '设置成功', 'result': result}
return jsonify(data)
def get(self):
result = setting_col.find_one({'key': 'task'}, {'_id': 0})
if result:
data = {'status': 200, 'msg': '获取信息成功', 'result': result}
else:
data = {'status': 400, 'msg': '请配置查询页数和周期', 'result': result}
return jsonify(data)
def check():
setting_col.update_one({'key': 'task'}, {'$set': {'key': 'task', 'pid': os.getpid()}}, upsert=True)
query_count = query_col.count({'enabled': True})
logger.info('需要处理的关键词总数: {}'.format(query_count))
if query_count:
logger.info('需要处理的关键词总数: {}'.format(query_count))
else:
logger.warning('请添加关键词')
return
if github_col.count({'rate_remaining': {'$gt': 5}}):
pass
else:
logger.error('请配置github账号')
return
if setting_col.count({'key': 'task', 'page': {'$exists': True}}):
setting_col.update_one({'key': 'task'}, {'$set': {'pid': os.getpid()}})
page = int(setting_col.find_one({'key': 'task'}).get('page'))
for p in range(0, page):
for query in query_col.find({'enabled': True}).sort('last', ASCENDING):
github_account = random.choice(
list(github_col.find({"rate_limit": {"$gt": 5}}).sort('rate_remaining', DESCENDING)))
github_username = github_account.get('username')
github_password = github_account.get('password')
github_token = github_account.get('token')
rate_remaining = github_account.get('rate_remaining')
logger.info(github_username)
logger.info(rate_remaining)
g = Github(github_username, github_token,
user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36')
search.schedule(args=(query, p, g, github_username),
delay=huey.pending_count() + huey.scheduled_count())
else:
logger.error('请在页面上配置任务参数')
def send_mail(content):
smtp_config = setting_col.find_one({'key': 'mail'})
receivers = [data.get('mail') for data in notice_col.find({})]
try:
if mail_notice(smtp_config, receivers, content):
logger.info('邮件发送成功')
else:
logger.critical('Error: 无法发送邮件')
except smtplib.SMTPException as error:
logger.critical('Error: 无法发送邮件 {}'.format(error))
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('webhook_url', type=str, help='webhook_url')
parser.add_argument('enabled',
type=inputs.boolean,
default=False,
help='enabled Notice')
parser.add_argument('test',
type=inputs.boolean,
default=False,
help='test')
parser.add_argument('www_host',
type=str,
help='Hostname (for webhook notice link)')
args = parser.parse_args()
if args.get('test'):
if not args.get('webhook_url'):
data = {'status': 404, 'msg': '错误的webhook地址', 'result': []}
return jsonify(data)
test_content = {
"msgtype": "markdown",
"markdown": {
"title": "GitHub泄露",
"text": '### 规则名称: [钉钉告警测试]()'
},
"at": {
"atMobiles": [],
"isAtAll": False
}
}
response = requests.post(args.get('webhook_url'),
json=test_content)
print(response.json())
print(response.text())
print(response.ok)
data = {'status': 201, 'msg': '已发送，请查收', 'result': []}
return jsonify(data)
webhook_setting = args
setting_col.update_many(
{'key': 'webhook'},
{'$set': dict({'key': 'webhook'}, **webhook_setting)},
upsert=True)
print(webhook_setting)
result = setting_col.find_one({'key': 'webhook'}, {'_id': 0})
data = {'status': 201, 'msg': '设置成功', 'result': result}
return jsonify(data)
def send_mail(content):
smtp_config = setting_col.find_one({'key': 'mail'})
receivers = [data.get('mail') for data in notice_col.find({})]
elementcmd = "kmg imu zengshuai \'[GitHub警告]<br />" + content + "\'"
try:
if mail_notice(smtp_config, receivers, content):
logger.info('邮件发送成功')
else:
os.system("kmg imu zengshuai 'haweye email Error'")
logger.critical('Error: 无法发送邮件')
except smtplib.SMTPException as error:
logger.critical('Error: 无法发送邮件 {}'.format(error))
try:
if os.system(elementcmd):
logger.info('element 信息发送成功')
else:
os.system("kmg imu zengshuai 'haweye element Error'")
loger.critical('Error: element信息发送失败')
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('webhook', type=str, help='DingTalk WebHook URL')
parser.add_argument('domain', type=str, help='System URL Host')
parser.add_argument('enabled', type=inputs.boolean, default=False, help='Enabled DingTalk Notice')
parser.add_argument('test', type=inputs.boolean, default=False, help='Test DingTalk Notice')
args = parser.parse_args()
__setting = args
__setting['webhook'] = str(args.get('webhook'))
if not (urlparse(__setting.get('webhook')).netloc == 'oapi.dingtalk.com' and urlparse(
__setting.get('webhook')).scheme == 'https'):
data = {'status': 400, 'msg': '错误的webhook地址', 'result': []}
return jsonify(data)
if args.get('test'):
test_content = {
"msgtype": "markdown",
"markdown": {"title": "GitHub泄露",
"text": '### 规则名称: [钉钉告警测试]()'
},
"at": {
"atMobiles": [
],
"isAtAll": False
}
}
response = requests.post(
args.get('webhook'),
json=test_content)
if response.ok:
if response.json().get('errmsg') == 'ok':
data = {'status': 201, 'msg': '已发送，请前往钉钉群查看', 'result': []}
else:
data = {'status': 400, 'msg': '发送失败，请检查webhook地址', 'result': []}
return jsonify(data)
else:
data = {'status': 400, 'msg': '发送失败，请检查服务器网络', 'result': []}
return jsonify(data)
setting_col.update_many({'key': 'dingtalk'}, {'$set': dict({'key': 'dingtalk'}, **__setting)},
upsert=True)
result = setting_col.find_one({'key': 'dingtalk'}, {'_id': 0})
data = {'status': 201, 'msg': '设置成功', 'result': result}
return jsonify(data)
from utils.notice import mail_notice
huey = RedisHuey('hawkeye', host=REDIS_HOST, port=int(REDIS_PORT))
base_path = os.path.split(os.path.realpath(__file__))[0]
extract = tldextract.TLDExtract(cache_file='{}/.tld_set'.format(base_path))
if setting_col.count({
'key': 'task',
'minute': {
'$exists': True
},
'page': {
'$exists': True
}
}):
minute = int(setting_col.find_one({'key': 'task'}).get('minute'))
setting_col.update_one(
{'key': 'task'},
{'$set': {
'key': 'task',
'pid': os.getpid(),
'last': timestamp()
}},
upsert=True)
else:
minute = 10
setting_col.update_one({'key': 'task'}, {
'$set': {
'key': 'task',
'pid': os.getpid(),
def get(self):
result = setting_col.find_one({'key': 'webhook'}, {'_id': 0})
data = {'status': 200, 'msg': '获取信息成功', 'result': result}
return jsonify(data)
def get(self):
parser = reqparse.RequestParser()
parser.add_argument('tag', type=str, help='')
args = parser.parse_args()
tag = args.get('tag')
if tag:
total = {
'total':
result_col.count({'tag': tag}),
'ignore':
result_col.count({
'tag': tag,
'security': 1
}),
'risk':
result_col.count({
'tag': tag,
'security': 0,
"desc": {
"$exists": True
}
})
}
today = {
'total':
result_col.count({
'tag': tag,
'timestamp': {
'$gte': today_start()
}
}),
'ignore':
result_col.count({
'tag': tag,
'timestamp': {
'$gte': today_start()
},
'security': 1
}),
'risk':
result_col.count({
'tag': tag,
'timestamp': {
'$gte': today_start()
},
'security': 0,
"desc": {
"$exists": True
}
}),
}
else:
total = {
'total':
result_col.count(),
'ignore':
result_col.count({'security': 1}),
'risk':
result_col.count({
'security': 0,
"desc": {
"$exists": True
}
})
}
today = {
'total':
result_col.count({'timestamp': {
'$gte': today_start()
}}),
'ignore':
result_col.count({
'timestamp': {
'$gte': today_start()
},
'security': 1
}),
'risk':
result_col.count({
'timestamp': {
'$gte': today_start()
},
'security': 0,
"desc": {
"$exists": True
}
}),
}
if setting_col.count({'key': 'task'}):
status = psutil.pid_exists(
int(setting_col.find_one({
'key': 'task'
}).get('pid')))
last = setting_col.find_one({'key': 'task'}).get('last')
else:
status = False
last = 0
engine = {
'status': status,
'last': last,
}
result = {'all': total, 'today': today, 'engine': engine}
data = {'status': 200, 'msg': '获取信息成功', 'result': result}
return jsonify(data)
def run():
# setting_col.update_one({'key': 'task'}, {'$set': {'key': 'task', 'pid': os.getpid()}}, upsert=True)
query_count = query_col.count({'enabled': True})
logger.info('需要处理的关键词总数: {}'.format(query_count))
if query_count:
logger.info('需要处理的关键词总数: {}'.format(query_count))
else:
logger.warning('请添加关键词')
return
if github_col.count({'rate_remaining': {'$gt': 5}}):
pass
else:
logger.error('请配置github账号')
return
if setting_col.count({'key': 'task', 'page': {'$exists': True}}):
setting_col.update_one({'key': 'task'}, {'$set': {'pid': os.getpid()}})
page = int(setting_col.find_one({'key': 'task'}).get('page'))
for p in range(0, page):
for query in query_col.find({'enabled': True}).sort('last', 1):
github_account = random.choice(
list(
github_col.find({
"rate_limit": {
"$gt": 5
}
}).sort('rate_remaining', -1)))
github_username = github_account.get('username')
github_password = github_account.get('password')
rate_remaining = github_account.get('rate_remaining')
logger.info(github_username)
logger.info(rate_remaining)
g = Github(
github_username,
github_password,
per_page=PER_PAGE,
user_agent=
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36'
)
# total = query.get('total')
# if total is None:
# repos = g.search_code(query=query.get('keyword'),
# sort="indexed", order="desc")
# total = repos.totalCount
api_total = query.get('api_total')
if api_total:
total = api_total
else:
repos = g.search_code(query=query.get('keyword'),
sort="indexed",
order="desc")
total = repos.totalCount
if total > 1000:
total = 1000
page_pre = int(query.get('page_pre')) if query.get(
'page_pre') is not None else -1
page_all = math.ceil(total / 30)
if page_all == 0:
continue
if page_pre + 1 >= page_all:
page_pre = -1
page_now = page_pre + 1
search(query, page_now, g, github_username)
else:
logger.error('请在页面上配置任务参数')
