def post(self):
parser = reqparse.RequestParser()
parser.add_argument('username', type=str, help='')
parser.add_argument('password', type=str, help='')
parser.add_argument('token', type=str, help='')
args = parser.parse_args()
username = args.get('username')
password = args.get('password')
token = args.get('token')
try:
g = Github(username, token)
github_col.save({
'_id':
md5(username),
'username':
username,
'password':
password,
'token':
token,
'mask_password':
password.replace(''.join(password[2:-2]), '****'),
'addat':
timestamp(),
'rate_limit':
int(g.get_rate_limit().search.limit),
'rate_remaining':
int(g.get_rate_limit().search.remaining)
})
result = list(github_col.find({}, {'_id': 0}))
data = {'status': 201, 'msg': '添加成功', 'result': result}
except BadCredentialsException:
data = {'status': 401, 'msg': '认证失败，请检查账号是否可用', 'result': []}
return jsonify(data)
def check():
setting_col.update_one({'key': 'task'}, {'$set': {'key': 'task', 'pid': os.getpid()}}, upsert=True)
query_count = query_col.count({'enabled': True})
logger.info('需要处理的关键词总数: {}'.format(query_count))
if query_count:
logger.info('需要处理的关键词总数: {}'.format(query_count))
else:
logger.warning('请添加关键词')
return
if github_col.count({'rate_remaining': {'$gt': 5}}):
pass
else:
logger.error('请配置github账号')
return
if setting_col.count({'key': 'task', 'page': {'$exists': True}}):
setting_col.update_one({'key': 'task'}, {'$set': {'pid': os.getpid()}})
page = int(setting_col.find_one({'key': 'task'}).get('page'))
for p in range(0, page):
for query in query_col.find({'enabled': True}).sort('last', ASCENDING):
github_account = random.choice(
list(github_col.find({"rate_limit": {"$gt": 5}}).sort('rate_remaining', DESCENDING)))
github_username = github_account.get('username')
github_password = github_account.get('password')
github_token = github_account.get('token')
rate_remaining = github_account.get('rate_remaining')
logger.info(github_username)
logger.info(rate_remaining)
g = Github(github_username, github_token,
user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36')
search.schedule(args=(query, p, g, github_username),
delay=huey.pending_count() + huey.scheduled_count())
else:
logger.error('请在页面上配置任务参数')
def delete(self):
parser = reqparse.RequestParser()
parser.add_argument('username', type=str, help='')
args = parser.parse_args()
github_col.delete_many({'username': args.get('username')})
result = list(github_col.find({}, {'_id': 0}))
data = {'status': 404, 'msg': '删除成功', 'result': result}
return jsonify(data)
def update_rate_remain():
for account in github_col.find():
github_username = account.get('username')
github_password = account.get('password')
try:
g = Github(github_username, github_password)
github_col.update_one({'username': github_username},
{'$set': {'rate_remaining': int(g.get_rate_limit().search.remaining),
'rate_limit': int(g.get_rate_limit().search.limit)}})
except Exception as error:
logger.error(error)
def new_github():
if github_col.count({'rate_remaining': {'$gt': 5}}):
pass
else:
logger.error('请配置github账号')
return
github_account = list(github_col.find().sort('rate_remaining',
DESCENDING))[0]
github_username = github_account.get('username')
github_password = github_account.get('password')
g = Github(github_username, github_password)
return g, github_username
def new_github():
if github_col.count({'rate_remaining': {'$gt': 5}}):
pass
else:
logger.error('请配置github账号')
return
github_account = random.choice(list(github_col.find({"rate_limit": {"$gt": 5}}).sort('rate_remaining', DESCENDING)))
github_username = github_account.get('username')
github_password = github_account.get('password')
github_token = github_account.get('token')
g = Github(github_username, github_token)
return g, github_username
def get(self):
result = list(github_col.find({}, {'_id': 0, 'password': 0}))
data = {'status': 200, 'msg': '获取信息成功', 'result': result}
return jsonify(data)
def run():
# setting_col.update_one({'key': 'task'}, {'$set': {'key': 'task', 'pid': os.getpid()}}, upsert=True)
query_count = query_col.count({'enabled': True})
logger.info('需要处理的关键词总数: {}'.format(query_count))
if query_count:
logger.info('需要处理的关键词总数: {}'.format(query_count))
else:
logger.warning('请添加关键词')
return
if github_col.count({'rate_remaining': {'$gt': 5}}):
pass
else:
logger.error('请配置github账号')
return
if setting_col.count({'key': 'task', 'page': {'$exists': True}}):
setting_col.update_one({'key': 'task'}, {'$set': {'pid': os.getpid()}})
page = int(setting_col.find_one({'key': 'task'}).get('page'))
for p in range(0, page):
for query in query_col.find({'enabled': True}).sort('last', 1):
github_account = random.choice(
list(
github_col.find({
"rate_limit": {
"$gt": 5
}
}).sort('rate_remaining', -1)))
github_username = github_account.get('username')
github_password = github_account.get('password')
rate_remaining = github_account.get('rate_remaining')
logger.info(github_username)
logger.info(rate_remaining)
g = Github(
github_username,
github_password,
per_page=PER_PAGE,
user_agent=
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36'
)
# total = query.get('total')
# if total is None:
# repos = g.search_code(query=query.get('keyword'),
# sort="indexed", order="desc")
# total = repos.totalCount
api_total = query.get('api_total')
if api_total:
total = api_total
else:
repos = g.search_code(query=query.get('keyword'),
sort="indexed",
order="desc")
total = repos.totalCount
if total > 1000:
total = 1000
page_pre = int(query.get('page_pre')) if query.get(
'page_pre') is not None else -1
page_all = math.ceil(total / 30)
if page_all == 0:
continue
if page_pre + 1 >= page_all:
page_pre = -1
page_now = page_pre + 1
search(query, page_now, g, github_username)
else:
logger.error('请在页面上配置任务参数')
