def check():
setting_col.update_one({'key': 'task'}, {'$set': {'key': 'task', 'pid': os.getpid()}}, upsert=True)
query_count = query_col.count({'enabled': True})
logger.info('需要处理的关键词总数: {}'.format(query_count))
if query_count:
logger.info('需要处理的关键词总数: {}'.format(query_count))
else:
logger.warning('请添加关键词')
return
if github_col.count({'rate_remaining': {'$gt': 5}}):
pass
else:
logger.error('请配置github账号')
return
if setting_col.count({'key': 'task', 'page': {'$exists': True}}):
setting_col.update_one({'key': 'task'}, {'$set': {'pid': os.getpid()}})
page = int(setting_col.find_one({'key': 'task'}).get('page'))
for p in range(0, page):
for query in query_col.find({'enabled': True}).sort('last', ASCENDING):
github_account = random.choice(
list(github_col.find({"rate_limit": {"$gt": 5}}).sort('rate_remaining', DESCENDING)))
github_username = github_account.get('username')
github_password = github_account.get('password')
github_token = github_account.get('token')
rate_remaining = github_account.get('rate_remaining')
logger.info(github_username)
logger.info(rate_remaining)
g = Github(github_username, github_token,
user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36')
search.schedule(args=(query, p, g, github_username),
delay=huey.pending_count() + huey.scheduled_count())
else:
logger.error('请在页面上配置任务参数')
def delete(self):
parser = reqparse.RequestParser()
parser.add_argument('_id', type=str, help='')
parser.add_argument('tag', type=str, help='')
args = parser.parse_args()
query_col.delete_many({'_id': args.get('_id')})
result_col.delete_many({'tag': args.get('tag')})
result = list(query_col.find({}).sort('enabled', -1))
data = {'status': 404, 'msg': '删除成功', 'result': result}
return jsonify(data)
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('keyword', type=str, help='')
parser.add_argument('tag', type=str, help='')
parser.add_argument('enabled', type=inputs.boolean, default=True, help='')
args = parser.parse_args()
if query_col.count({'tag': args.get('tag')}):
query_col.update_one({'tag': args.get('tag')}, {'$set': args})
msg = '更新成功'
else:
new_query = args
new_query['_id'] = md5(''.join([str(v) for v in new_query.values()]))
query_col.insert_one(new_query)
msg = '添加成功'
result = list(query_col.find({}).sort('enabled', -1))
data = {'status': 200, 'msg': msg, 'result': result}
return jsonify(data)
def get(self):
result = list(query_col.find({}).sort('enabled', -1))
data = {'status': 200, 'msg': '获取信息成功', 'result': result}
return jsonify(data)
def run():
# setting_col.update_one({'key': 'task'}, {'$set': {'key': 'task', 'pid': os.getpid()}}, upsert=True)
query_count = query_col.count({'enabled': True})
logger.info('需要处理的关键词总数: {}'.format(query_count))
if query_count:
logger.info('需要处理的关键词总数: {}'.format(query_count))
else:
logger.warning('请添加关键词')
return
if github_col.count({'rate_remaining': {'$gt': 5}}):
pass
else:
logger.error('请配置github账号')
return
if setting_col.count({'key': 'task', 'page': {'$exists': True}}):
setting_col.update_one({'key': 'task'}, {'$set': {'pid': os.getpid()}})
page = int(setting_col.find_one({'key': 'task'}).get('page'))
for p in range(0, page):
for query in query_col.find({'enabled': True}).sort('last', 1):
github_account = random.choice(
list(
github_col.find({
"rate_limit": {
"$gt": 5
}
}).sort('rate_remaining', -1)))
github_username = github_account.get('username')
github_password = github_account.get('password')
rate_remaining = github_account.get('rate_remaining')
logger.info(github_username)
logger.info(rate_remaining)
g = Github(
github_username,
github_password,
per_page=PER_PAGE,
user_agent=
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36'
)
# total = query.get('total')
# if total is None:
# repos = g.search_code(query=query.get('keyword'),
# sort="indexed", order="desc")
# total = repos.totalCount
api_total = query.get('api_total')
if api_total:
total = api_total
else:
repos = g.search_code(query=query.get('keyword'),
sort="indexed",
order="desc")
total = repos.totalCount
if total > 1000:
total = 1000
page_pre = int(query.get('page_pre')) if query.get(
'page_pre') is not None else -1
page_all = math.ceil(total / 30)
if page_all == 0:
continue
if page_pre + 1 >= page_all:
page_pre = -1
page_now = page_pre + 1
search(query, page_now, g, github_username)
else:
logger.error('请在页面上配置任务参数')
