def post(self): parser = reqparse.RequestParser() parser.add_argument('username', type=str, help='') parser.add_argument('password', type=str, help='') parser.add_argument('token', type=str, help='') args = parser.parse_args() username = args.get('username') password = args.get('password') token = args.get('token') try: g = Github(username, token) github_col.save({ '_id': md5(username), 'username': username, 'password': password, 'token': token, 'mask_password': password.replace(''.join(password[2:-2]), '****'), 'addat': timestamp(), 'rate_limit': int(g.get_rate_limit().search.limit), 'rate_remaining': int(g.get_rate_limit().search.remaining) }) result = list(github_col.find({}, {'_id': 0})) data = {'status': 201, 'msg': '添加成功', 'result': result} except BadCredentialsException: data = {'status': 401, 'msg': '认证失败,请检查账号是否可用', 'result': []} return jsonify(data)
def check(): setting_col.update_one({'key': 'task'}, {'$set': {'key': 'task', 'pid': os.getpid()}}, upsert=True) query_count = query_col.count({'enabled': True}) logger.info('需要处理的关键词总数: {}'.format(query_count)) if query_count: logger.info('需要处理的关键词总数: {}'.format(query_count)) else: logger.warning('请添加关键词') return if github_col.count({'rate_remaining': {'$gt': 5}}): pass else: logger.error('请配置github账号') return if setting_col.count({'key': 'task', 'page': {'$exists': True}}): setting_col.update_one({'key': 'task'}, {'$set': {'pid': os.getpid()}}) page = int(setting_col.find_one({'key': 'task'}).get('page')) for p in range(0, page): for query in query_col.find({'enabled': True}).sort('last', ASCENDING): github_account = random.choice( list(github_col.find({"rate_limit": {"$gt": 5}}).sort('rate_remaining', DESCENDING))) github_username = github_account.get('username') github_password = github_account.get('password') github_token = github_account.get('token') rate_remaining = github_account.get('rate_remaining') logger.info(github_username) logger.info(rate_remaining) g = Github(github_username, github_token, user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36') search.schedule(args=(query, p, g, github_username), delay=huey.pending_count() + huey.scheduled_count()) else: logger.error('请在页面上配置任务参数')
def delete(self): parser = reqparse.RequestParser() parser.add_argument('username', type=str, help='') args = parser.parse_args() github_col.delete_many({'username': args.get('username')}) result = list(github_col.find({}, {'_id': 0})) data = {'status': 404, 'msg': '删除成功', 'result': result} return jsonify(data)
def update_rate_remain(): for account in github_col.find(): github_username = account.get('username') github_password = account.get('password') try: g = Github(github_username, github_password) github_col.update_one({'username': github_username}, {'$set': {'rate_remaining': int(g.get_rate_limit().search.remaining), 'rate_limit': int(g.get_rate_limit().search.limit)}}) except Exception as error: logger.error(error)
def new_github(): if github_col.count({'rate_remaining': {'$gt': 5}}): pass else: logger.error('请配置github账号') return github_account = list(github_col.find().sort('rate_remaining', DESCENDING))[0] github_username = github_account.get('username') github_password = github_account.get('password') g = Github(github_username, github_password) return g, github_username
def new_github(): if github_col.count({'rate_remaining': {'$gt': 5}}): pass else: logger.error('请配置github账号') return github_account = random.choice(list(github_col.find({"rate_limit": {"$gt": 5}}).sort('rate_remaining', DESCENDING))) github_username = github_account.get('username') github_password = github_account.get('password') github_token = github_account.get('token') g = Github(github_username, github_token) return g, github_username
def get(self): result = list(github_col.find({}, {'_id': 0, 'password': 0})) data = {'status': 200, 'msg': '获取信息成功', 'result': result} return jsonify(data)
def run(): # setting_col.update_one({'key': 'task'}, {'$set': {'key': 'task', 'pid': os.getpid()}}, upsert=True) query_count = query_col.count({'enabled': True}) logger.info('需要处理的关键词总数: {}'.format(query_count)) if query_count: logger.info('需要处理的关键词总数: {}'.format(query_count)) else: logger.warning('请添加关键词') return if github_col.count({'rate_remaining': {'$gt': 5}}): pass else: logger.error('请配置github账号') return if setting_col.count({'key': 'task', 'page': {'$exists': True}}): setting_col.update_one({'key': 'task'}, {'$set': {'pid': os.getpid()}}) page = int(setting_col.find_one({'key': 'task'}).get('page')) for p in range(0, page): for query in query_col.find({'enabled': True}).sort('last', 1): github_account = random.choice( list( github_col.find({ "rate_limit": { "$gt": 5 } }).sort('rate_remaining', -1))) github_username = github_account.get('username') github_password = github_account.get('password') rate_remaining = github_account.get('rate_remaining') logger.info(github_username) logger.info(rate_remaining) g = Github( github_username, github_password, per_page=PER_PAGE, user_agent= 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36' ) # total = query.get('total') # if total is None: # repos = g.search_code(query=query.get('keyword'), # sort="indexed", order="desc") # total = repos.totalCount api_total = query.get('api_total') if api_total: total = api_total else: repos = g.search_code(query=query.get('keyword'), sort="indexed", order="desc") total = repos.totalCount if total > 1000: total = 1000 page_pre = int(query.get('page_pre')) if query.get( 'page_pre') is not None else -1 page_all = math.ceil(total / 30) if page_all == 0: continue if page_pre + 1 >= page_all: page_pre = -1 page_now = page_pre + 1 search(query, page_now, g, github_username) else: logger.error('请在页面上配置任务参数')