def check(): setting_col.update_one({'key': 'task'}, {'$set': {'key': 'task', 'pid': os.getpid()}}, upsert=True) query_count = query_col.count({'enabled': True}) logger.info('需要处理的关键词总数: {}'.format(query_count)) if query_count: logger.info('需要处理的关键词总数: {}'.format(query_count)) else: logger.warning('请添加关键词') return if github_col.count({'rate_remaining': {'$gt': 5}}): pass else: logger.error('请配置github账号') return if setting_col.count({'key': 'task', 'page': {'$exists': True}}): setting_col.update_one({'key': 'task'}, {'$set': {'pid': os.getpid()}}) page = int(setting_col.find_one({'key': 'task'}).get('page')) for p in range(0, page): for query in query_col.find({'enabled': True}).sort('last', ASCENDING): github_account = random.choice( list(github_col.find({"rate_limit": {"$gt": 5}}).sort('rate_remaining', DESCENDING))) github_username = github_account.get('username') github_password = github_account.get('password') github_token = github_account.get('token') rate_remaining = github_account.get('rate_remaining') logger.info(github_username) logger.info(rate_remaining) g = Github(github_username, github_token, user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36') search.schedule(args=(query, p, g, github_username), delay=huey.pending_count() + huey.scheduled_count()) else: logger.error('请在页面上配置任务参数')
def new_github(): if github_col.count({'rate_remaining': {'$gt': 5}}): pass else: logger.error('请配置github账号') return github_account = list(github_col.find().sort('rate_remaining', DESCENDING))[0] github_username = github_account.get('username') github_password = github_account.get('password') g = Github(github_username, github_password) return g, github_username
def new_github(): if github_col.count({'rate_remaining': {'$gt': 5}}): pass else: logger.error('请配置github账号') return github_account = random.choice(list(github_col.find({"rate_limit": {"$gt": 5}}).sort('rate_remaining', DESCENDING))) github_username = github_account.get('username') github_password = github_account.get('password') github_token = github_account.get('token') g = Github(github_username, github_token) return g, github_username
def run(): # setting_col.update_one({'key': 'task'}, {'$set': {'key': 'task', 'pid': os.getpid()}}, upsert=True) query_count = query_col.count({'enabled': True}) logger.info('需要处理的关键词总数: {}'.format(query_count)) if query_count: logger.info('需要处理的关键词总数: {}'.format(query_count)) else: logger.warning('请添加关键词') return if github_col.count({'rate_remaining': {'$gt': 5}}): pass else: logger.error('请配置github账号') return if setting_col.count({'key': 'task', 'page': {'$exists': True}}): setting_col.update_one({'key': 'task'}, {'$set': {'pid': os.getpid()}}) page = int(setting_col.find_one({'key': 'task'}).get('page')) for p in range(0, page): for query in query_col.find({'enabled': True}).sort('last', 1): github_account = random.choice( list( github_col.find({ "rate_limit": { "$gt": 5 } }).sort('rate_remaining', -1))) github_username = github_account.get('username') github_password = github_account.get('password') rate_remaining = github_account.get('rate_remaining') logger.info(github_username) logger.info(rate_remaining) g = Github( github_username, github_password, per_page=PER_PAGE, user_agent= 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36' ) # total = query.get('total') # if total is None: # repos = g.search_code(query=query.get('keyword'), # sort="indexed", order="desc") # total = repos.totalCount api_total = query.get('api_total') if api_total: total = api_total else: repos = g.search_code(query=query.get('keyword'), sort="indexed", order="desc") total = repos.totalCount if total > 1000: total = 1000 page_pre = int(query.get('page_pre')) if query.get( 'page_pre') is not None else -1 page_all = math.ceil(total / 30) if page_all == 0: continue if page_pre + 1 >= page_all: page_pre = -1 page_now = page_pre + 1 search(query, page_now, g, github_username) else: logger.error('请在页面上配置任务参数')