async def preProcess(loop, limit, start, statistic): """准备工作""" semaphore = asyncio.Semaphore(configPraser.getSemaphore()) # 对速度做出限制 """多协程""" tasks = [asyncio.ensure_future(AsyncApiHelper.downloadInformation(pull_number, semaphore, statistic)) for pull_number in range(start, max(start - limit, 0), -1)] await asyncio.wait(tasks)
async def preProcessUserFollowList(loop, statistic, userList): semaphore = asyncio.Semaphore(configPraser.getSemaphore()) # 对速度做出限制 mysql = await getMysqlObj(loop) if configPraser.getPrintMode(): print("mysql init success") tasks = [asyncio.ensure_future(AsyncApiHelper.downloadUserFollowList(login, semaphore, mysql, statistic)) for login in userList] # 可以通过nodes 过多次嵌套节省请求数量 await asyncio.wait(tasks)
async def preProcessUnmatchCommits(loop, statistic): semaphore = asyncio.Semaphore(configPraser.getSemaphore()) # 对速度做出限制 mysql = await getMysqlObj(loop) if configPraser.getPrintMode(): print("mysql init success") res = await AsyncSqlHelper.query(mysql, SqlUtils.STR_SQL_QUERY_UNMATCH_COMMITS, None) print(res) tasks = [asyncio.ensure_future(AsyncApiHelper.downloadCommits(item[0], item[1], semaphore, mysql, statistic)) for item in res] # 可以通过nodes 过多次嵌套节省请求数量 await asyncio.wait(tasks)
async def preProcess(loop, limit, start, statistic): """准备工作""" semaphore = asyncio.Semaphore(configPraser.getSemaphore()) # 对速度做出限制 """初始化数据库""" mysql = await getMysqlObj(loop) if configPraser.getPrintMode(): print("mysql init success") """多协程""" if configPraser.getApiVersion() == StringKeyUtils.API_VERSION_RESET: tasks = [asyncio.ensure_future(AsyncApiHelper.downloadInformation(pull_number, semaphore, mysql, statistic)) for pull_number in range(start, max(start - limit, 0), -1)] elif configPraser.getApiVersion() == StringKeyUtils.API_VERSION_GRAPHQL: tasks = [ asyncio.ensure_future(AsyncApiHelper.downloadInformationByV4(pull_number, semaphore, mysql, statistic)) for pull_number in range(start, max(start - limit, 0), -1)] await asyncio.wait(tasks)
async def preProcessUnmatchCommitFile(loop, statistic): semaphore = asyncio.Semaphore(configPraser.getSemaphore()) # 对速度做出限制 mysql = await getMysqlObj(loop) if configPraser.getPrintMode(): print("mysql init success") print("mysql init success") fetch_size = 2000 total = await AsyncSqlHelper.query(mysql, SqlUtils.STR_SQL_QUERY_UNMATCH_COMMIT_FILE_COUNT_BY_HAS_FETCHED_FILE, None) fetch_loop = int(total[0][0] / fetch_size) for i in range(0, fetch_loop): start = random.randint(0, fetch_loop - 1) res = await AsyncSqlHelper.query(mysql, SqlUtils.STR_SQL_QUERY_UNMATCH_COMMIT_FILE_BY_HAS_FETCHED_FILE, [start * fetch_size]) print(res) tasks = [ asyncio.ensure_future(AsyncApiHelper.downloadCommits(item[0], item[1], semaphore, mysql, statistic)) for item in res] # 可以通过nodes 过多次嵌套节省请求数量 await asyncio.wait(tasks)
def getPullRequestTimeLine(owner, repo, nodes): # 获取多个个pull request的时间线上面的信息 并对上面的comment做拼接 AsyncApiHelper.setRepo(owner, repo) t1 = datetime.now() statistic = statisticsHelper() statistic.startTime = t1 semaphore = asyncio.Semaphore(configPraser.getSemaphore()) # 对速度做出限制 loop = asyncio.get_event_loop() coro = getMysqlObj(loop) task = loop.create_task(coro) loop.run_until_complete(task) mysql = task.result() # nodes = np.array(nodes).reshape(10, -1) nodesGroup = [] if nodes.__len__() % 10 == 0: nodesGroup = np.array(nodes).reshape(10, -1) else: offset = 10 - nodes.__len__() % 10 nodes.extend([None for i in range(0, offset)]) nodesGroup = np.array(nodes).reshape(20, -1) # for index in range(0, nodes.__len__(), 10): # if index + 10 < nodes.__len__(): # nodesGroup.append(nodes[index:index + 10]) # else: # nodesGroup.append(nodes[index:nodes.__len__()]) tasks = [ asyncio.ensure_future(AsyncApiHelper.downloadRPTimeLine([x for x in nodegroup.tolist() if x is not None], semaphore, mysql, statistic)) for nodegroup in nodesGroup] # 可以通过nodes 过多次嵌套节省请求数量 tasks = asyncio.gather(*tasks) loop.run_until_complete(tasks) print('cost time:', datetime.now() - t1) return tasks.result()
async def preProcessNoOriginLineReviewComment(loop, statistic, owner, repo, min_num, max_num): semaphore = asyncio.Semaphore(configPraser.getSemaphore()) # 对速度做出限制 mysql = await getMysqlObj(loop) if configPraser.getPrintMode(): print("mysql init success") print("mysql init success") repoName = owner + '/' + repo values = [repoName, repoName, min_num, max_num] total = await AsyncSqlHelper.query(mysql, SqlUtils.STR_SQL_QUERY_NO_ORIGINAL_LINE_REVIEW_COMMENT_COUNT, values) fetch_loop = math.ceil(total[0][0] / 2000) for i in range(0, fetch_loop): res = await AsyncSqlHelper.query(mysql, SqlUtils.STR_SQL_QUERY_NO_ORIGINAL_LINE_REVIEW_COMMENT , values) print("fetched size:", res.__len__()) tasks = [asyncio.ensure_future( AsyncApiHelper.downloadSingleReviewComment(repoName, item[0], semaphore, mysql, statistic)) for item in res] # 可以通过nodes 过多次嵌套节省请求数量 await asyncio.wait(tasks)