async def preProcess(loop, limit, start, statistic):
"""准备工作"""
semaphore = asyncio.Semaphore(configPraser.getSemaphore()) # 对速度做出限制
"""多协程"""
tasks = [asyncio.ensure_future(AsyncApiHelper.downloadInformation(pull_number, semaphore, statistic))
for pull_number in range(start, max(start - limit, 0), -1)]
await asyncio.wait(tasks)
async def preProcessUserFollowList(loop, statistic, userList):
semaphore = asyncio.Semaphore(configPraser.getSemaphore()) # 对速度做出限制
mysql = await getMysqlObj(loop)
if configPraser.getPrintMode():
print("mysql init success")
tasks = [asyncio.ensure_future(AsyncApiHelper.downloadUserFollowList(login, semaphore, mysql, statistic))
for login in userList] # 可以通过nodes 过多次嵌套节省请求数量
await asyncio.wait(tasks)
async def preProcessUnmatchCommits(loop, statistic):
semaphore = asyncio.Semaphore(configPraser.getSemaphore()) # 对速度做出限制
mysql = await getMysqlObj(loop)
if configPraser.getPrintMode():
print("mysql init success")
res = await AsyncSqlHelper.query(mysql, SqlUtils.STR_SQL_QUERY_UNMATCH_COMMITS, None)
print(res)
tasks = [asyncio.ensure_future(AsyncApiHelper.downloadCommits(item[0], item[1], semaphore, mysql, statistic))
for item in res] # 可以通过nodes 过多次嵌套节省请求数量
await asyncio.wait(tasks)
async def preProcess(loop, limit, start, statistic):
"""准备工作"""
semaphore = asyncio.Semaphore(configPraser.getSemaphore()) # 对速度做出限制
"""初始化数据库"""
mysql = await getMysqlObj(loop)
if configPraser.getPrintMode():
print("mysql init success")
"""多协程"""
if configPraser.getApiVersion() == StringKeyUtils.API_VERSION_RESET:
tasks = [asyncio.ensure_future(AsyncApiHelper.downloadInformation(pull_number, semaphore, mysql, statistic))
for pull_number in range(start, max(start - limit, 0), -1)]
elif configPraser.getApiVersion() == StringKeyUtils.API_VERSION_GRAPHQL:
tasks = [
asyncio.ensure_future(AsyncApiHelper.downloadInformationByV4(pull_number, semaphore, mysql, statistic))
for pull_number in range(start, max(start - limit, 0), -1)]
await asyncio.wait(tasks)
async def preProcessUnmatchCommitFile(loop, statistic):
semaphore = asyncio.Semaphore(configPraser.getSemaphore()) # 对速度做出限制
mysql = await getMysqlObj(loop)
if configPraser.getPrintMode():
print("mysql init success")
print("mysql init success")
fetch_size = 2000
total = await AsyncSqlHelper.query(mysql, SqlUtils.STR_SQL_QUERY_UNMATCH_COMMIT_FILE_COUNT_BY_HAS_FETCHED_FILE,
None)
fetch_loop = int(total[0][0] / fetch_size)
for i in range(0, fetch_loop):
start = random.randint(0, fetch_loop - 1)
res = await AsyncSqlHelper.query(mysql, SqlUtils.STR_SQL_QUERY_UNMATCH_COMMIT_FILE_BY_HAS_FETCHED_FILE,
[start * fetch_size])
print(res)
tasks = [
asyncio.ensure_future(AsyncApiHelper.downloadCommits(item[0], item[1], semaphore, mysql, statistic))
for item in res] # 可以通过nodes 过多次嵌套节省请求数量
await asyncio.wait(tasks)
def getPullRequestTimeLine(owner, repo, nodes):
# 获取多个个pull request的时间线上面的信息 并对上面的comment做拼接
AsyncApiHelper.setRepo(owner, repo)
t1 = datetime.now()
statistic = statisticsHelper()
statistic.startTime = t1
semaphore = asyncio.Semaphore(configPraser.getSemaphore()) # 对速度做出限制
loop = asyncio.get_event_loop()
coro = getMysqlObj(loop)
task = loop.create_task(coro)
loop.run_until_complete(task)
mysql = task.result()
# nodes = np.array(nodes).reshape(10, -1)
nodesGroup = []
if nodes.__len__() % 10 == 0:
nodesGroup = np.array(nodes).reshape(10, -1)
else:
offset = 10 - nodes.__len__() % 10
nodes.extend([None for i in range(0, offset)])
nodesGroup = np.array(nodes).reshape(20, -1)
# for index in range(0, nodes.__len__(), 10):
# if index + 10 < nodes.__len__():
# nodesGroup.append(nodes[index:index + 10])
# else:
# nodesGroup.append(nodes[index:nodes.__len__()])
tasks = [
asyncio.ensure_future(AsyncApiHelper.downloadRPTimeLine([x for x in nodegroup.tolist() if x is not None],
semaphore, mysql, statistic))
for nodegroup in nodesGroup] # 可以通过nodes 过多次嵌套节省请求数量
tasks = asyncio.gather(*tasks)
loop.run_until_complete(tasks)
print('cost time:', datetime.now() - t1)
return tasks.result()
async def preProcessNoOriginLineReviewComment(loop, statistic, owner, repo, min_num, max_num):
semaphore = asyncio.Semaphore(configPraser.getSemaphore()) # 对速度做出限制
mysql = await getMysqlObj(loop)
if configPraser.getPrintMode():
print("mysql init success")
print("mysql init success")
repoName = owner + '/' + repo
values = [repoName, repoName, min_num, max_num]
total = await AsyncSqlHelper.query(mysql, SqlUtils.STR_SQL_QUERY_NO_ORIGINAL_LINE_REVIEW_COMMENT_COUNT,
values)
fetch_loop = math.ceil(total[0][0] / 2000)
for i in range(0, fetch_loop):
res = await AsyncSqlHelper.query(mysql, SqlUtils.STR_SQL_QUERY_NO_ORIGINAL_LINE_REVIEW_COMMENT
, values)
print("fetched size:", res.__len__())
tasks = [asyncio.ensure_future(
AsyncApiHelper.downloadSingleReviewComment(repoName, item[0], semaphore, mysql, statistic))
for item in res] # 可以通过nodes 过多次嵌套节省请求数量
await asyncio.wait(tasks)