def getConcurrenceByResourceId(self, labelid):
"""根据资源id获取当前并发数"""
query_param = 'groupName=paddlepaddle&origin=SELF&labelId=%s' % labelid
url = 'https://xly.bce.baidu.com/open-api/sa_server/sa/open/api/rest/v2/agents/details?%s' % query_param
headers = {
"Content-Type": "application/json",
"IPIPE-UID": "Paddle-bot"
}
res = xlyOpenApiRequest().get_method(url,
param=query_param,
headers=headers)
return res
def rerunJob(self, triggerId):
"""重新构建"""
rerun_url = 'https://xly.bce.baidu.com/open-api/ipipe/agile/pipeline/doRebuild?pipeTriggerId=%s' % triggerId
headers = {
"Content-Type": "application/json",
"IPIPE-UID": "Paddle-bot"
}
query_param = 'pipeTriggerId=%s' % triggerId
res = xlyOpenApiRequest().get_method(rerun_url,
param=query_param,
headers=headers)
return res
def getAllResource(self):
"""PaddlePaddle下所有的SA资源"""
url = 'https://xly.bce.baidu.com/open-api/sa_server/sa/open/api/rest/v2/labels/details?groupName=paddlepaddle'
headers = {
"Content-Type": "application/json",
"IPIPE-UID": "Paddle-bot"
}
query_param = 'groupName=paddlepaddle'
res = xlyOpenApiRequest().get_method(url,
param=query_param,
headers=headers)
return res
async def markCIFailed(self):
"""
mark success/pending ci to failed
"""
CIStatusList = await self.getCIstatus()
REQUIRED_CI = localConfig.cf.get('%s/%s' % (self.user, self.repo),
'REQUIRED_CI')
DATA = {"data": "FAIL", "message": "Paddle-bot", "type": "MARK"}
json_str = json.dumps(DATA)
headers = {
"Content-Type": "application/json",
"IPIPE-UID": "Paddle-bot"
}
for item in CIStatusList:
PR = item['PR']
commit = item['commit']
ci_list = item['CI']
mark_ci_list = []
for ci in ci_list:
if ci['ciName'] in REQUIRED_CI and ci['status'] in [
'success', 'pending'
]:
markId = ci['markId']
mark_url = self.mark_url.format(markId)
res = xlyOpenApiRequest().post_method(mark_url,
json_str,
headers=headers)
if res.status_code == 200 or res.status_code == 201:
mark_ci_list.append(ci['ciName'])
print('%s_%s_%s mark success!' %
(PR, commit, ci['ciName']))
logger.info('%s_%s_%s mark success!' %
(PR, commit, ci['ciName']))
else:
print('%s_%s_%s mark failed!' %
(PR, commit, ci['ciName']))
logger.error('%s_%s_%s mark failed!' %
(PR, commit, ci['ciName']))
if len(mark_ci_list) > 0:
marked = self.queryIfHasMark(PR, commit)
if marked == False:
self.inform(item)
else:
print('%s_%s has marked!!!!' % (PR, commit))
logger.info('%s_%s has marked!!!!' % (PR, commit))
data = {
'TIME': time.strftime("%Y%m%d %H:%M:%S", time.localtime()),
'PR': PR,
'COMMITID': commit,
'CINAME': mark_ci_list
}
self.save_markci_job(data)
def cancelJob(self, jobId):
"""取消任务"""
cancel_url = 'https://xly.bce.baidu.com/open-api/ipipe/rest/v1/job-builds/%s/operation-requests' % jobId
DATA = {"type": "CANCEL"}
json_str = json.dumps(DATA)
headers = {
"Content-Type": "application/json",
"IPIPE-UID": "Paddle-bot"
}
res = xlyOpenApiRequest().post_method(cancel_url,
json_str,
headers=headers)
return res
def getStageMessge(self, targetId):
"""
获取任务的stage 信息
"""
url = localConfig.cf.get('ipipeConf', 'stage_url') + str(targetId)
headers = {
"Content-Type": "application/json",
"IPIPE-UID": "Paddle-bot"
}
response = xlyOpenApiRequest().get_method(url, headers=headers)
if response.status_code == 200 or response.status_code == 201:
return response.json()
else:
logger.error("url: %s" % url)
logger.error("response: %s %s" % (response, response.text))
return None
def cancelJob(self):
timeout_running_job, alarm_running_job = self.filter_timeout_task()
mailContent = ''
if len(timeout_running_job) > 0:
mailContent += " <p>以下任务被判定为运行超时, 已自动取消，请排查超时原因！</p> <p>超时规则: Coverage超过180min, Py3超过120min, Inference/CPU 超过60min, Mac/Mac-python3 超过60min, Windows超过180min, Windows-OPENBLAS超过80min</p> <table border='1' align=center> <caption><font size='3'><b>自动取消运行中的任务列表</b></font></caption><tr align=center><td bgcolor='#d0d0d0'>PR</td><td bgcolor='#d0d0d0'>CIName</td><td bgcolor='#d0d0d0'>已运行时间/min</td><td bgcolor='#d0d0d0'>repo</td><td bgcolor='#d0d0d0'>任务链接</td></tr>"
for task in timeout_running_job:
target_url = 'https://xly.bce.baidu.com/paddlepaddle/paddle/newipipe/detail/%s/job/%s' % (
task['targetId'], task['jobId'])
data = {
'TIME': time.strftime("%Y%m%d %H:%M:%S", time.localtime()),
'PR': task['PR'],
'COMMITID': task['commitId'],
'CINAME': task['CIName'],
'RUNNINGTIME': [task['running']],
'TASKURL': target_url
}
self.save_cancel_job(data)
cancel_url = 'https://xly.bce.baidu.com/open-api/ipipe/rest/v1/job-builds/%s/operation-requests' % task[
'jobId']
DATA = {"type": "CANCEL"}
json_str = json.dumps(DATA)
headers = {
"Content-Type": "application/json",
"IPIPE-UID": "Paddle-bot"
}
res = xlyOpenApiRequest().post_method(cancel_url,
json_str,
headers=headers)
if res.status_code == 200 or res.status_code == 201:
mailContent += "<tr align=center><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>" % (
task['PR'], task['CIName'], task['running'],
task['repoName'], target_url)
mailContent += "</table>"
if len(alarm_running_job) > 0:
mailContent += "<p>此外, 以下任务已经运行超过60min, 请查看任务是否卡住.</p>"
mailContent += "<table border='1' align=center> <caption><font size='3'><b>任务已运行超过60min</b></font></caption><tr align=center><td bgcolor='#d0d0d0'>PR</td><td bgcolor='#d0d0d0'>CIName</td><td bgcolor='#d0d0d0'>已运行时间/min</td><td bgcolor='#d0d0d0'>repo</td><td bgcolor='#d0d0d0'>任务链接</td></tr>"
for task in alarm_running_job:
task_url = 'https://xly.bce.baidu.com/paddlepaddle/paddle/newipipe/detail/%s/job/%s' % (
task['targetId'], task['jobId'])
mailContent += "<tr align=center><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>" % (
task['PR'], task['CIName'], task['running'],
task['repoName'], task_url)
mailContent += "</table>"
if mailContent != '':
self.sendMail(mailContent)
async def markCIFailed(self):
"""
mark success/pending ci to failed
"""
CIStatusList = await self.getCIstatus()
REQUIRED_CI = localConfig.cf.get('%s/%s' % (self.user, self.repo),
'REQUIRED_CI')
DATA = {"data": "FAIL", "message": "Paddle-bot", "type": "MARK"}
json_str = json.dumps(DATA)
headers = {
"Content-Type": "application/json",
"IPIPE-UID": "Paddle-bot"
}
for item in CIStatusList:
MARK_List = []
PR = item['PR']
commit = item['commit']
ci_list = item['CI']
for ci in ci_list:
if ci['ciName'] in REQUIRED_CI and ci['status'] in [
'success', 'pending'
]:
markId = ci['markId']
mark_url = self.mark_url.format(markId)
res = xlyOpenApiRequest().post_method(mark_url,
json_str,
headers=headers)
if res.status_code == 200 or res.status_code == 201:
MARK_List.append(True)
print('%s_%s_%s mark success!' %
(PR, commit, ci['ciName']))
logger.error('%s_%s_%s mark success!' %
(PR, commit, ci['ciName']))
else:
MARK_List.append(False)
print('%s_%s_%s mark failed!' %
(PR, commit, ci['ciName']))
logger.error('%s_%s_%s mark failed!' %
(PR, commit, ci['ciName']))
if True in MARK_List:
await self.inform(item)
def getJobList(self, jobStatus):
"""
this function will get all Container job list. eg.V100/P4 type jobs.
Args:
jobStatus(str): job status. running/waiting/sarunning/sawaiting.
cardType_list(str): card type.
Returns:
all_task_list(list): all task list
"""
url = 'https://xly.bce.baidu.com/open-api/ipipe/rest/v1/paddle-api/status?key=%s' % jobStatus
param = 'key=%s' % jobStatus
headers = {
"Content-Type": "application/json",
"IPIPE-UID": "Paddle-bot"
}
start = int(time.time())
response = xlyOpenApiRequest().get_method(url, param,
headers=headers).json()
end = int(time.time())
print('end-start: %s' % (end - start))
print(response)
return response