def get_new_commits(request, name, stage):
env = environs_helper.get_env_by_stage(request, name, stage)
current_deploy = deploys_helper.get(request, env['deployId'])
current_build = builds_helper.get_build(request, current_deploy['buildId'])
startSha = current_build['commit']
repo = current_build['repo']
scm_url = systems_helper.get_scm_url(request)
diffUrl = "%s/%s/compare/%s...%s" % (scm_url, repo, startSha, startSha)
last_deploy = common.get_last_completed_deploy(request, env)
if not last_deploy:
return render(request, 'deploys/deploy_commits.html', {
"env": env,
"title": "No previous deploy found!",
"startSha": startSha,
"endSha": startSha,
"repo": repo,
"diffUrl": diffUrl,
})
last_build = builds_helper.get_build(request, last_deploy['buildId'])
endSha = last_build['commit']
diffUrl = "%s/%s/compare/%s...%s" % (scm_url, repo, endSha, startSha)
return render(request, 'deploys/deploy_commits.html', {
"env": env,
"startSha": startSha,
"endSha": endSha,
"repo": repo,
"title": "Commits since last deploy",
"diffUrl": diffUrl,
})
def get_new_commits(request, name, stage):
env = environs_helper.get_env_by_stage(request, name, stage)
current_deploy = deploys_helper.get(request, env['deployId'])
current_build = builds_helper.get_build(request, current_deploy['buildId'])
startSha = current_build['commit']
repo = current_build['repo']
scm_url = systems_helper.get_scm_url(request)
diffUrl = "%s/%s/compare/%s...%s" % (scm_url, repo, startSha, startSha)
last_deploy = common.get_last_completed_deploy(request, env)
if not last_deploy:
return render(
request, 'deploys/deploy_commits.html', {
"env": env,
"title": "No previous deploy found!",
"startSha": startSha,
"endSha": startSha,
"repo": repo,
"diffUrl": diffUrl,
})
last_build = builds_helper.get_build(request, last_deploy['buildId'])
endSha = last_build['commit']
diffUrl = "%s/%s/compare/%s...%s" % (scm_url, repo, endSha, startSha)
return render(
request, 'deploys/deploy_commits.html', {
"env": env,
"startSha": startSha,
"endSha": endSha,
"repo": repo,
"title": "Commits since last deploy",
"diffUrl": diffUrl,
})
def compare_deploys(request, name, stage):
start_deploy_id = request.GET.get('start_deploy', None)
start_deploy = deploys_helper.get(request, start_deploy_id)
start_build = builds_helper.get_build(request, start_deploy['buildId'])
startSha = start_build['commit']
repo = start_build['repo']
end_deploy_id = request.GET.get('end_deploy', None)
if end_deploy_id:
end_deploy = deploys_helper.get(request, end_deploy_id)
else:
env = environs_helper.get_env_by_stage(request, name, stage)
end_deploy = common.get_previous_deploy(request, env, start_deploy)
if not end_deploy:
end_deploy = start_deploy
end_build = builds_helper.get_build(request, end_deploy['buildId'])
endSha = end_build['commit']
commits, truncated, new_start_sha = common.get_commits_batch(
request, repo, startSha, endSha, keep_first=True)
html = render_to_string(
'builds/commits.tmpl', {
"commits": commits,
"start_sha": new_start_sha,
"end_sha": endSha,
"repo": repo,
"truncated": truncated,
"show_checkbox": False,
})
return HttpResponse(html)
def compare_deploys_2(request, name, stage):
env = environs_helper.get_env_by_stage(request, name, stage)
configs = {}
for key, value in request.GET.iteritems():
if key.startswith('chkbox_'):
index = key[len('chkbox_'):]
configs[index] = value
indexes = configs.keys()
start_build_id = configs[indexes[0]]
end_build_id = configs[indexes[1]]
if int(indexes[0]) > int(indexes[1]):
start_build_id = configs[indexes[1]]
end_build_id = configs[indexes[0]]
start_build = builds_helper.get_build(request, start_build_id)
startSha = start_build['commit']
repo = start_build['repo']
end_build = builds_helper.get_build(request, end_build_id)
endSha = end_build['commit']
scm_url = systems_helper.get_scm_url(request)
diffUrl = "%s/%s/compare/%s...%s" % (scm_url, repo, endSha, startSha)
return render(request, 'deploys/deploy_commits.html', {
"env": env,
"startSha": startSha,
"endSha": endSha,
"repo": repo,
"title": "Commits between the two deploys",
"diffUrl": diffUrl,
})
def compare_deploys(request, name, stage):
start_deploy_id = request.GET.get('start_deploy', None)
start_deploy = deploys_helper.get(request, start_deploy_id)
start_build = builds_helper.get_build(request, start_deploy['buildId'])
startSha = start_build['commit']
repo = start_build['repo']
end_deploy_id = request.GET.get('end_deploy', None)
if end_deploy_id:
end_deploy = deploys_helper.get(request, end_deploy_id)
else:
env = environs_helper.get_env_by_stage(request, name, stage)
end_deploy = common.get_previous_deploy(request, env, start_deploy)
if not end_deploy:
end_deploy = start_deploy
end_build = builds_helper.get_build(request, end_deploy['buildId'])
endSha = end_build['commit']
commits, truncated, new_start_sha = common.get_commits_batch(request, repo, startSha,
endSha, keep_first=True)
html = render_to_string('builds/commits.tmpl', {
"commits": commits,
"start_sha": new_start_sha,
"end_sha": endSha,
"repo": repo,
"truncated": truncated,
"show_checkbox": False,
})
return HttpResponse(html)
def compare_deploys_2(request, name, stage):
env = environs_helper.get_env_by_stage(request, name, stage)
configs = {}
for key, value in request.GET.iteritems():
if key.startswith('chkbox_'):
index = key[len('chkbox_'):]
configs[index] = value
indexes = configs.keys()
start_build_id = configs[indexes[0]]
end_build_id = configs[indexes[1]]
if int(indexes[0]) > int(indexes[1]):
start_build_id = configs[indexes[1]]
end_build_id = configs[indexes[0]]
start_build = builds_helper.get_build(request, start_build_id)
startSha = start_build['commit']
repo = start_build['repo']
end_build = builds_helper.get_build(request, end_build_id)
endSha = end_build['commit']
scm_url = systems_helper.get_scm_url(request)
diffUrl = "%s/%s/compare/%s...%s" % (scm_url, repo, endSha, startSha)
return render(
request, 'deploys/deploy_commits.html', {
"env": env,
"startSha": startSha,
"endSha": endSha,
"repo": repo,
"title": "Commits between the two deploys",
"diffUrl": diffUrl,
})
def get_duplicate_commit_deploy_message(request, name, stage, buildId):
current_deploy = deploys_helper.get_current(request, name, stage)
current_build = builds_helper.get_build(request, current_deploy['buildId'])
current_commit = current_build['commit']
next_build = builds_helper.get_build(request, buildId)
next_commit = next_build['commit']
if current_commit == next_commit:
return render(request, 'deploys/duplicate_commit_deploy_message.tmpl',
{"commit": next_build['commitShort']})
return HttpResponse('')
def get_all_builds(request):
name = request.GET.get('name')
branch = request.GET.get('branch')
index = int(request.GET.get('page_index', '1'))
size = int(request.GET.get('page_size', common.DEFAULT_BUILD_SIZE))
builds = builds_helper.get_builds(request,
name=name,
branch=branch,
pageIndex=index,
pageSize=size)
scm_url = systems_helper.get_scm_url(request)
current_build_id = request.GET.get('current_build_id', None)
current_build = None
if current_build_id:
current_build = builds_helper.get_build(request, current_build_id)
html = render_to_string(
'builds/pick_a_build.tmpl', {
"builds": builds,
"current_build": current_build,
"scm_url": scm_url,
"buildName": name,
"branch": branch,
"pageIndex": index,
"pageSize": common.DEFAULT_BUILD_SIZE,
"disablePrevious": index <= 1,
"disableNext": len(builds) < common.DEFAULT_BUILD_SIZE,
})
return HttpResponse(html)
def gen_report(request, env, progress, sortByStatus="false"):
agentStats = []
firstTimeAgentStats = []
deployStats = {}
deprecatedDeployStats = []
# always set the current
deploy = deploys_helper.get(request, env['deployId'])
build = builds_helper.get_build(request, deploy["buildId"])
stageDistMap = genStageDistMap()
stateDistMap = genStateDistMap()
currentDeployStat = DeployStatistics(deploy=deploy, build=build, stageDistMap=stageDistMap,
stateDistMap=stateDistMap)
deployStats[env['deployId']] = currentDeployStat
for agent in progress["agents"]:
if agent["firstDeploy"]:
firstTimeAgentStats.append(addToEnvReport(request, deployStats, agent, env))
else:
agentStats.append(addToEnvReport(request, deployStats, agent, env))
if sortByStatus == "true":
agentStats.sort(cmp=lambda x, y: _compare_agent_status(x, y))
for key, value in deployStats.iteritems():
if key != env['deployId']:
deprecatedDeployStats.append(value)
return AgentReport(firstTimeAgentStats=firstTimeAgentStats,
agentStats=agentStats, currentDeployStat=currentDeployStat,
deprecatedDeployStats=deprecatedDeployStats,
missingHosts=progress["missingHosts"],
envName=env['envName'], stageName=env['stageName'])
def addToEnvReport(request, deployStats, agent, env):
deployId = agent['deployId']
if deployId not in deployStats:
deploy = deploys_helper.get(request, deployId)
build = builds_helper.get_build(request, deploy['buildId'])
stageDistMap = genStageDistMap()
stateDistMap = genStateDistMap()
deployStat = DeployStatistics(deploy=deploy, build=build, stageDistMap=stageDistMap,
stateDistMap=stateDistMap)
deployStats[deployId] = deployStat
else:
deployStat = deployStats[deployId]
deployStat.stageDistMap[agent['deployStage']] += 1
deployStat.stateDistMap[agent['state']] += 1
deployStat.total += 1
isCurrent = (deployId == env['deployId'])
isStale = False
duration = (time.time() * 1000 - agent['lastUpdateDate']) / 1000
if duration >= DEFAULT_STALE_THRESHOLD:
isStale = True
return AgentStatistics(agent=agent, isCurrent=isCurrent, isStale=isStale)
def promote(request, name, stage, deploy_id):
envs = environs_helper.get_all_env_stages(request, name)
stages, env = common.get_all_stages(envs, stage)
env_wrappers = []
for temp_env in envs:
env_wrapper = {}
env_wrapper["env"] = temp_env
env_wrapper["env_promote"] = environs_helper.get_env_promotes_config(request,
temp_env['envName'],
temp_env['stageName'])
env_wrappers.append(env_wrapper)
deploy = deploys_helper.get(request, deploy_id)
build = builds_helper.get_build(request, deploy['buildId'])
html = render_to_string("environs/env_promote.html", {
"stages": stages,
"envs": envs,
"env": env,
"env_wrappers": env_wrappers,
"deploy": deploy,
"build": build,
"csrf_token": get_token(request),
})
return HttpResponse(html)
def get_duplicate_commit_deploy_message(request, name, stage, buildId):
env = environs_helper.get_env_by_stage(request, name, stage)
if env.get('deployId') is None:
return HttpResponse('')
current_deploy = deploys_helper.get_current(request, name, stage)
current_build = builds_helper.get_build(request, current_deploy['buildId'])
current_commit = current_build['commit']
next_build = builds_helper.get_build(request, buildId)
next_commit = next_build['commit']
if current_commit == next_commit:
return render(request, 'deploys/duplicate_commit_deploy_message.tmpl',{
"commit":next_build['commitShort']})
return HttpResponse('')
def get_deploy_status(self, request, name, stage):
env = environs_helper.get_env_by_stage(request, name, stage)
deploy = deploys_helper.get(request, env['deployId'])
build = builds_helper.get_build(request, deploy['buildId'])
canary_message = None
if is_prod() and stage == "canary" and deploy['state'] == "SUCCEEDING":
expected = build['commit']
running_build = self.get_canary_version_internal()
if expected[:7] == running_build[:7]:
canary_message = \
"curl https://canary.pinterest.com. " \
"Canary host is running on the correct version: {}".format(expected[:7])
else:
canary_message = "curl https://canary.pinterest.com. Running version: {}, " \
"Expect version: {}".format(running_build[:7], expected[:7])
deploy['state'] = "RUNNING"
if deploy['state'] == "SUCCEEDING" or deploy['state'] == "SUCCEEDED":
state = "succeeding"
elif deploy['state'] == "ABORTED" or deploy['state'] == "FAILING":
state = "failing"
else:
state = "running"
return state, render(request, 'ngapp2/ngapp2_deploy.tmpl', {
"build": build,
"env": env,
"deploy": deploy,
"canaryMessage": canary_message
})
def get_all_builds(request):
name = request.GET.get('name')
branch = request.GET.get('branch')
index = int(request.GET.get('page_index', '1'))
size = int(request.GET.get('page_size', common.DEFAULT_BUILD_SIZE))
builds = builds_helper.get_builds(request, name=name, branch=branch, pageIndex=index,
pageSize=size)
scm_url = systems_helper.get_scm_url(request)
current_build_id = request.GET.get('current_build_id', None)
current_build = None
if current_build_id:
current_build = builds_helper.get_build(request, current_build_id)
html = render_to_string('builds/pick_a_build.tmpl', {
"builds": builds,
"current_build": current_build,
"scm_url": scm_url,
"buildName": name,
"branch": branch,
"pageIndex": index,
"pageSize": common.DEFAULT_BUILD_SIZE,
"disablePrevious": index <= 1,
"disableNext": len(builds) < common.DEFAULT_BUILD_SIZE,
})
return HttpResponse(html)
def addToEnvReport(request, deployStats, agent, env):
deployId = agent['deployId']
if deployId not in deployStats:
deploy = deploys_helper.get(request, deployId)
build = builds_helper.get_build(request, deploy['buildId'])
stageDistMap = genStageDistMap()
stateDistMap = genStateDistMap()
deployStat = DeployStatistics(deploy=deploy,
build=build,
stageDistMap=stageDistMap,
stateDistMap=stateDistMap)
deployStats[deployId] = deployStat
else:
deployStat = deployStats[deployId]
deployStat.stageDistMap[agent['deployStage']] += 1
deployStat.stateDistMap[agent['state']] += 1
deployStat.total += 1
isCurrent = (deployId == env['deployId'])
isStale = False
duration = (time.time() * 1000 - agent['lastUpdateDate']) / 1000
if duration >= DEFAULT_STALE_THRESHOLD:
isStale = True
return AgentStatistics(agent=agent, isCurrent=isCurrent, isStale=isStale)
def get_all_deploys(request):
env_stage_a = environs_helper.get_env_by_stage(request, NGAPP_A, "prod")
env_stage_b = environs_helper.get_env_by_stage(request, NGAPP_B, "prod")
index = int(request.GET.get('page_index', '1'))
size = int(request.GET.get('page_size', '%d' % DEFAULT_PAGE_SIZE))
filter = {}
filter['envId'] = [env_stage_a['id'], env_stage_b['id']]
filter['pageIndex'] = index
filter['pageSize'] = size
result = deploys_helper.get_all(request, **filter)
deploy_summaries = []
for deploy in result['deploys']:
build = builds_helper.get_build(request, deploy['buildId'])
summary = {}
summary['deploy'] = deploy
summary['build'] = build
deploy_summaries.append(summary)
return render(request, 'ngapp2/ngapp2_history.html', {
"deploy_summaries": deploy_summaries,
"pageIndex": index,
"pageSize": DEFAULT_PAGE_SIZE,
"disablePrevious": index <= 1,
"disableNext": len(result['deploys']) < DEFAULT_PAGE_SIZE,
})
def get_notify_authors_message(request, env, current_build):
try:
last_succ_deploy = get_last_ngapp2_deploy(request, env['envName'],
env['stageName'])
last_build = builds_helper.get_build(request,
last_succ_deploy['buildId'])
start_commit = current_build.get('commit')
end_commit = last_build.get('commit')
repo = last_build['repo']
commits = common.get_commits_between(request,
repo=repo,
startSha=start_commit,
endSha=end_commit)
authors = set()
for commit in commits:
author = commit.get('author')
if not author or author.lower() == "unknown":
continue
authors.add("<@{}>".format(author))
mentions = ",".join(authors)
return "This deploy features {} commits from the " \
"following pingineers: {}. See changes <{}|here>".format(len(commits), mentions,
get_ngapp2_compare_deploy_url(
current_build.get("id"),
last_build.get("id")))
except:
logger.error(traceback.format_exc())
return ""
def promote(request, name, stage, deploy_id):
envs = environs_helper.get_all_env_stages(request, name)
stages, env = common.get_all_stages(envs, stage)
env_wrappers = []
for temp_env in envs:
env_wrapper = {}
env_wrapper["env"] = temp_env
env_wrapper["env_promote"] = environs_helper.get_env_promotes_config(
request, temp_env['envName'], temp_env['stageName'])
env_wrappers.append(env_wrapper)
deploy = deploys_helper.get(request, deploy_id)
build = builds_helper.get_build(request, deploy['buildId'])
html = render_to_string(
"environs/env_promote.html", {
"stages": stages,
"envs": envs,
"env": env,
"env_wrappers": env_wrappers,
"deploy": deploy,
"build": build,
"csrf_token": get_token(request),
})
return HttpResponse(html)
def get_all_deploys(request):
env_stage_a = environs_helper.get_env_by_stage(request, NGAPP_A, "prod")
env_stage_b = environs_helper.get_env_by_stage(request, NGAPP_B, "prod")
index = int(request.GET.get('page_index', '1'))
size = int(request.GET.get('page_size', '%d' % DEFAULT_PAGE_SIZE))
filter = {}
filter['envId'] = [env_stage_a['id'], env_stage_b['id']]
filter['pageIndex'] = index
filter['pageSize'] = size
result = deploys_helper.get_all(request, **filter)
deploy_summaries = []
for deploy in result['deploys']:
build = builds_helper.get_build(request, deploy['buildId'])
summary = {}
summary['deploy'] = deploy
summary['build'] = build
deploy_summaries.append(summary)
return render(
request, 'ngapp2/ngapp2_history.html', {
"deploy_summaries": deploy_summaries,
"pageIndex": index,
"pageSize": DEFAULT_PAGE_SIZE,
"disablePrevious": index <= 1,
"disableNext": len(result['deploys']) < DEFAULT_PAGE_SIZE,
})
def get_deploy_status(self, request, name, stage):
env = environs_helper.get_env_by_stage(request, name, stage)
deploy = deploys_helper.get(request, env['deployId'])
build = builds_helper.get_build(request, deploy['buildId'])
canary_message = None
if is_prod() and stage == "canary" and deploy['state'] == "SUCCEEDING":
expected = build['commit']
running_build = self.get_canary_version_internal()
if expected[:7] == running_build[:7]:
canary_message = \
"curl https://canary.pinterest.com. " \
"Canary host is running on the correct version: {}".format(expected[:7])
else:
canary_message = "curl https://canary.pinterest.com. Running version: {}, " \
"Expect version: {}".format(running_build[:7], expected[:7])
deploy['state'] = "RUNNING"
if deploy['state'] == "SUCCEEDING" or deploy['state'] == "SUCCEEDED":
state = "succeeding"
elif deploy['state'] == "ABORTED" or deploy['state'] == "FAILING":
state = "failing"
else:
state = "running"
return state, render(
request, 'ngapp2/ngapp2_deploy.tmpl', {
"build": build,
"env": env,
"deploy": deploy,
"canaryMessage": canary_message
})
def get_deploy(request, name, stage, deploy_id):
deploy = deploys_helper.get(request, deploy_id)
build = builds_helper.get_build(request, deploy['buildId'])
env = environs_helper.get_env_by_stage(request, name, stage)
return render(request, 'environs/env_deploy_details.html', {
"deploy": deploy,
"build": build,
"env": env,
})
def get_deploy(request, name, stage, deploy_id):
deploy = deploys_helper.get(request, deploy_id)
build = builds_helper.get_build(request, deploy['buildId'])
env = environs_helper.get_env_by_stage(request, name, stage)
return render(request, 'environs/env_deploy_details.html', {
"deploy": deploy,
"build": build,
"env": env,
})
def get_pred_deploys(request, name, stage):
index = int(request.GET.get('page_index', '1'))
size = int(request.GET.get('page_size', DEFAULT_PAGE_SIZE))
env = environs_helper.get_env_by_stage(request, name, stage)
env_promote = environs_helper.get_env_promotes_config(request, name, stage)
show_lock = False
predStage = env_promote.get('predStage')
if env_promote['type'] != "MANUAL" and predStage:
show_lock = True
current_startDate = 0
if not predStage or predStage == "BUILD":
deploys = []
else:
pred_env = environs_helper.get_env_by_stage(request, name, predStage)
result = deploys_helper.get_all(request,
envId=[pred_env['id']],
pageIndex=index,
pageSize=size)
deploys = result["deploys"]
if env.get('deployId'):
deploy = deploys_helper.get(request, env['deployId'])
build = builds_helper.get_build(request, deploy['buildId'])
current_startDate = build['publishDate']
deploy_wrappers = []
for deploy in deploys:
build = builds_helper.get_build(request, deploy['buildId'])
if build['publishDate'] > current_startDate:
deploy_wrapper = {}
deploy_wrapper['deploy'] = deploy
deploy_wrapper['build'] = build
deploy_wrappers.append(deploy_wrapper)
html = render_to_string(
'deploys/simple_pred_deploys.tmpl', {
"deploy_wrappers": deploy_wrappers,
"envName": name,
"stageName": predStage,
"show_lock": show_lock,
"current_startDate": current_startDate,
})
return HttpResponse(html)
def get(self, request, deploy_id):
deploy = deploys_helper.get(request, deploy_id)
build = builds_helper.get_build(request, deploy['buildId'])
env = None
if deploy.get('envId'):
env = environs_helper.get(request, deploy['envId'])
return render(request, 'deploys/deploy_details.html', {
"deploy": deploy,
"build": build,
"env": env,
})
def deploy_build(request, name, stage, build_id):
env = environs_helper.get_env_by_stage(request, name, stage)
current_build = None
if env.get('deployId'):
current_deploy = deploys_helper.get(request, env['deployId'])
current_build = builds_helper.get_build(request, current_deploy['buildId'])
build = builds_helper.get_build(request, build_id)
builds = [build]
scm_url = systems_helper.get_scm_url(request)
html = render_to_string('deploys/deploy_build.html', {
"env": env,
"builds": builds,
"current_build": current_build,
"scm_url": scm_url,
"buildName": env.get('buildName'),
"branch": env.get('branch'),
"csrf_token": get_token(request),
})
return HttpResponse(html)
def get_pred_deploys(request, name, stage):
index = int(request.GET.get('page_index', '1'))
size = int(request.GET.get('page_size', DEFAULT_PAGE_SIZE))
env = environs_helper.get_env_by_stage(request, name, stage)
env_promote = environs_helper.get_env_promotes_config(request, name, stage)
show_lock = False
predStage = env_promote.get('predStage')
if env_promote['type'] != "MANUAL" and predStage:
show_lock = True
current_startDate = 0
if not predStage or predStage == "BUILD":
deploys = []
else:
pred_env = environs_helper.get_env_by_stage(request, name, predStage)
result = deploys_helper.get_all(request, envId=[pred_env['id']], pageIndex=index,
pageSize=size)
deploys = result["deploys"]
if env.get('deployId'):
deploy = deploys_helper.get(request, env['deployId'])
build = builds_helper.get_build(request, deploy['buildId'])
current_startDate = build['publishDate']
deploy_wrappers = []
for deploy in deploys:
build = builds_helper.get_build(request, deploy['buildId'])
if build['publishDate'] > current_startDate:
deploy_wrapper = {}
deploy_wrapper['deploy'] = deploy
deploy_wrapper['build'] = build
deploy_wrappers.append(deploy_wrapper)
html = render_to_string('deploys/simple_pred_deploys.tmpl', {
"deploy_wrappers": deploy_wrappers,
"envName": name,
"stageName": predStage,
"show_lock": show_lock,
"current_startDate": current_startDate,
})
return HttpResponse(html)
def get_all_deploys(request):
env_stage_a = environs_helper.get_env_by_stage(request, NGAPP_A, "prod")
env_stage_b = environs_helper.get_env_by_stage(request, NGAPP_B, "prod")
index = int(request.GET.get('page_index', '1'))
size = int(request.GET.get('page_size', '%d' % DEFAULT_PAGE_SIZE))
filter = {}
filter['envId'] = [env_stage_a['id'], env_stage_b['id']]
filter['pageIndex'] = index
filter['pageSize'] = size
result = deploys_helper.get_all(request, **filter)
deploy_summaries = []
for deploy in result['deploys']:
build = builds_helper.get_build(request, deploy['buildId'])
summary = {}
summary['deploy'] = deploy
summary['build'] = build
deploy_summaries.append(summary)
# get the rollback history from S3 between the start_timestamp and end_timestamp
# from the current page, assumed the deploy_summaries is in descend order.
# if it is the first page, set the end_timestamp to now.
if len(deploy_summaries):
start_timestamp = deploy_summaries[-1]['deploy']['startDate']
if index == 1:
end_timestamp = time.time() * 1000
else:
end_timestamp = deploy_summaries[0]['deploy']['startDate']
else:
start_timestamp = 0
end_timestamp = time.time() * 1000
s3 = s3_helper.S3Helper(bucket_name=S3_INTERNAL_TOOLS_BUCKET_NAME)
history_key = get_rollback_history_key()
rollbacks = s3.list(history_key)
for rollback in rollbacks:
timestamp = rollback.name[len(history_key) + 1:]
timestamp = float(timestamp)
if start_timestamp <= timestamp and timestamp <= end_timestamp:
summary = json.loads(s3.download_string(rollback.name))
deploy_summaries.append(summary)
# order the history by deploy.start_date descend
deploy_summaries.sort(key=lambda summary: summary['deploy']['startDate'],
reverse=True)
return render(
request, 'ngapp2/ngapp2_history.html', {
"deploy_summaries": deploy_summaries,
"pageIndex": index,
"pageSize": DEFAULT_PAGE_SIZE,
"disablePrevious": index <= 1,
"disableNext": len(result['deploys']) < DEFAULT_PAGE_SIZE,
})
def deploy_build(request, name, stage, build_id):
env = environs_helper.get_env_by_stage(request, name, stage)
current_build = None
if env.get('deployId'):
current_deploy = deploys_helper.get(request, env['deployId'])
current_build = builds_helper.get_build(request,
current_deploy['buildId'])
build = builds_helper.get_build(request, build_id)
builds = [build]
scm_url = systems_helper.get_scm_url(request)
html = render_to_string(
'deploys/deploy_build.html', {
"env": env,
"builds": builds,
"current_build": current_build,
"scm_url": scm_url,
"buildName": env.get('buildName'),
"branch": env.get('branch'),
"csrf_token": get_token(request),
})
return HttpResponse(html)
def get_all_deploys(request):
env_stage_a = environs_helper.get_env_by_stage(request, NGAPP_A, "prod")
env_stage_b = environs_helper.get_env_by_stage(request, NGAPP_B, "prod")
index = int(request.GET.get('page_index', '1'))
size = int(request.GET.get('page_size', '%d' % DEFAULT_PAGE_SIZE))
filter = {}
filter['envId'] = [env_stage_a['id'], env_stage_b['id']]
filter['pageIndex'] = index
filter['pageSize'] = size
result = deploys_helper.get_all(request, **filter)
deploy_summaries = []
for deploy in result['deploys']:
build = builds_helper.get_build(request, deploy['buildId'])
summary = {}
summary['deploy'] = deploy
summary['build'] = build
deploy_summaries.append(summary)
# get the rollback history from S3 between the start_timestamp and end_timestamp
# from the current page, assumed the deploy_summaries is in descend order.
# if it is the first page, set the end_timestamp to now.
if len(deploy_summaries):
start_timestamp = deploy_summaries[-1]['deploy']['startDate']
if index == 1:
end_timestamp = time.time() * 1000
else:
end_timestamp = deploy_summaries[0]['deploy']['startDate']
else:
start_timestamp = 0
end_timestamp = time.time() * 1000
s3 = s3_helper.S3Helper(bucket_name=S3_INTERNAL_TOOLS_BUCKET_NAME)
history_key = get_rollback_history_key()
rollbacks = s3.list(history_key)
for rollback in rollbacks:
timestamp = rollback.name[len(history_key)+1:]
timestamp = float(timestamp)
if start_timestamp <= timestamp and timestamp <= end_timestamp:
summary = json.loads(s3.download_string(rollback.name))
deploy_summaries.append(summary)
# order the history by deploy.start_date descend
deploy_summaries.sort(key=lambda summary: summary['deploy']['startDate'], reverse=True)
return render(request, 'ngapp2/ngapp2_history.html', {
"deploy_summaries": deploy_summaries,
"pageIndex": index,
"pageSize": DEFAULT_PAGE_SIZE,
"disablePrevious": index <= 1,
"disableNext": len(result['deploys']) < DEFAULT_PAGE_SIZE,
})
def _gen_deploy_summary(request, deploys, for_env=None):
deploy_summaries = []
for deploy in deploys:
if for_env:
env = for_env
else:
env = environs_helper.get(request, deploy['envId'])
build = builds_helper.get_build(request, deploy['buildId'])
summary = {}
summary['deploy'] = deploy
summary['env'] = env
summary['build'] = build
deploy_summaries.append(summary)
return deploy_summaries
def _gen_deploy_summary(request, deploys, for_env=None):
deploy_summaries = []
for deploy in deploys:
if for_env:
env = for_env
else:
env = environs_helper.get(request, deploy['envId'])
build = builds_helper.get_build(request, deploy['buildId'])
summary = {}
summary['deploy'] = deploy
summary['env'] = env
summary['build'] = build
deploy_summaries.append(summary)
return deploy_summaries
def compare_deploys(request):
configs = {}
for key, value in request.GET.iteritems():
if key.startswith('chkbox_'):
index = key[len('chkbox_'):]
configs[index] = value
indexes = configs.keys()
start_build_id = configs[indexes[0]]
end_build_id = configs[indexes[1]]
if int(indexes[0]) > int(indexes[1]):
start_build_id = configs[indexes[1]]
end_build_id = configs[indexes[0]]
start_build = builds_helper.get_build(request, start_build_id)
startSha = start_build['commit']
repo = start_build['repo']
end_build = builds_helper.get_build(request, end_build_id)
endSha = end_build['commit']
return render(request, 'ngapp2/ngapp2_deploy_commits.html', {
"startSha": startSha,
"endSha": endSha,
"repo": repo,
})
def compare_deploys(request):
configs = {}
for key, value in request.GET.iteritems():
if key.startswith('chkbox_'):
index = key[len('chkbox_'):]
configs[index] = value
indexes = configs.keys()
start_build_id = configs[indexes[0]]
end_build_id = configs[indexes[1]]
if int(indexes[0]) > int(indexes[1]):
start_build_id = configs[indexes[1]]
end_build_id = configs[indexes[0]]
start_build = builds_helper.get_build(request, start_build_id)
startSha = start_build['commit']
repo = start_build['repo']
end_build = builds_helper.get_build(request, end_build_id)
endSha = end_build['commit']
return render(request, 'ngapp2/ngapp2_deploy_commits.html', {
"startSha": startSha,
"endSha": endSha,
"repo": repo,
})
def get(self, request):
ngapp2_deploy_utils = Ngapp2DeployUtils()
ok_to_serve = ngapp2_deploy_utils.get_ok_to_serve()
curr_build = ok_to_serve.get('default')
ok_to_serve_version = ok_to_serve.get('versions')
prev_build = ngapp2_deploy_utils.get_previous_build()
env_stage_a = environs_helper.get_env_by_stage(request, NGAPP_A,
"prod")
env_stage_b = environs_helper.get_env_by_stage(request, NGAPP_B,
"prod")
deploy_a = None
build_a = None
deploy_b = None
if env_stage_a.get('deployId'):
deploy_a = deploys_helper.get(request, env_stage_a['deployId'])
build_a = builds_helper.get_build(request, deploy_a['buildId'])
if env_stage_b.get('deployId'):
deploy_b = deploys_helper.get(request, env_stage_b['deployId'])
stage = getattr(NgappDeployStep,
ngapp2_deploy_utils.get_status_from_zk().upper())
if stage != NgappDeployStep.SERVING_BUILD:
deploying_env = ngapp2_deploy_utils.get_deploying_env_from_zk()
else:
if build_a and build_a['commit'][:7] == curr_build[:7]:
deploying_env = env_stage_b['envName']
else:
deploying_env = env_stage_a['envName']
ngapp2_deploy_utils.set_deploying_env_to_zk(deploying_env)
if deploying_env == env_stage_a['envName']:
curr_env = EnvStatus(env_stage_b, deploy_b, curr_build)
prev_env = EnvStatus(env_stage_a, deploy_a, prev_build)
else:
curr_env = EnvStatus(env_stage_a, deploy_a, curr_build)
prev_env = EnvStatus(env_stage_b, deploy_b, prev_build)
return render(
request, 'ngapp2/ngapp2_deploy.html', {
"prev_env": prev_env,
"curr_env": curr_env,
"deploy_stage": stage,
"deploying_env": deploying_env,
"serve_version_count": len(ok_to_serve_version)
})
def gen_report(request, env, progress, sortByStatus="false"):
agentStats = []
firstTimeAgentStats = []
deployStats = {}
deprecatedDeployStats = []
# always set the current
deploy = deploys_helper.get(request, env['deployId'])
build = builds_helper.get_build(request, deploy["buildId"])
stageDistMap = genStageDistMap()
stateDistMap = genStateDistMap()
currentDeployStat = DeployStatistics(deploy=deploy, build=build, stageDistMap=stageDistMap,
stateDistMap=stateDistMap)
deployStats[env['deployId']] = currentDeployStat
for agent in progress["agents"]:
if agent["firstDeploy"]:
firstTimeAgentStats.append(addToEnvReport(request, deployStats, agent, env))
else:
agentStats.append(addToEnvReport(request, deployStats, agent, env))
if sortByStatus == "true":
agentStats.sort(cmp=lambda x, y: _compare_agent_status(x, y))
for key, value in deployStats.iteritems():
if key != env['deployId']:
deprecatedDeployStats.append(value)
provisioning_hosts = progress["provisioningHosts"]
if IS_PINTEREST:
basic_cluster_info = clusters_helper.get_cluster(request, env['envName'], env['stageName'])
if basic_cluster_info and basic_cluster_info.get('capacity'):
hosts_in_cluster = clusters_helper.get_host_names(request, env['envName'], env['stageName'])
num_to_fake = basic_cluster_info.get('capacity') - len(hosts_in_cluster)
for i in range(num_to_fake):
faked_host = {}
faked_host['hostName'] = 'UNKNOWN'
faked_host['hostId'] = 'UNKNOWN'
faked_host['state'] = 'PROVISIONED'
provisioning_hosts.append(faked_host)
return AgentReport(firstTimeAgentStats=firstTimeAgentStats,
agentStats=agentStats,
currentDeployStat=currentDeployStat,
deprecatedDeployStats=deprecatedDeployStats,
missingHosts=progress["missingHosts"],
provisioningHosts=provisioning_hosts,
envName=env['envName'], stageName=env['stageName'])
def gen_report(request, env, progress, sortByStatus="false"):
agentStats = []
firstTimeAgentStats = []
deployStats = {}
deprecatedDeployStats = []
# always set the current
deploy = deploys_helper.get(request, env['deployId'])
build = builds_helper.get_build(request, deploy["buildId"])
stageDistMap = genStageDistMap()
stateDistMap = genStateDistMap()
currentDeployStat = DeployStatistics(deploy=deploy, build=build, stageDistMap=stageDistMap,
stateDistMap=stateDistMap)
deployStats[env['deployId']] = currentDeployStat
for agent in progress["agents"]:
if agent["firstDeploy"]:
firstTimeAgentStats.append(addToEnvReport(request, deployStats, agent, env))
else:
agentStats.append(addToEnvReport(request, deployStats, agent, env))
if sortByStatus == "true":
agentStats.sort(cmp=lambda x, y: _compare_agent_status(x, y))
for key, value in deployStats.iteritems():
if key != env['deployId']:
deprecatedDeployStats.append(value)
provisioning_hosts = progress["provisioningHosts"]
basic_cluster_info = clusters_helper.get_cluster(request, env['envName'], env['stageName'])
if basic_cluster_info and basic_cluster_info.get('capacity'):
hosts_in_cluster = clusters_helper.get_hosts(request, env['envName'], env['stageName'], [])
num_to_fake = basic_cluster_info.get('capacity') - len(hosts_in_cluster)
for i in range(num_to_fake):
faked_host = {}
faked_host['hostName'] = 'UNKNOWN'
faked_host['hostId'] = 'UNKNOWN'
faked_host['state'] = 'PROVISIONED'
provisioning_hosts.append(faked_host)
return AgentReport(firstTimeAgentStats=firstTimeAgentStats,
agentStats=agentStats,
currentDeployStat=currentDeployStat,
deprecatedDeployStats=deprecatedDeployStats,
missingHosts=progress["missingHosts"],
provisioningHosts=provisioning_hosts,
envName=env['envName'], stageName=env['stageName'])
def get(self, request, name, stage):
env = environs_helper.get_env_by_stage(request, name, stage)
env_promote = environs_helper.get_env_promotes_config(request, name, stage)
current_build = None
if 'deployId' in env and env['deployId']:
deploy = deploys_helper.get(request, env['deployId'])
current_build = builds_helper.get_build(request, deploy['buildId'])
return render(request, 'deploys/new_deploy.html', {
"env": env,
"env_promote": env_promote,
"buildName": env['buildName'],
"current_build": current_build,
"pageIndex": 1,
"pageSize": common.DEFAULT_BUILD_SIZE,
})
def _get_ongoing_deploys(request, index, size):
# ongoing deploys are defined as deploys with states as:
deploy_states = ["RUNNING", "FAILING"]
deployResult = deploys_helper.get_all(request, deployState=deploy_states,
pageIndex=index, pageSize=size)
deploy_summaries = []
for deploy in deployResult['deploys']:
env = environs_helper.get(request, deploy['envId'])
build = builds_helper.get_build(request, deploy['buildId'])
summary = {}
summary['deploy'] = deploy
summary['env'] = env
summary['build'] = build
deploy_summaries.append(summary)
return deploy_summaries
def _get_ongoing_deploys(request, index, size):
# ongoing deploys are defined as deploys with states as:
deploy_states = ["RUNNING", "FAILING"]
deployResult = deploys_helper.get_all(request, deployState=deploy_states,
pageIndex=index, pageSize=size)
deploy_summaries = []
for deploy in deployResult['deploys']:
env = environs_helper.get(request, deploy['envId'])
build = builds_helper.get_build(request, deploy['buildId'])
summary = {}
summary['deploy'] = deploy
summary['env'] = env
summary['build'] = build
deploy_summaries.append(summary)
return deploy_summaries
def rollback(request, name, stage):
query_dict = request.GET
to_deploy_id = query_dict.get('to_deploy_id', None)
envs = environs_helper.get_all_env_stages(request, name)
stages, env = common.get_all_stages(envs, stage)
result = deploys_helper.get_all(request,
envId=[env['id']],
pageIndex=1,
pageSize=DEFAULT_ROLLBACK_DEPLOY_NUM)
deploys = result.get("deploys")
# remove the first deploy if exists
if deploys:
deploys.pop(0)
# append the build info
deploy_summaries = []
branch = None
commit = None
build_id = None
for deploy in deploys:
build = builds_helper.get_build(request, deploy['buildId'])
summary = {}
summary['deploy'] = deploy
summary['build'] = build
if not to_deploy_id and deploy['state'] == 'SUCCEEDED':
to_deploy_id = deploy['id']
if to_deploy_id and to_deploy_id == deploy['id']:
branch = build['branch']
commit = build['commitShort']
build_id = build['id']
deploy_summaries.append(summary)
html = render_to_string(
"environs/env_rollback.html", {
"stages": stages,
"envs": envs,
"env": env,
"deploy_summaries": deploy_summaries,
"to_deploy_id": to_deploy_id,
"branch": branch,
"commit": commit,
"build_id": build_id,
"csrf_token": get_token(request),
})
return HttpResponse(html)
def get_hotfix_detail(request, id):
hotfix = hotfixs_helper.get(request, id)
deploy = deploys_helper.get(request, hotfix['baseDeployId'])
build = builds_helper.get_build(request, deploy['buildId'])
urlPattern = systems_helper.get_url_pattern(request)
commits = []
_create_commits(commits, urlPattern['template'], hotfix)
jenkins_url = "%s/%s/%s" % (BUILD_URL, hotfix['jobName'],
hotfix['jobNum']) if hotfix['jobName'] and hotfix['jobNum'] else ''
html = render_to_string('hotfixs/hotfix_detail.tmpl', {
"hotfix": hotfix,
"commits": commits,
"deploy": deploy,
"build": build,
"jenkins_url": jenkins_url
})
return HttpResponse(html)
def sendStartMessage(request, user, envName, stageName, notifyAuthor):
env = environs_helper.get_env_by_stage(request, envName, stageName)
deploy = deploys_helper.get(request, env['deployId'])
build = builds_helper.get_build(request, deploy['buildId'])
branch = build['branch']
Ngapp2DeployUtils().reset_finish_message_flag(stageName)
message = "{}/{}: deploy of {}/{} started. See details <{}|here>.".format(
envName, stageName, branch, build['commitShort'], get_deploy_url(envName, stageName))
systems_helper.send_chat_message(request, user.name, get_slack_channel(), message)
if not notifyAuthor:
return
author_msg = get_notify_authors_message(request, env, build)
if author_msg and is_prod():
systems_helper.send_chat_message(request, user.name, get_slack_channel(), author_msg)
elif author_msg:
logger.info("get author list message:{}".format(author_msg))
def get(self, request, name, stage):
env = environs_helper.get_env_by_stage(request, name, stage)
env_promote = environs_helper.get_env_promotes_config(
request, name, stage)
current_build = None
if 'deployId' in env and env['deployId']:
deploy = deploys_helper.get(request, env['deployId'])
current_build = builds_helper.get_build(request, deploy['buildId'])
return render(
request, 'deploys/new_deploy.html', {
"env": env,
"env_promote": env_promote,
"buildName": env['buildName'],
"current_build": current_build,
"pageIndex": 1,
"pageSize": common.DEFAULT_BUILD_SIZE,
})
def get_hotfix(request, name, stage, id):
env = environs_helper.get_env_by_stage(request, name, stage)
hotfix = hotfixs_helper.get(request, id)
deploy = deploys_helper.get(request, hotfix['baseDeployId'])
build = builds_helper.get_build(request, deploy['buildId'])
urlPattern = systems_helper.get_url_pattern(request)
commits = []
_create_commits(commits, urlPattern['template'], hotfix)
jenkins_url = "%s/%s/%s" % (BUILD_URL, hotfix['jobName'],
hotfix['jobNum']) if hotfix['jobName'] and hotfix['jobNum'] else ''
return render(request, 'hotfixs/hotfix_detail.html', {
"env": env,
"hotfix": hotfix,
"commits": commits,
"deploy": deploy,
"build": build,
"jenkins_url": jenkins_url
})
def sendStartMessage(request, user, envName, stageName, notifyAuthor):
env = environs_helper.get_env_by_stage(request, envName, stageName)
deploy = deploys_helper.get(request, env['deployId'])
build = builds_helper.get_build(request, deploy['buildId'])
branch = build['branch']
Ngapp2DeployUtils().reset_finish_message_flag(stageName)
message = "{}/{}: deploy of {}/{} started. See details <{}|here>.".format(
envName, stageName, branch, build['commitShort'], get_deploy_url(envName, stageName))
systems_helper.send_chat_message(request, user.name, get_slack_channel(), message)
if not notifyAuthor:
return
author_msg = get_notify_authors_message(request, env, build)
if author_msg and is_prod():
systems_helper.send_chat_message(request, user.name, get_slack_channel(), author_msg)
elif author_msg:
logger.info("get author list message:{}".format(author_msg))
def warn_no_succ_deploy_in_pred(request, name, stage, buildId):
""" Returns a warning message if a build doesn't have a successful deploy on the preceding stage.
TODO: we would have call backend twice since the getAllDeploys call does not support filtering on multiple states;
Also, getAllDeploys return all deploys with commits after the specific commit, it would be good if there is options
to return the exact matched deploys.
"""
env_promote = environs_helper.get_env_promotes_config(request, name, stage)
pred_stage = env_promote.get('predStageName')
if not pred_stage or pred_stage == BUILD_STAGE:
return HttpResponse("")
pred_env = environs_helper.get_env_by_stage(request, name, pred_stage)
build = builds_helper.get_build(request, buildId)
filter = {}
filter['envId'] = [pred_env['id']]
filter['commit'] = build['commit']
filter['repo'] = build['repo']
filter['oldestFirst'] = True
filter['deployState'] = "SUCCEEDING"
filter['pageIndex'] = 1
filter['pageSize'] = 1
result = deploys_helper.get_all(request, **filter)
succeeding_deploys = result['deploys']
if succeeding_deploys:
return HttpResponse("")
filter['deployState'] = "SUCCEEDED"
result = deploys_helper.get_all(request, **filter)
succeeded_deploys = result['deploys']
if succeeded_deploys:
return HttpResponse("")
html = render_to_string('warn_no_success_deploy_in_pred.tmpl', {
'envName': name,
'predStageName': pred_stage,
})
return HttpResponse(html)
def get(self, request):
ngapp2_deploy_utils = Ngapp2DeployUtils()
ok_to_serve = ngapp2_deploy_utils.get_ok_to_serve()
curr_build = ok_to_serve.get('default')
ok_to_serve_version = ok_to_serve.get('versions')
prev_build = ngapp2_deploy_utils.get_previous_build()
env_stage_a = environs_helper.get_env_by_stage(request, NGAPP_A, "prod")
env_stage_b = environs_helper.get_env_by_stage(request, NGAPP_B, "prod")
deploy_a = None
build_a = None
deploy_b = None
if env_stage_a.get('deployId'):
deploy_a = deploys_helper.get(request, env_stage_a['deployId'])
build_a = builds_helper.get_build(request, deploy_a['buildId'])
if env_stage_b.get('deployId'):
deploy_b = deploys_helper.get(request, env_stage_b['deployId'])
stage = getattr(NgappDeployStep, ngapp2_deploy_utils.get_status_from_zk().upper())
if stage != NgappDeployStep.SERVING_BUILD:
deploying_env = ngapp2_deploy_utils.get_deploying_env_from_zk()
else:
if build_a and build_a['commit'][:7] == curr_build[:7]:
deploying_env = env_stage_b['envName']
else:
deploying_env = env_stage_a['envName']
ngapp2_deploy_utils.set_deploying_env_to_zk(deploying_env)
if deploying_env == env_stage_a['envName']:
curr_env = EnvStatus(env_stage_b, deploy_b, curr_build)
prev_env = EnvStatus(env_stage_a, deploy_a, prev_build)
else:
curr_env = EnvStatus(env_stage_a, deploy_a, curr_build)
prev_env = EnvStatus(env_stage_b, deploy_b, prev_build)
return render(request, 'ngapp2/ngapp2_deploy.html', {
"prev_env": prev_env,
"curr_env": curr_env,
"deploy_stage": stage,
"deploying_env": deploying_env,
"serve_version_count": len(ok_to_serve_version)
})
def warn_no_succ_deploy_in_pred(request, name, stage, buildId):
""" Returns a warning message if a build doesn't have a successful deploy on the preceding stage.
TODO: we would have call backend twice since the getAllDeploys call does not support filtering on multiple states;
Also, getAllDeploys return all deploys with commits after the specific commit, it would be good if there is options
to return the exact matched deploys.
"""
env_promote = environs_helper.get_env_promotes_config(request, name, stage)
pred_stage = env_promote.get('predStageName')
if not pred_stage or pred_stage == BUILD_STAGE:
return HttpResponse("")
pred_env = environs_helper.get_env_by_stage(request, name, pred_stage)
build = builds_helper.get_build(request, buildId)
filter = {}
filter['envId'] = [pred_env['id']]
filter['commit'] = build['commit']
filter['repo'] = build['repo']
filter['oldestFirst'] = True
filter['deployState'] = "SUCCEEDING"
filter['pageIndex'] = 1
filter['pageSize'] = 1
result = deploys_helper.get_all(request, **filter)
succeeding_deploys = result['deploys']
if succeeding_deploys:
return HttpResponse("")
filter['deployState'] = "SUCCEEDED"
result = deploys_helper.get_all(request, **filter)
succeeded_deploys = result['deploys']
if succeeded_deploys:
return HttpResponse("")
html = render_to_string('warn_no_success_deploy_in_pred.tmpl', {
'envName': name,
'predStageName': pred_stage,
})
return HttpResponse(html)
def get_hotfix_detail(request, id):
hotfix = hotfixs_helper.get(request, id)
deploy = deploys_helper.get(request, hotfix['baseDeployId'])
build = builds_helper.get_build(request, deploy['buildId'])
urlPattern = systems_helper.get_url_pattern(request)
commits = []
_create_commits(commits, urlPattern['template'], hotfix)
jenkins_url = "%s/%s/%s" % (
BUILD_URL, hotfix['jobName'],
hotfix['jobNum']) if hotfix['jobName'] and hotfix['jobNum'] else ''
html = render_to_string(
'hotfixs/hotfix_detail.tmpl', {
"hotfix": hotfix,
"commits": commits,
"deploy": deploy,
"build": build,
"jenkins_url": jenkins_url
})
return HttpResponse(html)
def get_builds(request, name, stage):
env = environs_helper.get_env_by_stage(request, name, stage)
env_promote = environs_helper.get_env_promotes_config(request, name, stage)
show_lock = False
if env_promote['type'] == 'AUTO' and env_promote['predStage'] and \
env_promote['predStage'] == environs_helper.BUILD_STAGE:
show_lock = True
if 'buildName' not in env and not env['buildName']:
html = render_to_string('builds/simple_builds.tmpl', {
"builds": [],
"env": env,
"show_lock": show_lock,
})
return HttpResponse(html)
current_publish_date = 0
if 'deployId' in env and env['deployId']:
deploy = deploys_helper.get(request, env['deployId'])
build = builds_helper.get_build(request, deploy['buildId'])
current_publish_date = build['publishDate']
# return only the new builds
index = int(request.GET.get('page_index', '1'))
size = int(request.GET.get('page_size', common.DEFAULT_BUILD_SIZE))
builds = builds_helper.get_builds(request,
name=env['buildName'],
pageIndex=index,
pageSize=size)
new_builds = []
for build in builds:
if build['publishDate'] > current_publish_date:
new_builds.append(build)
html = render_to_string(
'builds/simple_builds.tmpl', {
"builds": new_builds,
"current_publish_date": current_publish_date,
"env": env,
"show_lock": show_lock,
})
return HttpResponse(html)
def rollback(request, name, stage):
query_dict = request.GET
to_deploy_id = query_dict.get('to_deploy_id', None)
envs = environs_helper.get_all_env_stages(request, name)
stages, env = common.get_all_stages(envs, stage)
result = deploys_helper.get_all(request, envId=[env['id']], pageIndex=1,
pageSize=DEFAULT_ROLLBACK_DEPLOY_NUM)
deploys = result.get("deploys")
# remove the first deploy if exists
if deploys:
deploys.pop(0)
# append the build info
deploy_summaries = []
branch = None
commit = None
build_id = None
for deploy in deploys:
build = builds_helper.get_build(request, deploy['buildId'])
summary = {}
summary['deploy'] = deploy
summary['build'] = build
if not to_deploy_id and deploy['state'] == 'SUCCEEDED':
to_deploy_id = deploy['id']
if to_deploy_id and to_deploy_id == deploy['id']:
branch = build['branch']
commit = build['commitShort']
build_id = build['id']
deploy_summaries.append(summary)
html = render_to_string("environs/env_rollback.html", {
"stages": stages,
"envs": envs,
"env": env,
"deploy_summaries": deploy_summaries,
"to_deploy_id": to_deploy_id,
"branch": branch,
"commit": commit,
"build_id": build_id,
"csrf_token": get_token(request),
})
return HttpResponse(html)
def get_hotfix(request, name, stage, id):
env = environs_helper.get_env_by_stage(request, name, stage)
hotfix = hotfixs_helper.get(request, id)
deploy = deploys_helper.get(request, hotfix['baseDeployId'])
build = builds_helper.get_build(request, deploy['buildId'])
urlPattern = systems_helper.get_url_pattern(request)
commits = []
_create_commits(commits, urlPattern['template'], hotfix)
jenkins_url = "%s/%s/%s" % (
BUILD_URL, hotfix['jobName'],
hotfix['jobNum']) if hotfix['jobName'] and hotfix['jobNum'] else ''
return render(
request, 'hotfixs/hotfix_detail.html', {
"env": env,
"hotfix": hotfix,
"commits": commits,
"deploy": deploy,
"build": build,
"jenkins_url": jenkins_url
})
def sendFinishMessage(request, user, envName, stageName, state):
ngapp2DeployUtils = Ngapp2DeployUtils()
# if we've already sent the message
if ngapp2DeployUtils.get_finish_message_flag(stageName) == "True":
return
env = environs_helper.get_env_by_stage(request, envName, stageName)
deploy = deploys_helper.get(request, env['deployId'])
build = builds_helper.get_build(request, deploy['buildId'])
branch = build['branch']
commit = build['commit']
weblink = get_deploy_url(envName, stageName)
state = state.lower()
if state == "succeeding" or state == "done":
template = "{}/{}: deploy of {}/{} completed successfully. See details <{}|here>"
else:
template = "{}/{}: deploy of {}/{} failed. See details <{}|here>"
message = template.format(envName, stageName, branch, commit[:7], weblink)
ngapp2DeployUtils.set_finish_message_flag(stageName)
systems_helper.send_chat_message(request, deploy['operator'], get_slack_channel(), message)
def sendFinishMessage(request, user, envName, stageName, state):
ngapp2DeployUtils = Ngapp2DeployUtils()
# if we've already sent the message
if ngapp2DeployUtils.get_finish_message_flag(stageName) == "True":
return
ngapp2DeployUtils.set_finish_message_flag(stageName)
env = environs_helper.get_env_by_stage(request, envName, stageName)
deploy = deploys_helper.get(request, env['deployId'])
build = builds_helper.get_build(request, deploy['buildId'])
branch = build['branch']
commit = build['commit']
weblink = get_deploy_url(envName, stageName)
state = state.lower()
if state == "succeeding" or state == "done":
template = "{}/{}: deploy of {}/{} completed successfully. See details <{}|here>"
else:
template = "{}/{}: deploy of {}/{} failed. See details <{}|here>"
message = template.format(envName, stageName, branch, commit[:7], weblink)
systems_helper.send_chat_message(request, deploy['operator'], get_slack_channel(), message)
def patch(request, name, stage):
env = environs_helper.get_env_by_stage(request, name, stage)
deploy_id = request.GET.get('base_deploy', None)
if not deploy_id:
deploy_id = env['deployId']
deploy = deploys_helper.get(request, deploy_id)
build = builds_helper.get_build(request, deploy['buildId'])
commits, truncated, new_start_sha = common.get_commits_batch(request, build['repo'],
env['branch'],
build['commit'],
keep_first=True)
return render(request, 'hotfixs/cherry_pick.html', {
"deploy": deploy,
"build": build,
"env": env,
"commits": commits,
"start_sha": new_start_sha,
"end_sha": build['commit'],
"repo": build['repo'],
"truncated": truncated,
})
def get_builds(request, name, stage):
env = environs_helper.get_env_by_stage(request, name, stage)
env_promote = environs_helper.get_env_promotes_config(request, name, stage)
show_lock = False
if env_promote['type'] == 'AUTO' and env_promote['predStage'] and \
env_promote['predStage'] == environs_helper.BUILD_STAGE:
show_lock = True
if 'buildName' not in env and not env['buildName']:
html = render_to_string('builds/simple_builds.tmpl', {
"builds": [],
"env": env,
"show_lock": show_lock,
})
return HttpResponse(html)
current_publish_date = 0
if 'deployId' in env and env['deployId']:
deploy = deploys_helper.get(request, env['deployId'])
build = builds_helper.get_build(request, deploy['buildId'])
current_publish_date = build['publishDate']
# return only the new builds
index = int(request.GET.get('page_index', '1'))
size = int(request.GET.get('page_size', common.DEFAULT_BUILD_SIZE))
builds = builds_helper.get_builds(request, name=env['buildName'], pageIndex=index,
pageSize=size)
new_builds = []
for build in builds:
if build['publishDate'] > current_publish_date:
new_builds.append(build)
html = render_to_string('builds/simple_builds.tmpl', {
"builds": new_builds,
"current_publish_date": current_publish_date,
"env": env,
"show_lock": show_lock,
})
return HttpResponse(html)
def gen_report(request, env, progress, sortByStatus="false"):
agentStats = []
firstTimeAgentStats = []
deployStats = {}
deprecatedDeployStats = []
# always set the current
deploy = deploys_helper.get(request, env['deployId'])
build = builds_helper.get_build(request, deploy["buildId"])
stageDistMap = genStageDistMap()
stateDistMap = genStateDistMap()
currentDeployStat = DeployStatistics(deploy=deploy,
build=build,
stageDistMap=stageDistMap,
stateDistMap=stateDistMap)
deployStats[env['deployId']] = currentDeployStat
for agent in progress["agents"]:
if agent["firstDeploy"]:
firstTimeAgentStats.append(
addToEnvReport(request, deployStats, agent, env))
else:
agentStats.append(addToEnvReport(request, deployStats, agent, env))
if sortByStatus == "true":
agentStats.sort(cmp=lambda x, y: _compare_agent_status(x, y))
for key, value in deployStats.iteritems():
if key != env['deployId']:
deprecatedDeployStats.append(value)
return AgentReport(firstTimeAgentStats=firstTimeAgentStats,
agentStats=agentStats,
currentDeployStat=currentDeployStat,
deprecatedDeployStats=deprecatedDeployStats,
missingHosts=progress["missingHosts"],
envName=env['envName'],
stageName=env['stageName'])
def get_notify_authors_message(request, env, current_build):
try:
last_succ_deploy = get_last_ngapp2_deploy(request, env['envName'], env['stageName'])
last_build = builds_helper.get_build(request, last_succ_deploy['buildId'])
start_commit = current_build.get('commit')
end_commit = last_build.get('commit')
repo = last_build['repo']
commits = common.get_commits_between(request, repo=repo, startSha=start_commit,
endSha=end_commit)
authors = set()
for commit in commits:
author = commit.get('author')
if not author or author.lower() == "unknown":
continue
authors.add("<@{}>".format(author))
mentions = ",".join(authors)
return "This deploy features {} commits from the " \
"following pingineers: {}. See changes <{}|here>".format(len(commits), mentions,
get_ngapp2_compare_deploy_url(
current_build.get("id"),
last_build.get("id")))
except:
logger.error(traceback.format_exc())
return ""
def patch(request, name, stage):
env = environs_helper.get_env_by_stage(request, name, stage)
deploy_id = request.GET.get('base_deploy', None)
if not deploy_id:
deploy_id = env['deployId']
deploy = deploys_helper.get(request, deploy_id)
build = builds_helper.get_build(request, deploy['buildId'])
commits, truncated, new_start_sha = common.get_commits_batch(
request,
build['repo'],
env['branch'],
build['commit'],
keep_first=True)
return render(
request, 'hotfixs/cherry_pick.html', {
"deploy": deploy,
"build": build,
"env": env,
"commits": commits,
"start_sha": new_start_sha,
"end_sha": build['commit'],
"repo": build['repo'],
"truncated": truncated,
})