def put(self, repo_name, operation):
if repo_name in dms:
dm = dms[repo_name]
if operation == 'cancel':
dm.status.lock_acquire()
dm.status.set_cancel_flag(True)
dm.status.lock_release()
# log to db
mongodb_client['deployment']['operation_log'].insert({
"userId":
self.get_current_user().user_id,
"username":
self.get_current_user().username,
"repoName":
repo_name,
"operation":
operation,
"statusSnapshot":
serialize_status(dm.get_status_info()),
"createTimeStamp":
int(time.time())
})
return
elif operation == 'enable_auto':
dm.status.lock_acquire()
dm.status.enable_auto_deploy()
dm.status.lock_release()
elif operation == 'disable_auto':
dm.status.lock_acquire()
dm.status.disable_auto_deploy()
dm.status.lock_release()
# log to db
mongodb_client['deployment']['operation_log'].insert({
"userId":
self.get_current_user().user_id,
"username":
self.get_current_user().username,
"repoName":
repo_name,
"operation":
operation,
"createTimeStamp":
int(time.time())
})
def put(self, repo_name, branch, operation):
if repo_name in dmc and branch in dmc[repo_name]:
dm = dmc[repo_name][branch]
if operation == 'cancel':
dm.status.lock_acquire()
dm.status.set_cancel_flag(True)
dm.status.lock_release()
# log to db
mongodb_client['deployment']['operation_log'].insert({
"userId": self.get_current_user().user_id,
"username": self.get_current_user().username,
"repoName": repo_name,
"branch": branch,
"operation": operation,
"statusSnapshot": serialize_status(dm.get_status_info()),
"createTimeStamp": int(time.time())
})
return
elif operation == 'enable_auto':
dm.status.lock_acquire()
dm.status.enable_auto_deploy()
dm.status.lock_release()
elif operation == 'disable_auto':
dm.status.lock_acquire()
dm.status.disable_auto_deploy()
dm.status.lock_release()
# log to db
mongodb_client['deployment']['operation_log'].insert({
"userId": self.get_current_user().user_id,
"username": self.get_current_user().username,
"repoName": repo_name,
"branch": branch,
"operation": operation,
"createTimeStamp": int(time.time())
})
def rollback(self, payload):
try:
event_id = payload.event_id
head_commit = payload.head_commit
repo = self.repo
logger_server.info("Start rollback Event[{event_id}]...".format(event_id=event_id))
# Change Status to Rollback and disable auto deployment
self.status.lock_acquire()
self.status.disable_auto_deploy()
self.status.set_status(DeployStatus.ROLLBACK)
self.status.init_host_status()
self.status.lock_release()
backup_package = self.status.get_backup_filename()
release_package = self.status.get_package_filename()
# adjust if has backup_package and release_package, before this stage need not deploy
if backup_package and release_package:
# Step 1.Get change files
self.stage("Get change files", 20)
try:
change_files = repo.get_change_files(self.status.get_last_commit(), head_commit)
if _DEBUG:
logger_server.debug("change files:" + str(change_files))
except:
# May be pull failed so head_commit is not a valid object
change_files = []
if change_files:
# Step 2.Get Services to restart
self.stage("Get Services to restart", 40)
restart_services = repo.get_service_to_restart(change_files)
if _DEBUG:
logger_server.debug("service to restart:" + str(restart_services))
# Step 3.Release backup package
self.stage("Release backup package", 50)
for one_host in self.status.get_hosts():
# Step 3.1 SYNC package
self.status.set_host_status(one_host, HostStatus.DEPLOYING)
if _DEBUG:
logger_server.debug("Rsync files to {host}".format(host=one_host))
self.stage("Rsync files to {host}".format(host=one_host),
self.status.get_process_percent() + self.status.calculate_process_interval(30, 2))
self.repo.release(backup_package, one_host)
# Step 3.2.Restart Services
if _DEBUG:
logger_server.debug("Restart services at {host}".format(host=one_host))
self.stage("Restart services at {host}".format(host=one_host),
self.status.get_process_percent() + self.status.calculate_process_interval(30, 2))
repo.restart_services(restart_services, one_host)
self.status.set_host_status(one_host, HostStatus.SUCCESS)
else:
logger_server.info("Nothing has been changed")
else:
logger_server.info("Nothing need deploy")
# Step 5/Step 1.Git Reset
self.stage("Git Reset", 90)
repo.reset(self.status.get_last_commit())
# Logging
logger_server.info(
"Rollback Event[{event_id}] done.".format(event_id=event_id))
# Change Status to Idle
self.status.lock_acquire()
self.status.set_status(DeployStatus.IDLE)
self.status.init_host_status()
self.stage(None)
self.status.update_last_release_tags()
self.status.set_running_task(None)
self.status.set_cancel_flag(False)
self.status.set_backup_filename(None)
self.status.set_package_filename(None)
self.status.update_last_commit()
if _DEBUG:
logger_server.debug("Status" + str(self.status.export_status()))
self.status.lock_release()
except Exception as ex:
mongodb_client['deployment']['deploy_log'].insert({'event_id': payload.event_id,
'type': 'rollback',
'result': 'fail',
'exception': str(ex),
'createdTimeStamp': int(time.time()),
'status_snapshot': serialize_status(
self.status.export_status())})
raise ex
def deploy(self, payload):
try:
event_id = payload.event_id
repo = self.repo
logger_server.info("Start deploy Event[{event_id}]...".format(event_id=event_id))
datetime_start = datetime.datetime.now()
# Change & init status to Running
self.status.set_status(DeployStatus.RUNNING)
self.status.set_running_task(payload)
self.status.init_host_status()
self.status.set_cancel_flag(False)
# Release lock to let other event occur
self.status.lock_release()
if _DEBUG:
logger_server.debug(self.status.export_status())
# Step 1.Clean Git Work Dir
self.stage("Clean Git Work Dir", 10)
repo.clean()
# Step 2.Fetch data
self.stage("Fetch data", 20)
fetch_content = repo.fetch()
# Step 3.Reset to tag
self.stage("Reset to tag", 30)
repo.reset(payload.tag)
commit_after_pull = self.repo.get_last_commit()
commit_before_pull = self.status.get_last_commit()
if _DEBUG:
logger_server.debug(
"Start commit:{start},end commit:{end}".format(start=commit_before_pull, end=commit_after_pull))
change_files = self.repo.get_change_files(commit_before_pull, commit_after_pull)
if _DEBUG:
logger_server.debug("Change files:" + str(change_files))
if change_files:
# Step 4.NPM & Python Package Install
self.stage("NPM & Python Package Install", 40)
repo.install_pkg(repo.get_pkg_to_install(change_files))
# Step 5.Run Post-actions
self.stage("Run Post-actions", 50)
repo.handle_post_actions()
# Step 6.Backup Deploy Directory
self.stage("Backup Deploy Directory", 60)
backup_tar_file = repo.backup_deploy_dir()
self.status.set_backup_filename(backup_tar_file)
if _DEBUG:
logger_server.debug("Backup File:" + backup_tar_file)
# Step 7.Tar directory
self.stage("Tar directory", 70)
package_file = repo.tar_git_dir(payload.tag)
self.status.set_package_filename(package_file)
if _DEBUG:
logger_server.debug("Package File:" + package_file)
# Step 8.Release Tar
self.stage("Release Package", 80)
for one_host in self.status.get_hosts():
try:
self.status.set_host_status(one_host, HostStatus.DEPLOYING)
# Step 8.1.SYNC package
if _DEBUG:
logger_server.debug("Rsync files to {host}".format(host=one_host))
self.stage("Rsync files to {host}".format(host=one_host),
self.status.get_process_percent() + self.status.calculate_process_interval(20, 2))
repo.release(package_file, one_host)
# Step 8.2.Get Services to restart
restart_services = repo.get_service_to_restart(change_files)
if _DEBUG:
logger_server.debug("service to restart:" + str(restart_services))
# Step 8.3.Restart Services
if _DEBUG:
logger_server.debug("Restart services at {host}".format(host=one_host))
self.stage("Restart services at {host}".format(host=one_host),
self.status.get_process_percent() + self.status.calculate_process_interval(20, 2))
repo.restart_services(restart_services, one_host)
self.status.set_host_status(one_host, HostStatus.SUCCESS)
except RepositoryException as ex:
self.status.set_host_status(one_host, HostStatus.FAULT)
raise ex
self.stage("Finish", 100)
else:
logger_server.info("Nothing has been changed")
# Logging
logger_server.info(
"Deploy Event[{event_id}] done.".format(event_id=event_id))
datetime_end = datetime.datetime.now()
mongodb_client['deployment']['deploy_log'].insert({'event_id': payload.event_id,
'type': 'deploy',
'repo_name': self.repo.repo_name,
'result': "success",
'cost_time': str(datetime_end - datetime_start),
'createdTimeStamp': int(time.time())})
mail_manager.send_success_mail(payload, self.repo.get_tag_info(payload.tag), datetime_start, datetime_end)
# Check if exist waiting task to run
self.status.lock_acquire()
self.status.update_last_commit()
if self.status.has_waiting_task():
self.deploy(self.status.get_first_waiting_task())
return
# Change Status to Idle, Trick: Here will be reached once during one working list
self.status.set_status(DeployStatus.IDLE)
self.status.init_host_status()
self.stage(None)
self.status.update_last_release_tags()
self.status.set_running_task(None)
self.status.set_cancel_flag(False)
self.status.set_backup_filename(None)
self.status.set_package_filename(None)
if _DEBUG:
logger_server.debug("Status" + str(self.status.export_status()))
self.status.lock_release()
except DeployCancel as ex:
datetime_end = datetime.datetime.now()
mongodb_client['deployment']['deploy_log'].insert({'event_id': payload.event_id,
'type': 'deploy_cancel',
'result': 'success',
'createdTimeStamp': int(time.time())})
mail_manager.send_cancel_success_mail(payload, self.repo.get_tag_info(payload.tag), datetime_start, datetime_end)
except Exception as ex:
exception_str = str(ex)
stack_info = traceback.format_exc()
mail_manager.send_error_mail(payload, self.repo.get_tag_info(payload.tag), datetime_start,
stack_info)
mongodb_client['deployment']['deploy_log'].insert({'event_id': payload.event_id,
'type': 'deploy',
'result': 'exception',
'exception': exception_str,
'trace': stack_info,
'createdTimeStamp': int(time.time()),
'status_snapshot': serialize_status(
self.status.export_status())})
try:
self.rollback(payload)
datetime_end = datetime.datetime.now()
mongodb_client['deployment']['deploy_log'].insert({'event_id': payload.event_id,
'type': 'deploy_rollback',
'result': 'success',
'exception': exception_str,
'trace': stack_info,
'createdTimeStamp': int(time.time()),
'status_snapshot': serialize_status(
self.status.export_status())})
mail_manager.send_rollback_success_mail(payload, self.repo.get_tag_info(payload.tag), datetime_start,
datetime_end, stack_info)
except Exception as rollback_ex:
rollback_exception_str = str(rollback_ex)
rollback_stack_info = traceback.format_exc()
datetime_end = datetime.datetime.now()
mongodb_client['deployment']['deploy_log'].insert({'event_id': payload.event_id,
'type': 'deploy_rollback',
'result': 'fail',
'exception_original': exception_str,
'trace_original': stack_info,
'exception_rollback': rollback_exception_str,
'trace_rollback': rollback_stack_info,
'createdTimeStamp': int(time.time()),
'status_snapshot': serialize_status(
self.status.export_status())})
mail_manager.send_rollback_fail_mail(payload, self.repo.get_tag_info(payload.tag), datetime_start,
datetime_end, stack_info, rollback_stack_info)
def rollback(self, payload):
try:
event_id = payload.event_id
head_commit = payload.head_commit
repo = self.repo
logger_server.info("Start rollback Event[{event_id}]...".format(event_id=event_id))
# Change Status to Rollback and disable auto deployment
self.status.lock_acquire()
self.status.disable_auto_deploy()
self.status.set_status(DeployStatus.ROLLBACK)
self.status.lock_release()
# Step 1.Get change files
self.stage("Get change files", 20)
try:
change_files = repo.get_change_files(self.status.get_last_commit(), head_commit)
if _DEBUG:
logger_server.debug("change files:" + str(change_files))
except:
# May be pull failed so head_commit is not a valid object
change_files = []
# Step 2.Get Services to restart
self.stage("Get Services to restart", 40)
restart_services = repo.get_service_to_restart(change_files)
if _DEBUG:
logger_server.debug("service to restart:" + str(restart_services))
# Step 3.Git Reset
self.stage("Git Reset", 60)
repo.reset(self.status.get_last_commit())
# Step 4.Release
self.stage("Release", 80)
for one_host in self.status.get_hosts():
try:
self.status.set_host_status(one_host, HostStatus.DEPLOYING)
# Step 4.1.SYNC
if _DEBUG:
logger_server.debug("Rsync files to {host}".format(host=one_host))
self.stage("Rsync files to {host}".format(host=one_host),
self.status.get_process_percent() + self.status.calculate_process_interval(20, 2))
repo.rsync(self.repo.git_path,
"deploy@{host}:{deploy}".format(host=one_host, deploy=self.repo.deploy_path),
"{git_path}/{file}".format(git_path=self.repo.git_path,file=self.repo.exclude_filename) if
self.repo.exclude_filename else None)
# Step 4.2.Restart Services
if _DEBUG:
logger_server.debug("Restart services at {host}".format(host=one_host))
self.stage("Restart services at {host}".format(host=one_host),
self.status.get_process_percent() + self.status.calculate_process_interval(20, 2))
repo.restart_services(restart_services, one_host)
self.status.set_host_status(one_host, HostStatus.SUCCESS)
except RepositoryException as ex:
self.status.set_host_status(one_host, HostStatus.FAULT)
raise ex
"""
# Step 4.Restart Services
self.stage("Restart Services", 80)
repo.restart_services(restart_services, self.status.get_hosts()[0])
"""
# Logging
logger_server.info(
"Rollback Event[{event_id}] done.".format(event_id=event_id))
# Change Status to Idle
self.status.lock_acquire()
self.status.set_status(DeployStatus.IDLE)
self.status.set_running_task(None)
self.stage(None)
self.status.update_last_commit()
if _DEBUG:
logger_server.debug("Status" + str(self.status.export_status()))
self.status.lock_release()
except Exception as ex:
mongodb_client['deployment']['deploy_log'].insert({'event_id': payload.event_id,
'type': 'rollback',
'result': 'fail',
'exception': str(ex),
'createdTimeStamp': int(time.time()),
'status_snapshot': serialize_status(
self.status.export_status())})
raise ex
def rollback(self, payload):
try:
event_id = payload.event_id
head_commit = payload.head_commit
repo = self.repo
logger_server.info("Start rollback Event[{event_id}]...".format(
event_id=event_id))
# Change Status to Rollback and disable auto deployment
self.status.lock_acquire()
self.status.disable_auto_deploy()
self.status.set_status(DeployStatus.ROLLBACK)
self.status.init_host_status()
self.status.lock_release()
backup_package = self.status.get_backup_filename()
release_package = self.status.get_package_filename()
# adjust if has backup_package and release_package, before this stage need not deploy
if backup_package and release_package:
# Step 1.Get change files
self.stage("Get change files", 20)
try:
change_files = repo.get_change_files(
self.status.get_last_commit(), head_commit)
if _DEBUG:
logger_server.debug("change files:" +
str(change_files))
except:
# May be pull failed so head_commit is not a valid object
change_files = []
if change_files:
# Step 2.Get Services to restart
self.stage("Get Services to restart", 40)
restart_services = repo.get_service_to_restart(
change_files)
if _DEBUG:
logger_server.debug("service to restart:" +
str(restart_services))
# Step 3.Release backup package
self.stage("Release backup package", 50)
for one_host in self.status.get_hosts():
# Step 3.1 SYNC package
self.status.set_host_status(one_host,
HostStatus.DEPLOYING)
if _DEBUG:
logger_server.debug(
"Rsync files to {host}".format(host=one_host))
self.stage(
"Rsync files to {host}".format(host=one_host),
self.status.get_process_percent() +
self.status.calculate_process_interval(30, 2))
self.repo.release(backup_package, one_host)
# Step 3.2.Restart Services
if _DEBUG:
logger_server.debug(
"Restart services at {host}".format(
host=one_host))
self.stage(
"Restart services at {host}".format(host=one_host),
self.status.get_process_percent() +
self.status.calculate_process_interval(30, 2))
repo.restart_services(restart_services, one_host)
self.status.set_host_status(one_host,
HostStatus.SUCCESS)
else:
logger_server.info("Nothing has been changed")
else:
logger_server.info("Nothing need deploy")
# Step 5/Step 1.Git Reset
self.stage("Git Reset", 90)
repo.reset(self.status.get_last_commit())
# Logging
logger_server.info(
"Rollback Event[{event_id}] done.".format(event_id=event_id))
# Change Status to Idle
self.status.lock_acquire()
self.status.set_status(DeployStatus.IDLE)
self.status.init_host_status()
self.stage(None)
self.status.update_last_release_tags()
self.status.set_running_task(None)
self.status.set_cancel_flag(False)
self.status.set_backup_filename(None)
self.status.set_package_filename(None)
self.status.update_last_commit()
if _DEBUG:
logger_server.debug("Status" +
str(self.status.export_status()))
self.status.lock_release()
except Exception as ex:
mongodb_client['deployment']['deploy_log'].insert({
'event_id':
payload.event_id,
'type':
'rollback',
'result':
'fail',
'exception':
str(ex),
'createdTimeStamp':
int(time.time()),
'status_snapshot':
serialize_status(self.status.export_status())
})
raise ex
def deploy(self, payload):
try:
event_id = payload.event_id
repo = self.repo
logger_server.info(
"Start deploy Event[{event_id}]...".format(event_id=event_id))
datetime_start = datetime.datetime.now()
# Change & init status to Running
self.status.set_status(DeployStatus.RUNNING)
self.status.set_running_task(payload)
self.status.init_host_status()
self.status.set_cancel_flag(False)
# Release lock to let other event occur
self.status.lock_release()
if _DEBUG:
logger_server.debug(self.status.export_status())
# Step 1.Clean Git Work Dir
self.stage("Clean Git Work Dir", 10)
repo.clean()
# Step 2.Fetch data
self.stage("Fetch data", 20)
fetch_content = repo.fetch()
# Step 3.Reset to tag
self.stage("Reset to tag", 30)
repo.reset(payload.tag)
commit_after_pull = self.repo.get_last_commit()
commit_before_pull = self.status.get_last_commit()
if _DEBUG:
logger_server.debug(
"Start commit:{start},end commit:{end}".format(
start=commit_before_pull, end=commit_after_pull))
change_files = self.repo.get_change_files(commit_before_pull,
commit_after_pull)
if _DEBUG:
logger_server.debug("Change files:" + str(change_files))
if change_files:
# Step 4.NPM & Python Package Install
self.stage("NPM & Python Package Install", 40)
repo.install_pkg(repo.get_pkg_to_install(change_files))
# Step 5.Run Post-actions
self.stage("Run Post-actions", 50)
repo.handle_post_actions()
# Step 6.Backup Deploy Directory
self.stage("Backup Deploy Directory", 60)
backup_tar_file = repo.backup_deploy_dir()
self.status.set_backup_filename(backup_tar_file)
if _DEBUG:
logger_server.debug("Backup File:" + backup_tar_file)
# Step 7.Tar directory
self.stage("Tar directory", 70)
package_file = repo.tar_git_dir(payload.tag)
self.status.set_package_filename(package_file)
if _DEBUG:
logger_server.debug("Package File:" + package_file)
# Step 8.Release Tar
self.stage("Release Package", 80)
for one_host in self.status.get_hosts():
try:
self.status.set_host_status(one_host,
HostStatus.DEPLOYING)
# Step 8.1.SYNC package
if _DEBUG:
logger_server.debug(
"Rsync files to {host}".format(host=one_host))
self.stage(
"Rsync files to {host}".format(host=one_host),
self.status.get_process_percent() +
self.status.calculate_process_interval(20, 2))
repo.release(package_file, one_host)
# Step 8.2.Get Services to restart
restart_services = repo.get_service_to_restart(
change_files)
if _DEBUG:
logger_server.debug("service to restart:" +
str(restart_services))
# Step 8.3.Restart Services
if _DEBUG:
logger_server.debug(
"Restart services at {host}".format(
host=one_host))
self.stage(
"Restart services at {host}".format(host=one_host),
self.status.get_process_percent() +
self.status.calculate_process_interval(20, 2))
repo.restart_services(restart_services, one_host)
self.status.set_host_status(one_host,
HostStatus.SUCCESS)
except RepositoryException as ex:
self.status.set_host_status(one_host, HostStatus.FAULT)
raise ex
self.stage("Finish", 100)
else:
logger_server.info("Nothing has been changed")
# Logging
logger_server.info(
"Deploy Event[{event_id}] done.".format(event_id=event_id))
datetime_end = datetime.datetime.now()
mongodb_client['deployment']['deploy_log'].insert({
'event_id':
payload.event_id,
'type':
'deploy',
'repo_name':
self.repo.repo_name,
'result':
"success",
'cost_time':
str(datetime_end - datetime_start),
'createdTimeStamp':
int(time.time())
})
mail_manager.send_success_mail(payload,
self.repo.get_tag_info(payload.tag),
datetime_start, datetime_end)
# Check if exist waiting task to run
self.status.lock_acquire()
self.status.update_last_commit()
if self.status.has_waiting_task():
self.deploy(self.status.get_first_waiting_task())
return
# Change Status to Idle, Trick: Here will be reached once during one working list
self.status.set_status(DeployStatus.IDLE)
self.status.init_host_status()
self.stage(None)
self.status.update_last_release_tags()
self.status.set_running_task(None)
self.status.set_cancel_flag(False)
self.status.set_backup_filename(None)
self.status.set_package_filename(None)
if _DEBUG:
logger_server.debug("Status" +
str(self.status.export_status()))
self.status.lock_release()
except DeployCancel as ex:
datetime_end = datetime.datetime.now()
mongodb_client['deployment']['deploy_log'].insert({
'event_id':
payload.event_id,
'type':
'deploy_cancel',
'result':
'success',
'createdTimeStamp':
int(time.time())
})
mail_manager.send_cancel_success_mail(
payload, self.repo.get_tag_info(payload.tag), datetime_start,
datetime_end)
except Exception as ex:
exception_str = str(ex)
stack_info = traceback.format_exc()
mail_manager.send_error_mail(payload,
self.repo.get_tag_info(payload.tag),
datetime_start, stack_info)
mongodb_client['deployment']['deploy_log'].insert({
'event_id':
payload.event_id,
'type':
'deploy',
'result':
'exception',
'exception':
exception_str,
'trace':
stack_info,
'createdTimeStamp':
int(time.time()),
'status_snapshot':
serialize_status(self.status.export_status())
})
try:
self.rollback(payload)
datetime_end = datetime.datetime.now()
mongodb_client['deployment']['deploy_log'].insert({
'event_id':
payload.event_id,
'type':
'deploy_rollback',
'result':
'success',
'exception':
exception_str,
'trace':
stack_info,
'createdTimeStamp':
int(time.time()),
'status_snapshot':
serialize_status(self.status.export_status())
})
mail_manager.send_rollback_success_mail(
payload, self.repo.get_tag_info(payload.tag),
datetime_start, datetime_end, stack_info)
except Exception as rollback_ex:
rollback_exception_str = str(rollback_ex)
rollback_stack_info = traceback.format_exc()
datetime_end = datetime.datetime.now()
mongodb_client['deployment']['deploy_log'].insert({
'event_id':
payload.event_id,
'type':
'deploy_rollback',
'result':
'fail',
'exception_original':
exception_str,
'trace_original':
stack_info,
'exception_rollback':
rollback_exception_str,
'trace_rollback':
rollback_stack_info,
'createdTimeStamp':
int(time.time()),
'status_snapshot':
serialize_status(self.status.export_status())
})
mail_manager.send_rollback_fail_mail(
payload, self.repo.get_tag_info(payload.tag),
datetime_start, datetime_end, stack_info,
rollback_stack_info)
def rollback(self, payload):
try:
event_id = payload.event_id
head_commit = payload.head_commit
repo = self.repo
logger_server.info("Start rollback Event[{event_id}]...".format(
event_id=event_id))
# Change Status to Rollback and disable auto deployment
self.status.lock_acquire()
self.status.disable_auto_deploy()
self.status.set_status(DeployStatus.ROLLBACK)
self.status.lock_release()
# Step 1.Get change files
self.stage("Get change files", 20)
try:
change_files = repo.get_change_files(
self.status.get_last_commit(), head_commit)
if _DEBUG:
logger_server.debug("change files:" + str(change_files))
except:
# May be pull failed so head_commit is not a valid object
change_files = []
# Step 2.Get Services to restart
self.stage("Get Services to restart", 40)
restart_services = repo.get_service_to_restart(change_files)
if _DEBUG:
logger_server.debug("service to restart:" +
str(restart_services))
# Step 3.Git Reset
self.stage("Git Reset", 60)
repo.reset(self.status.get_last_commit())
# Step 4.Restart Services
self.stage("Restart Services", 80)
repo.restart_services(restart_services, self.status.get_hosts()[0])
# Logging
logger_server.info(
"Rollback Event[{event_id}] done.".format(event_id=event_id))
# Change Status to Idle
self.status.lock_acquire()
self.status.set_status(DeployStatus.IDLE)
self.status.set_running_task(None)
self.stage(None)
self.status.update_last_commit()
if _DEBUG:
logger_server.debug("Status" +
str(self.status.export_status()))
self.status.lock_release()
except Exception as ex:
mongodb_client['deployment']['deploy_log'].insert({
'event_id':
payload.event_id,
'type':
'rollback',
'result':
'fail',
'exception':
str(ex),
'createdTimeStamp':
int(time.time()),
'status_snapshot':
serialize_status(self.status.export_status())
})
raise ex
def deploy(self, payload):
try:
event_id = payload.event_id
repo = self.repo
logger_server.info(
"Start deploy Event[{event_id}]...".format(event_id=event_id))
datetime_start = datetime.datetime.now()
# Change & init status to Running
self.status.set_status(DeployStatus.RUNNING)
self.status.set_running_task(payload)
self.status.init_host_status()
self.status.set_cancel_flag(False)
# Release lock to let other event occur
self.status.lock_release()
if _DEBUG:
logger_server.debug(self.status.export_status())
# Step 1.Clean Git Work Dir
self.stage("Clean Git Work Dir", 20)
repo.clean()
# Step 2.Pull data
self.stage("Pull data", 30)
pull_content = repo.pull()
# Step 3.Get change files
self.stage("Get change files", 40)
commit_after_pull = self.repo.get_last_commit()
commit_before_pull = self.status.get_last_commit()
if _DEBUG:
logger_server.debug(
"Start commit:{start},end commit:{end}".format(
start=commit_before_pull, end=commit_after_pull))
change_files = self.repo.get_change_files(commit_before_pull,
commit_after_pull)
if _DEBUG:
logger_server.debug("Change files:" + str(change_files))
if change_files:
# Step 4.NPM & Python Package Install
self.stage("NPM & Python Package Install", 60)
repo.install_pkg(repo.get_pkg_to_install(change_files))
# Step 5.Run Post-actions
self.stage("Run Post-actions", 70)
repo.handle_post_actions()
# Step 6.Get Services to restarts
self.stage("Get Services to restart", 80)
restart_services = repo.get_service_to_restart(change_files)
if _DEBUG:
logger_server.debug("service to restart:" +
str(restart_services))
# Step 7.Restart Services
self.stage("Restart Services", 90)
repo.restart_services(restart_services,
self.status.get_hosts()[0])
self.stage("Finish", 100)
else:
logger_server.info("Nothing has been changed")
# Logging
logger_server.info(
"Deploy Event[{event_id}] done.".format(event_id=event_id))
datetime_end = datetime.datetime.now()
mongodb_client['deployment']['deploy_log'].insert({
'event_id':
payload.event_id,
'type':
'deploy',
'repo_name':
self.repo.repo_name,
'result':
"success",
'cost_time':
str(datetime_end - datetime_start),
'createdTimeStamp':
int(time.time())
})
# Check if exist waiting task to run
self.status.lock_acquire()
self.status.update_last_commit()
if self.status.has_waiting_task():
self.deploy(self.status.get_first_waiting_task())
return
# Change Status to Idle, Trick: Here will be reached once during one working list
self.status.set_status(DeployStatus.IDLE)
self.stage(None)
self.status.set_running_task(None)
self.status.set_cancel_flag(False)
if _DEBUG:
logger_server.debug("Status" +
str(self.status.export_status()))
self.status.lock_release()
except DeployCancel as ex:
mongodb_client['deployment']['deploy_log'].insert({
'event_id':
payload.event_id,
'type':
'deploy_cancel',
'result':
'success',
'createdTimeStamp':
int(time.time())
})
except Exception as ex:
exception_str = str(ex)
stack_info = traceback.format_exc()
mongodb_client['deployment']['deploy_log'].insert({
'event_id':
payload.event_id,
'type':
'deploy',
'result':
'exception',
'exception':
exception_str,
'trace':
stack_info,
'createdTimeStamp':
int(time.time()),
'status_snapshot':
serialize_status(self.status.export_status())
})
mail_manager.send_error_mail(payload, '', datetime_start,
stack_info)
try:
self.rollback(payload)
datetime_end = datetime.datetime.now()
mongodb_client['deployment']['deploy_log'].insert({
'event_id':
payload.event_id,
'type':
'deploy_rollback',
'result':
'success',
'exception':
exception_str,
'trace':
stack_info,
'createdTimeStamp':
int(time.time()),
'status_snapshot':
serialize_status(self.status.export_status())
})
mail_manager.send_rollback_success_mail(
payload, '', datetime_start, datetime_end, stack_info)
except Exception as rollback_ex:
rollback_exception_str = str(rollback_ex)
rollback_stack_info = traceback.format_exc()
datetime_end = datetime.datetime.now()
mongodb_client['deployment']['deploy_log'].insert({
'event_id':
payload.event_id,
'type':
'deploy_rollback',
'result':
'fail',
'exception_original':
exception_str,
'trace_original':
stack_info,
'exception_rollback':
rollback_exception_str,
'trace_rollback':
rollback_stack_info,
'createdTimeStamp':
int(time.time()),
'status_snapshot':
serialize_status(self.status.export_status())
})
mail_manager.send_rollback_fail_mail(payload, '',
datetime_start,
datetime_end, stack_info,
rollback_stack_info)
def rollback(self, payload):
try:
event_id = payload.event_id
head_commit = payload.head_commit
repo = self.repo
logger_server.info("Start rollback Event[{event_id}]...".format(event_id=event_id))
# Change Status to Rollback and disable auto deployment
self.status.lock_acquire()
self.status.disable_auto_deploy()
self.status.set_status(DeployStatus.ROLLBACK)
self.status.lock_release()
# Step 1.Get change files
self.stage("Get change files", 20)
try:
change_files = repo.get_change_files(self.status.get_last_commit(), head_commit)
if _DEBUG:
logger_server.debug("change files:" + str(change_files))
except:
# May be pull failed so head_commit is not a valid object
change_files = []
# Step 2.Get Services to restart
self.stage("Get Services to restart", 40)
restart_services = repo.get_service_to_restart(change_files)
if _DEBUG:
logger_server.debug("service to restart:" + str(restart_services))
# Step 3.Git Reset
self.stage("Git Reset", 60)
repo.reset(self.status.get_last_commit())
# Step 4.Restart Services
self.stage("Restart Services", 80)
repo.restart_services(restart_services, self.status.get_hosts()[0])
# Logging
logger_server.info("Rollback Event[{event_id}] done.".format(event_id=event_id))
# Change Status to Idle
self.status.lock_acquire()
self.status.set_status(DeployStatus.IDLE)
self.status.set_running_task(None)
self.stage(None)
self.status.update_last_commit()
if _DEBUG:
logger_server.debug("Status" + str(self.status.export_status()))
self.status.lock_release()
except Exception as ex:
mongodb_client["deployment"]["deploy_log"].insert(
{
"event_id": payload.event_id,
"type": "rollback",
"result": "fail",
"exception": str(ex),
"createdTimeStamp": int(time.time()),
"status_snapshot": serialize_status(self.status.export_status()),
}
)
raise ex
def deploy(self, payload):
try:
event_id = payload.event_id
repo = self.repo
logger_server.info("Start deploy Event[{event_id}]...".format(event_id=event_id))
datetime_start = datetime.datetime.now()
# Change & init status to Running
self.status.set_status(DeployStatus.RUNNING)
self.status.set_running_task(payload)
self.status.init_host_status()
self.status.set_cancel_flag(False)
# Release lock to let other event occur
self.status.lock_release()
if _DEBUG:
logger_server.debug(self.status.export_status())
# Step 1.Clean Git Work Dir
self.stage("Clean Git Work Dir", 20)
repo.clean()
# Step 2.Pull data
self.stage("Pull data", 30)
pull_content = repo.pull()
# Step 3.Get change files
self.stage("Get change files", 40)
commit_after_pull = self.repo.get_last_commit()
commit_before_pull = self.status.get_last_commit()
if _DEBUG:
logger_server.debug(
"Start commit:{start},end commit:{end}".format(start=commit_before_pull, end=commit_after_pull)
)
change_files = self.repo.get_change_files(commit_before_pull, commit_after_pull)
if _DEBUG:
logger_server.debug("Change files:" + str(change_files))
if change_files:
# Step 4.NPM & Python Package Install
self.stage("NPM & Python Package Install", 60)
repo.install_pkg(repo.get_pkg_to_install(change_files))
# Step 5.Run Post-actions
self.stage("Run Post-actions", 70)
repo.handle_post_actions()
# Step 6.Get Services to restarts
self.stage("Get Services to restart", 80)
restart_services = repo.get_service_to_restart(change_files)
if _DEBUG:
logger_server.debug("service to restart:" + str(restart_services))
# Step 7.Restart Services
self.stage("Restart Services", 90)
repo.restart_services(restart_services, self.status.get_hosts()[0])
self.stage("Finish", 100)
else:
logger_server.info("Nothing has been changed")
# Logging
logger_server.info("Deploy Event[{event_id}] done.".format(event_id=event_id))
datetime_end = datetime.datetime.now()
mongodb_client["deployment"]["deploy_log"].insert(
{
"event_id": payload.event_id,
"type": "deploy",
"repo_name": self.repo.repo_name,
"result": "success",
"cost_time": str(datetime_end - datetime_start),
"createdTimeStamp": int(time.time()),
}
)
# Check if exist waiting task to run
self.status.lock_acquire()
self.status.update_last_commit()
if self.status.has_waiting_task():
self.deploy(self.status.get_first_waiting_task())
return
# Change Status to Idle, Trick: Here will be reached once during one working list
self.status.set_status(DeployStatus.IDLE)
self.stage(None)
self.status.set_running_task(None)
self.status.set_cancel_flag(False)
if _DEBUG:
logger_server.debug("Status" + str(self.status.export_status()))
self.status.lock_release()
except DeployCancel as ex:
mongodb_client["deployment"]["deploy_log"].insert(
{
"event_id": payload.event_id,
"type": "deploy_cancel",
"result": "success",
"createdTimeStamp": int(time.time()),
}
)
except Exception as ex:
exception_str = str(ex)
stack_info = traceback.format_exc()
mongodb_client["deployment"]["deploy_log"].insert(
{
"event_id": payload.event_id,
"type": "deploy",
"result": "exception",
"exception": exception_str,
"trace": stack_info,
"createdTimeStamp": int(time.time()),
"status_snapshot": serialize_status(self.status.export_status()),
}
)
mail_manager.send_error_mail(payload, "", datetime_start, stack_info)
try:
self.rollback(payload)
datetime_end = datetime.datetime.now()
mongodb_client["deployment"]["deploy_log"].insert(
{
"event_id": payload.event_id,
"type": "deploy_rollback",
"result": "success",
"exception": exception_str,
"trace": stack_info,
"createdTimeStamp": int(time.time()),
"status_snapshot": serialize_status(self.status.export_status()),
}
)
mail_manager.send_rollback_success_mail(payload, "", datetime_start, datetime_end, stack_info)
except Exception as rollback_ex:
rollback_exception_str = str(rollback_ex)
rollback_stack_info = traceback.format_exc()
datetime_end = datetime.datetime.now()
mongodb_client["deployment"]["deploy_log"].insert(
{
"event_id": payload.event_id,
"type": "deploy_rollback",
"result": "fail",
"exception_original": exception_str,
"trace_original": stack_info,
"exception_rollback": rollback_exception_str,
"trace_rollback": rollback_stack_info,
"createdTimeStamp": int(time.time()),
"status_snapshot": serialize_status(self.status.export_status()),
}
)
mail_manager.send_rollback_fail_mail(
payload, "", datetime_start, datetime_end, stack_info, rollback_stack_info
)
