def project_detection(self):
errors = []
# LOCAL_SERVER_USER => git
# LOCAL_SERVER_USER => target_servers
for server_info in self.servers:
waller = Waller(host=server_info['host'], user=server_info['user'], port=server_info['port'])
result = waller.run('id', exception=False, wenv=self.config())
if result.failed:
errors.append({
'title': '远程目标机器免密码登录失败',
'why': '远程目标机器：%s 错误：%s' % (server_info['host'], result.stdout),
'how': '在宿主机中配置免密码登录，把宿主机用户%s的~/.ssh/ssh_rsa.pub添加到远程目标机器用户%s的~/.ssh/authorized_keys。了解更多：http://walle-web.io/docs/troubleshooting.html' % (
pwd.getpwuid(os.getuid())[0], server_info['host']),
})
# maybe this is no webroot's parent dir
command = '[ -d {webroot} ] || mkdir -p {webroot}'.format(webroot=os.path.basename(self.project_info['target_root']))
result = waller.run(command, exception=False, wenv=self.config(console=False))
# 检查 webroot 父目录是否存在,是否为软链
command = '[ -L "%s" ] && echo "true" || echo "false"' % (self.project_info['target_root'])
result = waller.run(command, exception=False, wenv=self.config())
if result.stdout == 'false':
errors.append({
'title': '远程目标机器webroot不能是已建好的目录',
'why': '远程目标机器%s webroot不能是已存在的目录，必须为软链接，你不必新建，walle会自行创建。' % (server_info['host']),
'how': '手工删除远程目标机器：%s webroot目录：%s' % (server_info['host'], self.project_info['target_root']),
})
# remote release directory
return errors
def project_detection(self):
errors = []
# LOCAL_SERVER_USER => git
# LOCAL_SERVER_USER => target_servers
for server_info in self.servers:
waller = Waller(host=server_info['host'], user=server_info['user'], port=server_info['port'])
result = waller.run('id', exception=False, wenv=self.config())
if result.failed:
errors.append({
'title': '远程目标机器免密码登录失败',
'why': '远程目标机器：%s 错误：%s' % (server_info['host'], result.stdout),
'how': '在宿主机中配置免密码登录，把宿主机用户%s的~/.ssh/ssh_rsa.pub添加到远程目标机器用户%s的~/.ssh/authorized_keys。了解更多：http://walle-web.io/docs/troubleshooting.html' % (
pwd.getpwuid(os.getuid())[0], server_info['host']),
})
# maybe this is no webroot's parent dir
command = '[ -d {webroot} ] || mkdir -p {webroot}'.format(webroot=os.path.basename(self.project_info['target_root']))
result = waller.run(command, exception=False, wenv=self.config(console=False))
# 检查 webroot 父目录是否存在,是否为软链
command = '[ -L "%s" ] && echo "true" || echo "false"' % (self.project_info['target_root'])
result = waller.run(command, exception=False, wenv=self.config())
if result.stdout == 'false':
errors.append({
'title': '远程目标机器webroot不能是已建好的目录',
'why': '远程目标机器%s webroot不能是已存在的目录，必须为软链接，你不必新建，walle会自行创建。' % (server_info['host']),
'how': '手工删除远程目标机器：%s webroot目录：%s' % (server_info['host'], self.project_info['target_root']),
})
# remote release directory
return errors
def project_detection(self):
errors = []
# walle user => walle LOCAL_SERVER_USER
# show ssh_rsa.pub （maybe not necessary）
command = 'whoami'
current_app.logger.info(command)
result = self.local.run(command, exception=False, wenv=self.config())
if result.failed:
errors.append({
'title':
u'本地免密码登录失败',
'why':
result.stdout,
'how':
u'在宿主机中配置免密码登录，把walle启动用户%s的~/.ssh/ssh_rsa.pub添加到LOCAL_SERVER_USER用户%s的~/.ssh/authorized_keys。了解更多：http://walle-web.io/docs/troubleshooting.html'
% (pwd.getpwuid(os.getuid())[0],
current_app.config.get('LOCAL_SERVER_USER')),
})
# LOCAL_SERVER_USER => git
# LOCAL_SERVER_USER => target_servers
for server_info in self.servers:
waller = Waller(host=server_info['host'],
user=server_info['user'],
port=server_info['port'])
result = waller.run('id', exception=False, wenv=self.config())
if result.failed:
errors.append({
'title':
u'远程目标机器免密码登录失败',
'why':
u'远程目标机器：%s 错误：%s' % (server_info['host'], result.stdout),
'how':
u'在宿主机中配置免密码登录，把宿主机用户%s的~/.ssh/ssh_rsa.pub添加到远程目标机器用户%s的~/.ssh/authorized_keys。了解更多：http://walle-web.io/docs/troubleshooting.html'
% (current_app.config.get('LOCAL_SERVER_USER'),
server_info['host']),
})
# 检查 webroot 父目录是否存在,是否为软链
command = '[ -L "%s" ] && echo "true" || echo "false"' % (
self.project_info['target_root'])
result = waller.run(command, exception=False, wenv=self.config())
if result.stdout == 'false':
errors.append({
'title':
u'远程目标机器webroot不能是已建好的目录',
'why':
u'远程目标机器%s webroot不能是已存在的目录，必须为软链接，你不必新建，walle会自行创建。' %
(server_info['host']),
'how':
u'手工删除远程目标机器：%s webroot目录：%s' %
(server_info['host'], self.project_info['target_root']),
})
# remote release directory
return errors
class Deployer:
'''
序列号
'''
stage = '0'
sequence = 0
stage_prev_deploy = 'prev_deploy'
stage_deploy = 'deploy'
stage_post_deploy = 'post_deploy'
stage_prev_release = 'prev_release'
stage_release = 'release'
stage_post_release = 'post_release'
task_id = '0'
user_id = '0'
taskMdl = None
TaskRecord = None
console = False
version = datetime.now().strftime('%Y%m%d%H%M%s')
local_codebase, dir_codebase_project, project_name = None, None, None
dir_release, dir_webroot = None, None
connections, success, errors = {}, {}, {}
release_version_tar, release_version = None, None
local = None
def __init__(self, task_id=None, project_id=None, console=False):
self.local_codebase = current_app.config.get('CODE_BASE')
self.local = Waller(
host=current_app.config.get('LOCAL_SERVER_HOST'),
user=current_app.config.get('LOCAL_SERVER_USER'),
port=current_app.config.get('LOCAL_SERVER_PORT'),
)
self.TaskRecord = RecordModel()
if task_id:
self.task_id = task_id
# task start
current_app.logger.info(self.task_id)
self.taskMdl = TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.task = self.taskMdl.get('target_user')
self.project_info = self.taskMdl.get('project_info')
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
self.project_name = self.project_info['id']
self.dir_codebase_project = self.local_codebase + str(
self.project_name)
self.init_repo()
# start to deploy
self.console = console
def config(self):
return {
'task_id': self.task_id,
'user_id': self.user_id,
'stage': self.stage,
'sequence': self.sequence,
'console': self.console
}
def start(self):
TaskModel().get_by_id(self.task_id).update(
{'status': TaskModel.status_doing})
self.taskMdl = TaskModel().item(self.task_id)
# ===================== fabric ================
# SocketHandler
def prev_deploy(self):
'''
# TODO
socketio.sleep(0.001)
1.代码检出前要做的基础工作
- 检查 当前用户
- 检查 python 版本
- 检查 git 版本
- 检查 目录是否存在
- 用户自定义命令
:return:
'''
self.stage = self.stage_prev_deploy
self.sequence = 1
# 检查 当前用户
command = 'whoami'
current_app.logger.info(command)
result = self.local.run(command, wenv=self.config())
# 检查 python 版本
command = 'python --version'
result = self.local.run(command, wenv=self.config())
# 检查 git 版本
command = 'git --version'
result = self.local.run(command, wenv=self.config())
# 检查 目录是否存在
self.init_repo()
# TODO to be removed
command = 'mkdir -p %s' % (self.dir_codebase_project)
result = self.local.run(command, wenv=self.config())
# 用户自定义命令
command = self.project_info['prev_deploy']
current_app.logger.info(command)
with self.local.cd(self.dir_codebase_project):
result = self.local.run(command, wenv=self.config())
def deploy(self):
'''
2.检出代码
:param project_name:
:return:
'''
self.stage = self.stage_deploy
self.sequence = 2
current_app.logger.info('git dir: %s',
self.dir_codebase_project + '/.git')
# 如果项目底下有 .git 目录则认为项目完整,可以直接检出代码
# TODO 不标准
if os.path.exists(self.dir_codebase_project + '/.git'):
with self.local.cd(self.dir_codebase_project):
command = 'pwd && git pull'
result = self.local.run(command, wenv=self.config())
else:
# 否则当作新项目检出完整代码
with self.local.cd(self.dir_codebase_project):
command = 'pwd && git clone %s .' % (
self.project_info['repo_url'])
current_app.logger.info('cd %s command: %s ',
self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
# copy to a local version
self.release_version = '%s_%s_%s' % (
self.project_name, self.task_id,
time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())))
with self.local.cd(self.local_codebase):
command = 'cp -rf %s %s' % (self.dir_codebase_project,
self.release_version)
current_app.logger.info('cd %s command: %s ',
self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
# 更新到指定 commit_id
with self.local.cd(self.local_codebase + self.release_version):
command = 'git reset -q --hard %s' % (
self.taskMdl.get('commit_id'))
result = self.local.run(command, wenv=self.config())
if result.exited != Code.Ok:
raise WalleError(Code.shell_git_fail, message=result.stdout)
def post_deploy(self):
'''
3.检出代码后要做的任务
- 用户自定义操作命令
- 代码编译
- 清除日志文件及无用文件
-
- 压缩打包
- 传送到版本库 release
:return:
'''
self.stage = self.stage_post_deploy
self.sequence = 3
# 用户自定义命令
command = self.project_info['post_deploy']
with self.local.cd(self.local_codebase + self.release_version):
result = self.local.run(command, wenv=self.config())
# 压缩打包
self.release_version_tar = '%s.tgz' % (self.release_version)
with self.local.cd(self.local_codebase):
command = 'tar zcf %s %s' % (self.release_version_tar,
self.release_version)
result = self.local.run(command, wenv=self.config())
def prev_release(self, waller):
'''
4.部署代码到目标机器前做的任务
- 检查 webroot 父目录是否存在
:return:
'''
self.stage = self.stage_prev_release
self.sequence = 4
# 检查 target_releases 父目录是否存在
command = 'mkdir -p %s' % (self.project_info['target_releases'])
result = waller.run(command, wenv=self.config())
# TODO 检查 webroot 父目录是否存在,是否为软链
# command = 'mkdir -p %s' % (self.project_info['target_root'])
# result = waller.run(command)
# current_app.logger.info('command: %s', dir(result))
# 用户自定义命令
command = self.project_info['prev_release']
current_app.logger.info(command)
with waller.cd(self.project_info['target_releases']):
result = waller.run(command, wenv=self.config())
# TODO md5
# 传送到版本库 release
current_app.logger.info('/tmp/walle/codebase/' +
self.release_version_tar)
result = waller.put('/tmp/walle/codebase/' + self.release_version_tar,
remote=self.project_info['target_releases'],
wenv=self.config())
current_app.logger.info('command: %s', dir(result))
# 解压
self.release_untar(waller)
def release(self, waller):
'''
5.部署代码到目标机器做的任务
- 打包代码 local
- scp local => remote
- 解压 remote
:return:
'''
self.stage = self.stage_release
self.sequence = 5
with waller.cd(self.project_info['target_releases']):
# 1. create a tmp link dir
current_link_tmp_dir = '%s/current-tmp-%s' % (
self.project_info['target_releases'], self.task_id)
command = 'ln -sfn %s/%s %s' % (
self.project_info['target_releases'], self.release_version,
current_link_tmp_dir)
result = waller.run(command, wenv=self.config())
# 2. make a soft link from release to tmp link
# 3. move tmp link to webroot
current_link_tmp_dir = '%s/current-tmp-%s' % (
self.project_info['target_releases'], self.task_id)
command = 'mv -fT %s %s' % (current_link_tmp_dir,
self.project_info['target_root'])
result = waller.run(command, wenv=self.config())
def release_untar(self, waller):
'''
解压版本包
:return:
'''
with waller.cd(self.project_info['target_releases']):
command = 'tar zxf %s' % (self.release_version_tar)
result = waller.run(command, wenv=self.config())
def post_release(self, waller):
'''
6.部署代码到目标机器后要做的任务
- 切换软链
- 重启 nginx
:return:
'''
self.stage = self.stage_post_release
self.sequence = 6
# 用户自定义命令
command = self.project_info['post_release']
current_app.logger.info(command)
with waller.cd(self.project_info['target_root']):
result = waller.run(command, wenv=self.config())
self.post_release_service(waller)
def post_release_service(self, waller):
'''
代码部署完成后,服务启动工作,如: nginx重启
:param connection:
:return:
'''
with waller.cd(self.project_info['target_root']):
command = 'sudo service nginx restart'
result = waller.run(command, wenv=self.config())
def list_tag(self):
with self.local.cd(self.dir_codebase_project):
command = 'git tag -l'
result = self.local.run(command, pty=False, wenv=self.config())
tags = result.stdout.strip()
tags = tags.split('\n')
return [color_clean(tag.strip()) for tag in tags]
return None
def list_branch(self):
with self.local.cd(self.dir_codebase_project):
command = 'git pull'
result = self.local.run(command, wenv=self.config())
if result.exited != Code.Ok:
raise WalleError(Code.shell_git_pull_fail,
message=result.stdout)
current_app.logger.info(self.dir_codebase_project)
command = 'git branch -r'
result = self.local.run(command, pty=False, wenv=self.config())
# if result.exited != Code.Ok:
# raise WalleError(Code.shell_run_fail)
# TODO 三种可能: false, error, success
branches = result.stdout.strip()
branches = branches.split('\n')
# 去除 origin/HEAD -> 当前指向
# 去除远端前缀
branches = [
branch.strip().lstrip('origin/') for branch in branches
if not branch.strip().startswith('origin/HEAD')
]
return branches
return None
def list_commit(self, branch):
with self.local.cd(self.dir_codebase_project):
command = 'git checkout %s && git pull' % (branch)
self.local.run(command, wenv=self.config())
command = 'git log -50 --pretty="%h #@_@# %an #@_@# %s"'
result = self.local.run(command, pty=False, wenv=self.config())
current_app.logger.info(result.stdout)
commit_log = result.stdout.strip()
current_app.logger.info(commit_log)
commit_list = commit_log.split('\n')
commits = []
for commit in commit_list:
if not re.search('^.+ #@_@# .+ #@_@# .*$', commit):
continue
commit_dict = commit.split(' #@_@# ')
current_app.logger.info(commit_dict)
commits.append({
'id': commit_dict[0],
'name': commit_dict[1],
'message': commit_dict[2],
})
return commits
# TODO
return None
def init_repo(self):
if not os.path.exists(self.dir_codebase_project):
# 检查 目录是否存在
command = 'mkdir -p %s' % (self.dir_codebase_project)
# TODO remove
current_app.logger.info(command)
self.local.run(command, wenv=self.config())
with self.local.cd(self.dir_codebase_project):
is_git_dir = self.local.run('git status',
exception=False,
wenv=self.config())
if is_git_dir.exited != Code.Ok:
# 否则当作新项目检出完整代码
# 检查 目录是否存在
command = 'rm -rf %s' % (self.dir_codebase_project)
self.local.run(command, wenv=self.config())
command = 'git clone %s %s' % (self.project_info['repo_url'],
self.dir_codebase_project)
current_app.logger.info('cd %s command: %s ',
self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
if result.exited != Code.Ok:
raise WalleError(Code.shell_git_init_fail,
message=result.stdout)
def end(self, success=True):
status = TaskModel.status_success if success else TaskModel.status_fail
current_app.logger.info('success:%s, status:%s' % (success, status))
TaskModel().get_by_id(self.task_id).update({'status': status})
def walle_deploy(self):
self.start()
try:
self.prev_deploy()
self.deploy()
self.post_deploy()
is_all_servers_success = True
for server_info in self.servers:
server = server_info['host']
try:
self.connections[server] = Waller(
host=server, user=self.project_info['target_user'])
self.prev_release(self.connections[server])
self.release(self.connections[server])
self.post_release(self.connections[server])
except Exception as e:
is_all_servers_success = False
current_app.logger.error(e)
self.errors[server] = e.message
self.end(is_all_servers_success)
except Exception as e:
self.end(False)
emit('fail', {
'event': 'console',
'data': {
'message': Code.code_msg[Code.deploy_fail]
}
},
room=self.task_id)
return {'success': self.success, 'errors': self.errors}
class DeploySocketIO:
'''
序列号
'''
stage = '0'
sequence = 0
stage_prev_deploy = 'prev_deploy'
stage_deploy = 'deploy'
stage_post_deploy = 'post_deploy'
stage_prev_release = 'prev_release'
stage_release = 'release'
stage_post_release = 'post_release'
task_id = '0'
user_id = '0'
taskMdl = None
TaskRecord = None
version = datetime.now().strftime('%Y%m%d%H%M%s')
project_name = 'walden'
dir_codebase = '/tmp/walle/codebase/'
dir_codebase_project = dir_codebase + project_name
# 定义远程机器
# env.hosts = ['172.16.0.231', '172.16.0.177']
dir_release = None
dir_webroot = None
connections, success, errors = {}, {}, {}
release_version_tar, release_version = None, None
local, websocket = None, None
def __init__(self, task_id=None, project_id=None, websocket=None):
self.local = Waller(host=current_app.config.get('LOCAL_SERVER_HOST'),
user=current_app.config.get('LOCAL_SERVER_USER'),
port=current_app.config.get('LOCAL_SERVER_PORT'))
self.TaskRecord = TaskModel.TaskRecordModel()
current_app.logger.info('DeploySocketIO.__init__ before')
emit('message', {'msg': 'DeploySocketIO.__init__'})
current_app.logger.info('DeploySocketIO.__init__ after')
# if websocket:
# websocket.send_updates(__name__)
# self.websocket = websocket
if task_id:
self.task_id = task_id
self.taskMdl = TaskModel.TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.task = self.taskMdl.get('target_user')
self.project_info = self.taskMdl.get('project_info')
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
def config(self):
return {
'task_id': self.task_id,
'user_id': self.user_id,
'stage': self.stage,
'sequence': self.sequence,
'websocket': self.websocket
}
# ===================== fabric ================
# SocketHandler
def deploy(self):
'''
1.代码检出前要做的基础工作
- 检查 当前用户
- 检查 python 版本
- 检查 git 版本
- 检查 目录是否存在
- 用户自定义命令
:return:
'''
current_app.logger.info('deploy ing')
emit('message', {'msg': 'deploy ing'})
current_app.logger.info('deploy end')
self.stage = self.stage_prev_deploy
self.sequence = 1
# TODO remove
# result = self.local.run('sleep 30', wenv=self.config())
# 检查 当前用户
command = 'whoami'
current_app.logger.info(command)
emit('message', {'msg': command})
result = self.local.run(command, wenv=self.config())
# 检查 python 版本
command = 'python --version'
result = self.local.run(command, wenv=self.config())
current_app.logger.info(command)
# 检查 git 版本
command = 'git --version'
result = self.local.run(command, wenv=self.config())
current_app.logger.info(command)
# 检查 目录是否存在
command = 'mkdir -p %s' % (self.dir_codebase_project)
# TODO remove
current_app.logger.info(command)
result = self.local.run(command, wenv=self.config())
# 用户自定义命令
command = self.project_info['prev_deploy']
current_app.logger.info(command)
with self.local.cd(self.dir_codebase_project):
result = self.local.run(command, wenv=self.config())
class Deployer:
'''
序列号
'''
stage = '0'
sequence = 0
stage_prev_deploy = 'prev_deploy'
stage_deploy = 'deploy'
stage_post_deploy = 'post_deploy'
stage_prev_release = 'prev_release'
stage_release = 'release'
stage_post_release = 'post_release'
task_id = '0'
user_id = '0'
taskMdl = None
TaskRecord = None
version = datetime.now().strftime('%Y%m%d%H%M%s')
project_name = 'walden'
dir_codebase = '/tmp/walle/codebase/'
dir_codebase_project = dir_codebase + project_name
# 定义远程机器
# env.hosts = ['172.16.0.231', '172.16.0.177']
dir_release = None
dir_webroot = None
connections, success, errors = {}, {}, {}
release_version_tar, release_version = None, None
local, websocket = None, None
def __init__(self, task_id=None, project_id=None, websocket=None):
self.local = Waller(host=current_app.config.get('LOCAL_SERVER_HOST'),
user=current_app.config.get('LOCAL_SERVER_USER'),
port=current_app.config.get('LOCAL_SERVER_PORT'))
self.TaskRecord = TaskModel.TaskRecordModel()
if websocket:
websocket.send_updates(__name__)
self.websocket = websocket
if task_id:
self.task_id = task_id
self.taskMdl = TaskModel.TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.task = self.taskMdl.get('target_user')
self.project_info = self.taskMdl.get('project_info')
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
def config(self):
return {
'task_id': self.task_id,
'user_id': self.user_id,
'stage': self.stage,
'sequence': self.sequence,
'websocket': self.websocket
}
# ===================== fabric ================
# SocketHandler
def prev_deploy(self):
'''
1.代码检出前要做的基础工作
- 检查 当前用户
- 检查 python 版本
- 检查 git 版本
- 检查 目录是否存在
- 用户自定义命令
:return:
'''
self.stage = self.stage_prev_deploy
self.sequence = 1
# TODO remove
# result = self.local.run('sleep 30', wenv=self.config())
# 检查 当前用户
command = 'whoami'
self.websocket.send_updates(command)
current_app.logger.info(command)
result = self.local.run(command, wenv=self.config())
# 检查 python 版本
command = 'python --version'
result = self.local.run(command, wenv=self.config())
current_app.logger.info(command)
# 检查 git 版本
command = 'git --version'
result = self.local.run(command, wenv=self.config())
current_app.logger.info(command)
# 检查 目录是否存在
command = 'mkdir -p %s' % (self.dir_codebase_project)
# TODO remove
current_app.logger.info(command)
result = self.local.run(command, wenv=self.config())
# 用户自定义命令
command = self.project_info['prev_deploy']
current_app.logger.info(command)
with self.local.cd(self.dir_codebase_project):
result = self.local.run(command, wenv=self.config())
# SocketHandler.send_to_all({
# 'type': 'user',
# 'id': 33,
# 'host': env.host_string,
# 'command': command,
# 'message': result.stdout,
# })
def deploy(self):
'''
2.检出代码
:param project_name:
:return:
'''
self.stage = self.stage_deploy
self.sequence = 2
current_app.logger.info('git dir: %s',
self.dir_codebase_project + '/.git')
# 如果项目底下有 .git 目录则认为项目完整,可以直接检出代码
# TODO 不标准
if os.path.exists(self.dir_codebase_project + '/.git'):
with self.local.cd(self.dir_codebase_project):
command = 'pwd && git pull'
result = self.local.run(command, wenv=self.config())
else:
# 否则当作新项目检出完整代码
with self.local.cd(self.dir_codebase_project):
command = 'pwd && git clone %s .' % (
self.project_info['repo_url'])
current_app.logger.info('cd %s command: %s ',
self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
# copy to a local version
self.release_version = '%s_%s_%s' % (
self.project_name, self.task_id,
time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())))
with self.local.cd(self.dir_codebase):
command = 'cp -rf %s %s' % (self.dir_codebase_project,
self.release_version)
current_app.logger.info('cd %s command: %s ',
self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
# 更新到指定 commit_id
with self.local.cd(self.dir_codebase + self.release_version):
command = 'git reset -q --hard %s' % (
self.taskMdl.get('commit_id'))
result = self.local.run(command, wenv=self.config())
# SocketHandler.send_to_all({
# 'type': 'user',
# 'id': 33,
# 'host': env.host_string,
# 'command': command,
# 'message': result.stdout,
# })
# 用户自定义命令
# command = self.project_info['deploy']
# current_app.logger.info(command)
# with self.local.cd(self.dir_codebase):
# result = self.local.run(command)
pass
def post_deploy(self):
'''
3.检出代码后要做的任务
- 用户自定义操作命令
- 代码编译
- 清除日志文件及无用文件
-
- 压缩打包
- 传送到版本库 release
:return:
'''
self.stage = self.stage_post_deploy
self.sequence = 3
# 用户自定义命令
command = self.project_info['post_deploy']
current_app.logger.info(command)
with self.local.cd(self.dir_codebase + self.release_version):
result = self.local.run(command, wenv=self.config())
# 压缩打包
self.release_version_tar = '%s.tgz' % (self.release_version)
with self.local.cd(self.dir_codebase):
command = 'tar zcvf %s %s' % (self.release_version_tar,
self.release_version)
result = self.local.run(command, wenv=self.config())
def prev_release(self, waller):
'''
4.部署代码到目标机器前做的任务
- 检查 webroot 父目录是否存在
:return:
'''
self.stage = self.stage_prev_release
self.sequence = 4
# 检查 target_releases 父目录是否存在
command = 'mkdir -p %s' % (self.project_info['target_releases'])
result = waller.run(command, wenv=self.config())
# TODO 检查 webroot 父目录是否存在,是否为软链
# command = 'mkdir -p %s' % (self.project_info['target_root'])
# result = waller.run(command)
# current_app.logger.info('command: %s', dir(result))
# TODO md5
# 传送到版本库 release
current_app.logger.info('/tmp/walle/codebase/' +
self.release_version_tar)
result = waller.put('/tmp/walle/codebase/' + self.release_version_tar,
remote=self.project_info['target_releases'])
current_app.logger.info('command: %s', dir(result))
# 解压
self.release_untar(waller)
def release(self, waller):
'''
5.部署代码到目标机器做的任务
- 打包代码 local
- scp local => remote
- 解压 remote
:return:
'''
self.stage = self.stage_release
self.sequence = 5
with waller.cd(self.project_info['target_releases']):
# 1. create a tmp link dir
current_link_tmp_dir = '%s/current-tmp-%s' % (
self.project_info['target_releases'], self.task_id)
command = 'ln -sfn %s/%s %s' % (
self.project_info['target_releases'], self.release_version,
current_link_tmp_dir)
result = waller.run(command, wenv=self.config())
# 2. make a soft link from release to tmp link
# 3. move tmp link to webroot
current_link_tmp_dir = '%s/current-tmp-%s' % (
self.project_info['target_releases'], self.task_id)
command = 'mv -fT %s %s' % (current_link_tmp_dir,
self.project_info['target_root'])
result = waller.run(command, wenv=self.config())
def release_untar(self, waller):
'''
解压版本包
:return:
'''
with waller.cd(self.project_info['target_releases']):
command = 'tar zxf %s' % (self.release_version_tar)
result = waller.run(command, wenv=self.config())
def post_release(self, waller):
'''
6.部署代码到目标机器后要做的任务
- 切换软链
- 重启 nginx
:return:
'''
self.stage = self.stage_post_release
self.sequence = 6
self.post_release_service(waller)
def post_release_service(self, waller):
'''
代码部署完成后,服务启动工作,如: nginx重启
:param connection:
:return:
'''
with waller.cd(self.project_info['target_root']):
command = 'sudo service nginx restart'
result = waller.run(command, wenv=self.config())
def list_tag(self):
with self.local.cd(self.dir_codebase_project):
command = 'git tag -l'
current_app.logger.info('cd %s command: %s ',
self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
current_app.logger.info(dir(result))
return result
return None
def list_branch(self):
with self.local.cd(self.dir_codebase_project):
command = 'git pull'
# result = self.local.run(command, wenv=self.config())
current_app.logger.info(self.dir_codebase_project)
command = 'git branch -r'
result = self.local.run(command, wenv=self.config())
# TODO 三种可能: false, error, success
branches = result.stdout.strip().split('\n')
# 去除 origin/HEAD -> 当前指向
# 去除远端前缀
branches = [
branch.strip().lstrip('origin/') for branch in branches
if not branch.startswith('origin/HEAD')
]
return branches
return None
def list_commit(self, branch):
with self.local.cd(self.dir_codebase_project):
command = 'git checkout %s && git pull' % (branch)
result = self.local.run(command, wenv=self.config())
# TODO 10是需要前端传的
command = 'git log -10 --pretty="%h #_# %an #_# %s"'
result = self.local.run(command, wenv=self.config())
commit_list = result.stdout.strip().split('\n')
commits = []
for commit in commit_list:
commit_dict = commit.split(' #_# ')
commits.append({
'id': commit_dict[0],
'name': commit_dict[1],
'message': commit_dict[2],
})
return commits
return None
def walle_deploy(self):
self.prev_deploy()
self.deploy()
self.post_deploy()
server = '172.16.0.231'
try:
self.connections[server] = Waller(
host=server, user=self.project_info['target_user'])
self.prev_release(self.connections[server])
self.release(self.connections[server])
self.post_release(self.connections[server])
except Exception, e:
current_app.logger.exception(e)
self.errors[server] = e.message
# for server_info in self.servers:
# server = server_info.host
# try:
# self.connections[server] = Waller(host=server, user=self.project_info['target_user'])
# self.prev_release(self.connections[server])
# self.release(self.connections[server])
# self.post_release(self.connections[server])
# except Exception, e:
# self.errors[server] = e.message
return {'success': self.success, 'errors': self.errors}
class Deployer:
'''
序列号
'''
stage = '0'
sequence = 0
stage_prev_deploy = 'prev_deploy'
stage_deploy = 'deploy'
stage_post_deploy = 'post_deploy'
stage_prev_release = 'prev_release'
stage_release = 'release'
stage_post_release = 'post_release'
task_id = '0'
user_id = '0'
taskMdl = None
TaskRecord = None
console = False
version = datetime.now().strftime('%Y%m%d%H%M%s')
project_name = None
dir_codebase = '/tmp/walle/codebase/'
dir_codebase_project = ''
# 定义远程机器
# env.hosts = ['172.16.0.231', '172.16.0.177']
dir_release = None
dir_webroot = None
connections, success, errors = {}, {}, {}
release_version_tar, release_version = None, None
local = None
def __init__(self, task_id=None, project_id=None, console=False):
self.local = Waller(host=current_app.config.get('LOCAL_SERVER_HOST'),
user=current_app.config.get('LOCAL_SERVER_USER'),
port=current_app.config.get('LOCAL_SERVER_PORT'))
self.TaskRecord = RecordModel()
if task_id:
self.task_id = task_id
# task start
TaskModel(id=self.task_id).update(status=TaskModel.status_doing)
self.taskMdl = TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.task = self.taskMdl.get('target_user')
self.project_info = self.taskMdl.get('project_info')
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
self.project_name = self.project_info['id']
self.dir_codebase_project = self.dir_codebase + str(self.project_name)
# start to deploy
def config(self):
return {'task_id': self.task_id, 'user_id': self.user_id, 'stage': self.stage, 'sequence': self.sequence, 'console': self.console}
# ===================== fabric ================
# SocketHandler
def prev_deploy(self):
'''
# TODO
socketio.sleep(0.001)
1.代码检出前要做的基础工作
- 检查 当前用户
- 检查 python 版本
- 检查 git 版本
- 检查 目录是否存在
- 用户自定义命令
:return:
'''
self.stage = self.stage_prev_deploy
self.sequence = 1
# TODO remove
# result = self.local.run('sleep 30', wenv=self.config())
# 检查 当前用户
command = 'whoami'
current_app.logger.info(command)
result = self.local.run(command, wenv=self.config())
# 检查 python 版本
command = 'python --version'
result = self.local.run(command, wenv=self.config())
current_app.logger.info(command)
# 检查 git 版本
command = 'git --version'
result = self.local.run(command, wenv=self.config())
current_app.logger.info(command)
# 检查 目录是否存在
self.init_repo()
# TODO to be removed
command = 'mkdir -p %s' % (self.dir_codebase_project)
# TODO remove
current_app.logger.info(command)
result = self.local.run(command, wenv=self.config())
# 用户自定义命令
command = self.project_info['prev_deploy']
current_app.logger.info(command)
with self.local.cd(self.dir_codebase_project):
result = self.local.run(command, wenv=self.config())
# SocketHandler.send_to_all({
# 'type': 'user',
# 'id': 33,
# 'host': env.host_string,
# 'command': command,
# 'message': result.stdout,
# })
def deploy(self):
'''
2.检出代码
:param project_name:
:return:
'''
# TODO
socketio.sleep(0.001)
self.stage = self.stage_deploy
self.sequence = 2
current_app.logger.info('git dir: %s', self.dir_codebase_project + '/.git')
# 如果项目底下有 .git 目录则认为项目完整,可以直接检出代码
# TODO 不标准
if os.path.exists(self.dir_codebase_project + '/.git'):
with self.local.cd(self.dir_codebase_project):
command = 'pwd && git pull'
result = self.local.run(command, wenv=self.config())
else:
# 否则当作新项目检出完整代码
with self.local.cd(self.dir_codebase_project):
command = 'pwd && git clone %s .' % (self.project_info['repo_url'])
current_app.logger.info('cd %s command: %s ', self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
# copy to a local version
self.release_version = '%s_%s_%s' % (
self.project_name, self.task_id, time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())))
with self.local.cd(self.dir_codebase):
command = 'cp -rf %s %s' % (self.dir_codebase_project, self.release_version)
current_app.logger.info('cd %s command: %s ', self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
# 更新到指定 commit_id
with self.local.cd(self.dir_codebase + self.release_version):
command = 'git reset -q --hard %s' % (self.taskMdl.get('commit_id'))
result = self.local.run(command, wenv=self.config())
# SocketHandler.send_to_all({
# 'type': 'user',
# 'id': 33,
# 'host': env.host_string,
# 'command': command,
# 'message': result.stdout,
# })
# 用户自定义命令
# command = self.project_info['deploy']
# current_app.logger.info(command)
# with self.local.cd(self.dir_codebase):
# result = self.local.run(command)
pass
def post_deploy(self):
'''
3.检出代码后要做的任务
- 用户自定义操作命令
- 代码编译
- 清除日志文件及无用文件
-
- 压缩打包
- 传送到版本库 release
:return:
'''
# TODO
socketio.sleep(0.001)
self.stage = self.stage_post_deploy
self.sequence = 3
# 用户自定义命令
command = self.project_info['post_deploy']
current_app.logger.info(command)
current_app.logger.info(self.dir_codebase)
current_app.logger.info(self.release_version)
with self.local.cd(self.dir_codebase + self.release_version):
result = self.local.run(command, wenv=self.config())
# 压缩打包
self.release_version_tar = '%s.tgz' % (self.release_version)
with self.local.cd(self.dir_codebase):
command = 'tar zcvf %s %s' % (self.release_version_tar, self.release_version)
result = self.local.run(command, wenv=self.config())
def prev_release(self, waller):
'''
4.部署代码到目标机器前做的任务
- 检查 webroot 父目录是否存在
:return:
'''
# TODO
socketio.sleep(0.001)
self.stage = self.stage_prev_release
self.sequence = 4
# 检查 target_releases 父目录是否存在
command = 'mkdir -p %s' % (self.project_info['target_releases'])
result = waller.run(command, wenv=self.config())
# TODO 检查 webroot 父目录是否存在,是否为软链
# command = 'mkdir -p %s' % (self.project_info['target_root'])
# result = waller.run(command)
# current_app.logger.info('command: %s', dir(result))
# TODO md5
# 传送到版本库 release
current_app.logger.info('/tmp/walle/codebase/' + self.release_version_tar)
result = waller.put('/tmp/walle/codebase/' + self.release_version_tar,
remote=self.project_info['target_releases'], wenv=self.config())
current_app.logger.info('command: %s', dir(result))
# 解压
self.release_untar(waller)
def release(self, waller):
'''
5.部署代码到目标机器做的任务
- 打包代码 local
- scp local => remote
- 解压 remote
:return:
'''
# TODO
socketio.sleep(0.001)
self.stage = self.stage_release
self.sequence = 5
with waller.cd(self.project_info['target_releases']):
# 1. create a tmp link dir
current_link_tmp_dir = '%s/current-tmp-%s' % (self.project_info['target_releases'], self.task_id)
command = 'ln -sfn %s/%s %s' % (
self.project_info['target_releases'], self.release_version, current_link_tmp_dir)
result = waller.run(command, wenv=self.config())
# 2. make a soft link from release to tmp link
# 3. move tmp link to webroot
current_link_tmp_dir = '%s/current-tmp-%s' % (self.project_info['target_releases'], self.task_id)
command = 'mv -fT %s %s' % (current_link_tmp_dir, self.project_info['target_root'])
result = waller.run(command, wenv=self.config())
def release_untar(self, waller):
'''
解压版本包
:return:
'''
# TODO
socketio.sleep(0.001)
with waller.cd(self.project_info['target_releases']):
command = 'tar zxf %s' % (self.release_version_tar)
result = waller.run(command, wenv=self.config())
def post_release(self, waller):
'''
6.部署代码到目标机器后要做的任务
- 切换软链
- 重启 nginx
:return:
'''
self.stage = self.stage_post_release
self.sequence = 6
self.post_release_service(waller)
def post_release_service(self, waller):
'''
代码部署完成后,服务启动工作,如: nginx重启
:param connection:
:return:
'''
with waller.cd(self.project_info['target_root']):
command = 'sudo service nginx restart'
result = waller.run(command, wenv=self.config())
def list_tag(self):
self.init_repo()
with self.local.cd(self.dir_codebase_project):
command = 'git tag -l'
current_app.logger.info('cd %s command: %s ', self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
current_app.logger.info(dir(result))
return result
return None
def list_branch(self):
self.init_repo()
with self.local.cd(self.dir_codebase_project):
command = 'git pull'
result = self.local.run(command, wenv=self.config())
current_app.logger.info(self.dir_codebase_project)
command = 'git branch -r'
result = self.local.run(command, wenv=self.config())
# TODO 三种可能: false, error, success
branches = result.stdout.strip().split('\n')
# 去除 origin/HEAD -> 当前指向
# 去除远端前缀
branches = [branch.strip().lstrip('origin/') for branch in branches if not branch.startswith('origin/HEAD')]
return branches
return None
def list_commit(self, branch):
self.init_repo()
with self.local.cd(self.dir_codebase_project):
command = 'git checkout %s && git pull' % (branch)
result = self.local.run(command, wenv=self.config())
# TODO 10是需要前端传的
command = 'git log -10 --pretty="%h #_# %an #_# %s"'
result = self.local.run(command, wenv=self.config())
commit_list = result.stdout.strip().split('\n')
commits = []
for commit in commit_list:
commit_dict = commit.split(' #_# ')
commits.append({
'id': commit_dict[0],
'name': commit_dict[1],
'message': commit_dict[2],
})
return commits
return None
def init_repo(self):
current_app.logger.info('git dir: %s', self.dir_codebase_project + '/.git')
# 如果项目底下有 .git 目录则认为项目完整,可以直接检出代码
# TODO 不标准
if not os.path.exists(self.dir_codebase_project):
# 检查 目录是否存在
command = 'mkdir -p %s' % (self.dir_codebase_project)
# TODO remove
current_app.logger.info(command)
result = self.local.run(command, wenv=self.config())
if not os.path.exists(self.dir_codebase_project + '/.git'):
# 否则当作新项目检出完整代码
with self.local.cd(self.dir_codebase_project):
command = 'pwd && git clone %s .' % (self.project_info['repo_url'])
current_app.logger.info('cd %s command: %s ', self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
def walle_deploy(self):
self.prev_deploy()
self.deploy()
self.post_deploy()
server = '172.16.0.231'
try:
self.connections[server] = Waller(host=server, user=self.project_info['target_user'])
self.prev_release(self.connections[server])
self.release(self.connections[server])
self.post_release(self.connections[server])
except Exception as e:
current_app.logger.exception(e)
self.errors[server] = e.message
# for server_info in self.servers:
# server = server_info.host
# try:
# self.connections[server] = Waller(host=server, user=self.project_info['target_user'])
# self.prev_release(self.connections[server])
# self.release(self.connections[server])
# self.post_release(self.connections[server])
# except Exception, e:
# self.errors[server] = e.message
return {'success': self.success, 'errors': self.errors}