def __init__(self, task_id=None, project_id=None, console=False):
self.local_codebase = current_app.config.get('CODE_BASE')
self.localhost = Waller(host='127.0.0.1')
self.TaskRecord = RecordModel()
if task_id:
self.task_id = task_id
# task start
current_app.logger.info(self.task_id)
self.taskMdl = TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.project_info = self.taskMdl.get('project_info')
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
self.servers = self.project_info['servers_info']
self.project_name = self.project_info['id']
self.dir_codebase_project = self.local_codebase + str(self.project_name)
# self.init_repo()
# start to deploy
self.console = console
def project_detection(self):
errors = []
# LOCAL_SERVER_USER => git
# LOCAL_SERVER_USER => target_servers
for server_info in self.servers:
waller = Waller(host=server_info['host'], user=server_info['user'], port=server_info['port'])
result = waller.run('id', exception=False, wenv=self.config())
if result.failed:
errors.append({
'title': '远程目标机器免密码登录失败',
'why': '远程目标机器：%s 错误：%s' % (server_info['host'], result.stdout),
'how': '在宿主机中配置免密码登录，把宿主机用户%s的~/.ssh/ssh_rsa.pub添加到远程目标机器用户%s的~/.ssh/authorized_keys。了解更多：http://walle-web.io/docs/troubleshooting.html' % (
pwd.getpwuid(os.getuid())[0], server_info['host']),
})
# maybe this is no webroot's parent dir
command = '[ -d {webroot} ] || mkdir -p {webroot}'.format(webroot=os.path.basename(self.project_info['target_root']))
result = waller.run(command, exception=False, wenv=self.config(console=False))
# 检查 webroot 父目录是否存在,是否为软链
command = '[ -L "%s" ] && echo "true" || echo "false"' % (self.project_info['target_root'])
result = waller.run(command, exception=False, wenv=self.config())
if result.stdout == 'false':
errors.append({
'title': '远程目标机器webroot不能是已建好的目录',
'why': '远程目标机器%s webroot不能是已存在的目录，必须为软链接，你不必新建，walle会自行创建。' % (server_info['host']),
'how': '手工删除远程目标机器：%s webroot目录：%s' % (server_info['host'], self.project_info['target_root']),
})
# remote release directory
return errors
def project_detection(self):
errors = []
# LOCAL_SERVER_USER => git
# LOCAL_SERVER_USER => target_servers
for server_info in self.servers:
waller = Waller(host=server_info['host'], user=server_info['user'], port=server_info['port'])
result = waller.run('id', exception=False, wenv=self.config())
if result.failed:
errors.append({
'title': '远程目标机器免密码登录失败',
'why': '远程目标机器：%s 错误：%s' % (server_info['host'], result.stdout),
'how': '在宿主机中配置免密码登录，把宿主机用户%s的~/.ssh/ssh_rsa.pub添加到远程目标机器用户%s的~/.ssh/authorized_keys。了解更多：http://walle-web.io/docs/troubleshooting.html' % (
pwd.getpwuid(os.getuid())[0], server_info['host']),
})
# maybe this is no webroot's parent dir
command = '[ -d {webroot} ] || mkdir -p {webroot}'.format(webroot=os.path.basename(self.project_info['target_root']))
result = waller.run(command, exception=False, wenv=self.config(console=False))
# 检查 webroot 父目录是否存在,是否为软链
command = '[ -L "%s" ] && echo "true" || echo "false"' % (self.project_info['target_root'])
result = waller.run(command, exception=False, wenv=self.config())
if result.stdout == 'false':
errors.append({
'title': '远程目标机器webroot不能是已建好的目录',
'why': '远程目标机器%s webroot不能是已存在的目录，必须为软链接，你不必新建，walle会自行创建。' % (server_info['host']),
'how': '手工删除远程目标机器：%s webroot目录：%s' % (server_info['host'], self.project_info['target_root']),
})
# remote release directory
return errors
def walle_rollback(self):
self.start()
try:
is_all_servers_success = True
self.release_version = self.taskMdl.get('link_id')
for server_info in self.servers:
host = server_info['host']
try:
waller = Waller(host=host, user=server_info['user'], port=server_info['port'], inline_ssh_env=True)
waller.init_env(env=self.custom_global_env)
self.connections[host] = waller
self.prev_release_custom(self.connections[host])
self.release(self.connections[host])
self.post_release(self.connections[host])
RecordModel().save_record(stage=RecordModel.stage_end, sequence=0, user_id=current_user.id,
task_id=self.task_id, status=RecordModel.status_success, host=host,
user=server_info['user'], command='')
emit('success', {'event': 'finish', 'data': {'host': host, 'message': host + ' 部署完成！'}}, room=self.task_id)
except Exception as e:
is_all_servers_success = False
current_app.logger.error(e)
self.errors[host] = e.message
RecordModel().save_record(stage=RecordModel.stage_end, sequence=0, user_id=current_user.id,
task_id=self.task_id, status=RecordModel.status_fail, host=host,
user=server_info['user'], command='')
emit('fail', {'event': 'finish', 'data': {'host': host, 'message': host + Code.code_msg[Code.deploy_fail]}}, room=self.task_id)
self.end(is_all_servers_success)
except Exception as e:
self.end(False)
return {'success': self.success, 'errors': self.errors}
def walle_rollback(self):
self.start()
try:
is_all_servers_success = True
for server_info in self.servers:
host = server_info['host']
try:
waller = Waller(host=host, user=server_info['user'], port=server_info['port'], inline_ssh_env=True)
waller.init_env(env=self.custom_global_env)
self.connections[self.task_id] = waller
self.prev_release_custom(self.connections[self.task_id])
self.release(self.connections[self.task_id])
self.post_release(self.connections[self.task_id])
RecordModel().save_record(stage=RecordModel.stage_end, sequence=0, user_id=current_user.id,
task_id=self.task_id, status=RecordModel.status_success, host=host,
user=server_info['user'], command='')
emit('success', {'event': 'finish', 'data': {'host': host, 'message': host + ' 部署完成！'}}, room=self.task_id)
except Exception as e:
is_all_servers_success = False
current_app.logger.exception(e)
self.errors[host] = e.message
RecordModel().save_record(stage=RecordModel.stage_end, sequence=0, user_id=current_user.id,
task_id=self.task_id, status=RecordModel.status_fail, host=host,
user=server_info['user'], command='')
emit('fail', {'event': 'finish', 'data': {'host': host, 'message': host + Code.code_msg[Code.deploy_fail]}}, room=self.task_id)
self.end(is_all_servers_success)
except Exception as e:
self.end(False)
return {'success': self.success, 'errors': self.errors}
def __init__(self, task_id=None, project_id=None, console=False):
self.local_codebase = current_app.config.get('CODE_BASE')
self.local = Waller(
host=current_app.config.get('LOCAL_SERVER_HOST'),
user=current_app.config.get('LOCAL_SERVER_USER'),
port=current_app.config.get('LOCAL_SERVER_PORT'),
)
self.TaskRecord = RecordModel()
if task_id:
self.task_id = task_id
# task start
current_app.logger.info(self.task_id)
self.taskMdl = TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.task = self.taskMdl.get('target_user')
self.project_info = self.taskMdl.get('project_info')
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
self.project_name = self.project_info['id']
self.dir_codebase_project = self.local_codebase + str(
self.project_name)
self.init_repo()
# start to deploy
self.console = console
def __init__(self, task_id=None, project_id=None, console=False):
self.local_codebase = current_app.config.get('CODE_BASE').rstrip('/') + '/'
self.localhost = Waller(host='127.0.0.1')
self.TaskRecord = RecordModel()
if task_id:
self.task_id = task_id
# task start
current_app.logger.info(self.task_id)
self.taskMdl = TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.project_info = self.taskMdl.get('project_info')
# copy to a local version
self.release_version = '{project_id}_{task_id}_{timestamp}'.format(
project_id=self.project_info['id'],
task_id=self.task_id,
timestamp=time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())),
)
current_app.logger.info(self.taskMdl)
self.custom_global_env = {
'WEBROOT': str(self.project_info['target_root']),
'VERSION': str(self.release_version),
'CURRENT_RELEASE': str(self.project_info['target_releases']),
'BRANCH': str(self.taskMdl.get('branch')),
'TAG': str(self.taskMdl.get('tag')),
'COMMIT_ID': str(self.taskMdl.get('commit_id')),
'PROJECT_NAME': str(self.project_info['name']).replace('"', '').replace("'", '').replace(" ", '_'),
'PROJECT_ID': str(self.project_info['id']),
'TASK_NAME': str(self.taskMdl.get('name')).replace('"', '').replace("'", '').replace(" ", '_'),
'TASK_ID': str(self.task_id),
'DEPLOY_USER': str(self.taskMdl.get('user_name')),
'DEPLOY_TIME': str(time.strftime('%Y%m%d-%H:%M:%S', time.localtime(time.time()))),
}
if self.project_info['task_vars']:
task_vars = [i.strip() for i in self.project_info['task_vars'].split('\n') if i.strip() and not i.strip().startswith('#')]
for var in task_vars:
var_list = var.split('=', 1)
if len(var_list) != 2:
continue
self.custom_global_env[var_list[0]] = var_list[1]
self.localhost.init_env(env=self.custom_global_env)
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
self.servers = self.project_info['servers_info']
self.project_name = self.project_info['id']
self.dir_codebase_project = self.local_codebase + str(self.project_name)
# self.init_repo()
# start to deploy
self.console = console
def project_detection(self):
errors = []
# walle user => walle LOCAL_SERVER_USER
# show ssh_rsa.pub （maybe not necessary）
command = 'whoami'
current_app.logger.info(command)
result = self.local.run(command, exception=False, wenv=self.config())
if result.failed:
errors.append({
'title':
u'本地免密码登录失败',
'why':
result.stdout,
'how':
u'在宿主机中配置免密码登录，把walle启动用户%s的~/.ssh/ssh_rsa.pub添加到LOCAL_SERVER_USER用户%s的~/.ssh/authorized_keys。了解更多：http://walle-web.io/docs/troubleshooting.html'
% (pwd.getpwuid(os.getuid())[0],
current_app.config.get('LOCAL_SERVER_USER')),
})
# LOCAL_SERVER_USER => git
# LOCAL_SERVER_USER => target_servers
for server_info in self.servers:
waller = Waller(host=server_info['host'],
user=server_info['user'],
port=server_info['port'])
result = waller.run('id', exception=False, wenv=self.config())
if result.failed:
errors.append({
'title':
u'远程目标机器免密码登录失败',
'why':
u'远程目标机器：%s 错误：%s' % (server_info['host'], result.stdout),
'how':
u'在宿主机中配置免密码登录，把宿主机用户%s的~/.ssh/ssh_rsa.pub添加到远程目标机器用户%s的~/.ssh/authorized_keys。了解更多：http://walle-web.io/docs/troubleshooting.html'
% (current_app.config.get('LOCAL_SERVER_USER'),
server_info['host']),
})
# 检查 webroot 父目录是否存在,是否为软链
command = '[ -L "%s" ] && echo "true" || echo "false"' % (
self.project_info['target_root'])
result = waller.run(command, exception=False, wenv=self.config())
if result.stdout == 'false':
errors.append({
'title':
u'远程目标机器webroot不能是已建好的目录',
'why':
u'远程目标机器%s webroot不能是已存在的目录，必须为软链接，你不必新建，walle会自行创建。' %
(server_info['host']),
'how':
u'手工删除远程目标机器：%s webroot目录：%s' %
(server_info['host'], self.project_info['target_root']),
})
# remote release directory
return errors
def __init__(self, task_id=None, project_id=None):
self.local = Waller(host=current_app.config.get('LOCAL_SERVER_HOST'),
user=current_app.config.get('LOCAL_SERVER_USER'),
port=current_app.config.get('LOCAL_SERVER_PORT'))
self.TaskRecord = TaskModel.TaskRecordModel()
if task_id:
self.task_id = task_id
self.taskMdl = TaskModel.TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.task = self.taskMdl.get('target_user')
self.project_info = self.taskMdl.get('project_info')
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
def walle_deploy(self):
self.start()
self.prev_deploy()
self.deploy()
self.post_deploy()
# server = '172.16.0.231'
# try:
# self.connections[server] = Waller(host=server, user=self.project_info['target_user'])
# self.prev_release(self.connections[server])
# self.release(self.connections[server])
# self.post_release(self.connections[server])
# except Exception as e:
# current_app.logger.exception(e)
# self.errors[server] = e.message
all_servers_success = True
for server_info in self.servers:
server = server_info['host']
try:
self.connections[server] = Waller(
host=server, user=self.project_info['target_user'])
self.prev_release(self.connections[server])
self.release(self.connections[server])
self.post_release(self.connections[server])
except Exception as e:
all_servers_success = False
self.errors[server] = e.message
self.end(all_servers_success)
return {'success': self.success, 'errors': self.errors}
def walle_deploy(self):
self.start()
try:
self.prev_deploy()
self.deploy()
self.post_deploy()
is_all_servers_success = True
for server_info in self.servers:
host = server_info['host']
try:
self.connections[host] = Waller(host=host, user=server_info['user'], port=server_info['port'])
self.prev_release(self.connections[host])
self.release(self.connections[host])
self.post_release(self.connections[host])
except Exception as e:
is_all_servers_success = False
current_app.logger.error(e)
self.errors[host] = e.message
self.end(is_all_servers_success)
except Exception as e:
self.end(False)
return {'success': self.success, 'errors': self.errors}
def walle_deploy(self):
self.start()
try:
self.prev_deploy()
self.deploy()
self.post_deploy()
is_all_servers_success = True
for server_info in self.servers:
server = server_info['host']
try:
self.connections[server] = Waller(
host=server, user=self.project_info['target_user'])
self.prev_release(self.connections[server])
self.release(self.connections[server])
self.post_release(self.connections[server])
except Exception as e:
is_all_servers_success = False
current_app.logger.error(e)
self.errors[server] = e.message
self.end(is_all_servers_success)
except Exception as e:
self.end(False)
emit('fail', {
'event': 'console',
'data': {
'message': Code.code_msg[Code.deploy_fail]
}
},
room=self.task_id)
return {'success': self.success, 'errors': self.errors}
def walle_deploy(self):
self.prev_deploy()
self.deploy()
self.post_deploy()
server = '172.16.0.231'
try:
self.connections[server] = Waller(
host=server, user=self.project_info['target_user'])
self.prev_release(self.connections[server])
self.release(self.connections[server])
self.post_release(self.connections[server])
except Exception as e:
current_app.logger.exception(e)
self.errors[server] = e.message
# for server_info in self.servers:
# server = server_info.host
# try:
# self.connections[server] = Waller(host=server, user=self.project_info['target_user'])
# self.prev_release(self.connections[server])
# self.release(self.connections[server])
# self.post_release(self.connections[server])
# except Exception, e:
# self.errors[server] = e.message
return {'success': self.success, 'errors': self.errors}
def __init__(self, task_id=None, project_id=None, console=False):
self.local_codebase = current_app.config.get('CODE_BASE').rstrip('/') + '/'
self.localhost = Waller(host='127.0.0.1')
self.TaskRecord = RecordModel()
if task_id:
self.task_id = task_id
# task start
current_app.logger.info(self.task_id)
self.taskMdl = TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.project_info = self.taskMdl.get('project_info')
# copy to a local version
self.release_version = '{project_id}_{task_id}_{timestamp}'.format(
project_id=self.project_info['id'],
task_id=self.task_id,
timestamp=time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())),
)
current_app.logger.info(self.taskMdl)
self.custom_global_env = {
'WEBROOT': str(self.project_info['target_root']),
'VERSION': str(self.release_version),
'CURRENT_RELEASE': str(self.project_info['target_releases']),
'BRANCH': str(self.taskMdl.get('branch')),
'TAG': str(self.taskMdl.get('tag')),
'COMMIT_ID': str(self.taskMdl.get('commit_id')),
'PROJECT_NAME': str(self.project_info['name']).replace('"', '').replace("'", '').replace(" ", '_'),
'PROJECT_ID': str(self.project_info['id']),
'TASK_NAME': str(self.taskMdl.get('name')).replace('"', '').replace("'", '').replace(" ", '_'),
'TASK_ID': str(self.task_id),
'DEPLOY_USER': str(self.taskMdl.get('user_name')),
'DEPLOY_TIME': str(time.strftime('%Y%m%d-%H:%M:%S', time.localtime(time.time()))),
}
if self.project_info['task_vars']:
task_vars = [i.strip() for i in self.project_info['task_vars'].split('\n') if i.strip() and not i.strip().startswith('#')]
for var in task_vars:
var_list = var.split('=', 1)
if len(var_list) != 2:
continue
self.custom_global_env[var_list[0]] = var_list[1]
self.localhost.init_env(env=self.custom_global_env)
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
self.servers = self.project_info['servers_info']
self.project_name = self.project_info['id']
self.dir_codebase_project = self.local_codebase + str(self.project_name)
# self.init_repo()
# start to deploy
self.console = console
def test(self):
server = '172.20.95.43'
# server = '172.16.0.231'
try:
self.connections[server] = Waller(host=server, user='******')
self.post_release_service(self.connections[server])
except Exception as e:
current_app.logger.exception(e)
self.errors[server] = e.message
def test(self):
# server = '172.20.95.43'
server = '172.16.0.231'
ws_dict = {
'user': '******',
'host': 'ip',
'cmd': 'Going to sleep !!!!!',
'status': 0,
'stage': 1,
'sequence': 1,
'success': 'Going to sleep !!!!!',
'error': '',
}
emit('console', {
'event': 'task:console',
'data': ws_dict
},
room=self.task_id)
socketio.sleep(60)
ws_dict = {
'user': '******',
'host': 'ip',
'cmd': 'sleep 60',
'status': 0,
'stage': 1,
'sequence': 1,
'success': 'sleep 60....',
'error': '',
}
emit('console', {
'event': 'task:console',
'data': ws_dict
},
room=self.task_id)
socketio.sleep(10)
ws_dict = {
'user': '******',
'host': 'ip',
'cmd': 'sleep 10',
'status': 0,
'stage': 1,
'sequence': 1,
'success': 'sleep 10....',
'error': '',
}
emit('console', {
'event': 'task:console',
'data': ws_dict
},
room=self.task_id)
try:
self.connections[server] = Waller(host=server, user='******')
self.post_release_service(self.connections[server])
except Exception as e:
current_app.logger.exception(e)
self.errors[server] = e.message
def __init__(self, task_id=None, project_id=None, websocket=None):
self.local = Waller(host=current_app.config.get('LOCAL_SERVER_HOST'),
user=current_app.config.get('LOCAL_SERVER_USER'),
port=current_app.config.get('LOCAL_SERVER_PORT'))
self.TaskRecord = TaskModel.TaskRecordModel()
current_app.logger.info('DeploySocketIO.__init__ before')
emit('message', {'msg': 'DeploySocketIO.__init__'})
current_app.logger.info('DeploySocketIO.__init__ after')
# if websocket:
# websocket.send_updates(__name__)
# self.websocket = websocket
if task_id:
self.task_id = task_id
self.taskMdl = TaskModel.TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.task = self.taskMdl.get('target_user')
self.project_info = self.taskMdl.get('project_info')
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
def __init__(self, task_id=None, project_id=None, console=False):
self.local = Waller(host=current_app.config.get('LOCAL_SERVER_HOST'),
user=current_app.config.get('LOCAL_SERVER_USER'),
port=current_app.config.get('LOCAL_SERVER_PORT'))
self.TaskRecord = RecordModel()
if task_id:
self.task_id = task_id
# task start
TaskModel(id=self.task_id).update(status=TaskModel.status_doing)
self.taskMdl = TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.task = self.taskMdl.get('target_user')
self.project_info = self.taskMdl.get('project_info')
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
self.project_name = self.project_info['id']
self.dir_codebase_project = self.dir_codebase + str(self.project_name)
def walle_deploy(self):
self.prev_deploy()
self.deploy()
self.post_deploy()
server = '172.16.0.231'
try:
self.connections[server] = Waller(
host=server, user=self.project_info['target_user'])
self.prev_release(self.connections[server])
self.release(self.connections[server])
self.post_release(self.connections[server])
except Exception, e:
current_app.logger.exception(e)
self.errors[server] = e.message
def __init__(self, task_id=None, project_id=None, console=False):
self.local = Waller(host=current_app.config.get('LOCAL_SERVER_HOST'),
user=current_app.config.get('LOCAL_SERVER_USER'),
port=current_app.config.get('LOCAL_SERVER_PORT'))
self.TaskRecord = RecordModel()
if task_id:
self.task_id = task_id
# task start
TaskModel(id=self.task_id).update(status=TaskModel.status_doing)
self.taskMdl = TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.task = self.taskMdl.get('target_user')
self.project_info = self.taskMdl.get('project_info')
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
self.project_name = self.project_info['id']
self.dir_codebase_project = self.dir_codebase + str(self.project_name)
class Deployer:
'''
序列号
'''
stage = '0'
sequence = 0
stage_prev_deploy = 'prev_deploy'
stage_deploy = 'deploy'
stage_post_deploy = 'post_deploy'
stage_prev_release = 'prev_release'
stage_release = 'release'
stage_post_release = 'post_release'
task_id = '0'
user_id = '0'
taskMdl = None
TaskRecord = None
console = False
version = datetime.now().strftime('%Y%m%d%H%M%s')
local_codebase, dir_codebase_project, project_name = None, None, None
dir_release, dir_webroot = None, None
connections, success, errors = {}, {}, {}
release_version_tar, release_version = None, None
local = None
def __init__(self, task_id=None, project_id=None, console=False):
self.local_codebase = current_app.config.get('CODE_BASE')
self.local = Waller(
host=current_app.config.get('LOCAL_SERVER_HOST'),
user=current_app.config.get('LOCAL_SERVER_USER'),
port=current_app.config.get('LOCAL_SERVER_PORT'),
)
self.TaskRecord = RecordModel()
if task_id:
self.task_id = task_id
# task start
current_app.logger.info(self.task_id)
self.taskMdl = TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.task = self.taskMdl.get('target_user')
self.project_info = self.taskMdl.get('project_info')
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
self.project_name = self.project_info['id']
self.dir_codebase_project = self.local_codebase + str(
self.project_name)
self.init_repo()
# start to deploy
self.console = console
def config(self):
return {
'task_id': self.task_id,
'user_id': self.user_id,
'stage': self.stage,
'sequence': self.sequence,
'console': self.console
}
def start(self):
TaskModel().get_by_id(self.task_id).update(
{'status': TaskModel.status_doing})
self.taskMdl = TaskModel().item(self.task_id)
# ===================== fabric ================
# SocketHandler
def prev_deploy(self):
'''
# TODO
socketio.sleep(0.001)
1.代码检出前要做的基础工作
- 检查 当前用户
- 检查 python 版本
- 检查 git 版本
- 检查 目录是否存在
- 用户自定义命令
:return:
'''
self.stage = self.stage_prev_deploy
self.sequence = 1
# 检查 当前用户
command = 'whoami'
current_app.logger.info(command)
result = self.local.run(command, wenv=self.config())
# 检查 python 版本
command = 'python --version'
result = self.local.run(command, wenv=self.config())
# 检查 git 版本
command = 'git --version'
result = self.local.run(command, wenv=self.config())
# 检查 目录是否存在
self.init_repo()
# TODO to be removed
command = 'mkdir -p %s' % (self.dir_codebase_project)
result = self.local.run(command, wenv=self.config())
# 用户自定义命令
command = self.project_info['prev_deploy']
current_app.logger.info(command)
with self.local.cd(self.dir_codebase_project):
result = self.local.run(command, wenv=self.config())
def deploy(self):
'''
2.检出代码
:param project_name:
:return:
'''
self.stage = self.stage_deploy
self.sequence = 2
current_app.logger.info('git dir: %s',
self.dir_codebase_project + '/.git')
# 如果项目底下有 .git 目录则认为项目完整,可以直接检出代码
# TODO 不标准
if os.path.exists(self.dir_codebase_project + '/.git'):
with self.local.cd(self.dir_codebase_project):
command = 'pwd && git pull'
result = self.local.run(command, wenv=self.config())
else:
# 否则当作新项目检出完整代码
with self.local.cd(self.dir_codebase_project):
command = 'pwd && git clone %s .' % (
self.project_info['repo_url'])
current_app.logger.info('cd %s command: %s ',
self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
# copy to a local version
self.release_version = '%s_%s_%s' % (
self.project_name, self.task_id,
time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())))
with self.local.cd(self.local_codebase):
command = 'cp -rf %s %s' % (self.dir_codebase_project,
self.release_version)
current_app.logger.info('cd %s command: %s ',
self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
# 更新到指定 commit_id
with self.local.cd(self.local_codebase + self.release_version):
command = 'git reset -q --hard %s' % (
self.taskMdl.get('commit_id'))
result = self.local.run(command, wenv=self.config())
if result.exited != Code.Ok:
raise WalleError(Code.shell_git_fail, message=result.stdout)
def post_deploy(self):
'''
3.检出代码后要做的任务
- 用户自定义操作命令
- 代码编译
- 清除日志文件及无用文件
-
- 压缩打包
- 传送到版本库 release
:return:
'''
self.stage = self.stage_post_deploy
self.sequence = 3
# 用户自定义命令
command = self.project_info['post_deploy']
with self.local.cd(self.local_codebase + self.release_version):
result = self.local.run(command, wenv=self.config())
# 压缩打包
self.release_version_tar = '%s.tgz' % (self.release_version)
with self.local.cd(self.local_codebase):
command = 'tar zcf %s %s' % (self.release_version_tar,
self.release_version)
result = self.local.run(command, wenv=self.config())
def prev_release(self, waller):
'''
4.部署代码到目标机器前做的任务
- 检查 webroot 父目录是否存在
:return:
'''
self.stage = self.stage_prev_release
self.sequence = 4
# 检查 target_releases 父目录是否存在
command = 'mkdir -p %s' % (self.project_info['target_releases'])
result = waller.run(command, wenv=self.config())
# TODO 检查 webroot 父目录是否存在,是否为软链
# command = 'mkdir -p %s' % (self.project_info['target_root'])
# result = waller.run(command)
# current_app.logger.info('command: %s', dir(result))
# 用户自定义命令
command = self.project_info['prev_release']
current_app.logger.info(command)
with waller.cd(self.project_info['target_releases']):
result = waller.run(command, wenv=self.config())
# TODO md5
# 传送到版本库 release
current_app.logger.info('/tmp/walle/codebase/' +
self.release_version_tar)
result = waller.put('/tmp/walle/codebase/' + self.release_version_tar,
remote=self.project_info['target_releases'],
wenv=self.config())
current_app.logger.info('command: %s', dir(result))
# 解压
self.release_untar(waller)
def release(self, waller):
'''
5.部署代码到目标机器做的任务
- 打包代码 local
- scp local => remote
- 解压 remote
:return:
'''
self.stage = self.stage_release
self.sequence = 5
with waller.cd(self.project_info['target_releases']):
# 1. create a tmp link dir
current_link_tmp_dir = '%s/current-tmp-%s' % (
self.project_info['target_releases'], self.task_id)
command = 'ln -sfn %s/%s %s' % (
self.project_info['target_releases'], self.release_version,
current_link_tmp_dir)
result = waller.run(command, wenv=self.config())
# 2. make a soft link from release to tmp link
# 3. move tmp link to webroot
current_link_tmp_dir = '%s/current-tmp-%s' % (
self.project_info['target_releases'], self.task_id)
command = 'mv -fT %s %s' % (current_link_tmp_dir,
self.project_info['target_root'])
result = waller.run(command, wenv=self.config())
def release_untar(self, waller):
'''
解压版本包
:return:
'''
with waller.cd(self.project_info['target_releases']):
command = 'tar zxf %s' % (self.release_version_tar)
result = waller.run(command, wenv=self.config())
def post_release(self, waller):
'''
6.部署代码到目标机器后要做的任务
- 切换软链
- 重启 nginx
:return:
'''
self.stage = self.stage_post_release
self.sequence = 6
# 用户自定义命令
command = self.project_info['post_release']
current_app.logger.info(command)
with waller.cd(self.project_info['target_root']):
result = waller.run(command, wenv=self.config())
self.post_release_service(waller)
def post_release_service(self, waller):
'''
代码部署完成后,服务启动工作,如: nginx重启
:param connection:
:return:
'''
with waller.cd(self.project_info['target_root']):
command = 'sudo service nginx restart'
result = waller.run(command, wenv=self.config())
def list_tag(self):
with self.local.cd(self.dir_codebase_project):
command = 'git tag -l'
result = self.local.run(command, pty=False, wenv=self.config())
tags = result.stdout.strip()
tags = tags.split('\n')
return [color_clean(tag.strip()) for tag in tags]
return None
def list_branch(self):
with self.local.cd(self.dir_codebase_project):
command = 'git pull'
result = self.local.run(command, wenv=self.config())
if result.exited != Code.Ok:
raise WalleError(Code.shell_git_pull_fail,
message=result.stdout)
current_app.logger.info(self.dir_codebase_project)
command = 'git branch -r'
result = self.local.run(command, pty=False, wenv=self.config())
# if result.exited != Code.Ok:
# raise WalleError(Code.shell_run_fail)
# TODO 三种可能: false, error, success
branches = result.stdout.strip()
branches = branches.split('\n')
# 去除 origin/HEAD -> 当前指向
# 去除远端前缀
branches = [
branch.strip().lstrip('origin/') for branch in branches
if not branch.strip().startswith('origin/HEAD')
]
return branches
return None
def list_commit(self, branch):
with self.local.cd(self.dir_codebase_project):
command = 'git checkout %s && git pull' % (branch)
self.local.run(command, wenv=self.config())
command = 'git log -50 --pretty="%h #@_@# %an #@_@# %s"'
result = self.local.run(command, pty=False, wenv=self.config())
current_app.logger.info(result.stdout)
commit_log = result.stdout.strip()
current_app.logger.info(commit_log)
commit_list = commit_log.split('\n')
commits = []
for commit in commit_list:
if not re.search('^.+ #@_@# .+ #@_@# .*$', commit):
continue
commit_dict = commit.split(' #@_@# ')
current_app.logger.info(commit_dict)
commits.append({
'id': commit_dict[0],
'name': commit_dict[1],
'message': commit_dict[2],
})
return commits
# TODO
return None
def init_repo(self):
if not os.path.exists(self.dir_codebase_project):
# 检查 目录是否存在
command = 'mkdir -p %s' % (self.dir_codebase_project)
# TODO remove
current_app.logger.info(command)
self.local.run(command, wenv=self.config())
with self.local.cd(self.dir_codebase_project):
is_git_dir = self.local.run('git status',
exception=False,
wenv=self.config())
if is_git_dir.exited != Code.Ok:
# 否则当作新项目检出完整代码
# 检查 目录是否存在
command = 'rm -rf %s' % (self.dir_codebase_project)
self.local.run(command, wenv=self.config())
command = 'git clone %s %s' % (self.project_info['repo_url'],
self.dir_codebase_project)
current_app.logger.info('cd %s command: %s ',
self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
if result.exited != Code.Ok:
raise WalleError(Code.shell_git_init_fail,
message=result.stdout)
def end(self, success=True):
status = TaskModel.status_success if success else TaskModel.status_fail
current_app.logger.info('success:%s, status:%s' % (success, status))
TaskModel().get_by_id(self.task_id).update({'status': status})
def walle_deploy(self):
self.start()
try:
self.prev_deploy()
self.deploy()
self.post_deploy()
is_all_servers_success = True
for server_info in self.servers:
server = server_info['host']
try:
self.connections[server] = Waller(
host=server, user=self.project_info['target_user'])
self.prev_release(self.connections[server])
self.release(self.connections[server])
self.post_release(self.connections[server])
except Exception as e:
is_all_servers_success = False
current_app.logger.error(e)
self.errors[server] = e.message
self.end(is_all_servers_success)
except Exception as e:
self.end(False)
emit('fail', {
'event': 'console',
'data': {
'message': Code.code_msg[Code.deploy_fail]
}
},
room=self.task_id)
return {'success': self.success, 'errors': self.errors}
class Deployer:
'''
序列号
'''
stage = 'init'
sequence = 0
stage_prev_deploy = 'prev_deploy'
stage_deploy = 'deploy'
stage_post_deploy = 'post_deploy'
stage_prev_release = 'prev_release'
stage_release = 'release'
stage_post_release = 'post_release'
task_id = '0'
user_id = '0'
taskMdl = None
TaskRecord = None
console = False
custom_global_env = {}
version = datetime.now().strftime('%Y%m%d%H%M%S')
local_codebase, dir_codebase_project, project_name = None, None, None
dir_release, dir_webroot = None, None
connections, success, errors = {}, {}, {}
release_version_tar, previous_release_version, release_version = None, None, None
local = None
def __init__(self, task_id=None, project_id=None, console=False):
self.local_codebase = current_app.config.get('CODE_BASE').rstrip('/') + '/'
self.localhost = Waller(host='127.0.0.1')
self.TaskRecord = RecordModel()
if task_id:
self.task_id = task_id
# task start
current_app.logger.info(self.task_id)
self.taskMdl = TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.project_info = self.taskMdl.get('project_info')
# copy to a local version
self.release_version = '{project_id}_{task_id}_{timestamp}'.format(
project_id=self.project_info['id'],
task_id=self.task_id,
timestamp=time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())),
)
current_app.logger.info(self.taskMdl)
self.custom_global_env = {
'WEBROOT': str(self.project_info['target_root']),
'CURRENT_RELEASE': str(self.release_version),
'BRANCH': str(self.taskMdl.get('branch')),
'TAG': str(self.taskMdl.get('tag')),
'COMMIT_ID': str(self.taskMdl.get('commit_id')),
'PROJECT_NAME': '"{}"'.format(self.project_info['name']),
'PROJECT_ID': str(self.project_info['id']),
'TASK_NAME': '"{}"'.format(self.taskMdl.get('name')),
'TASK_ID': str(self.task_id),
'DEPLOY_USER': str(self.taskMdl.get('user_name')),
'DEPLOY_TIME': str(time.strftime('%Y%m%d-%H:%M:%S', time.localtime(time.time()))),
}
if self.project_info['task_vars']:
task_vars = [i.strip() for i in self.project_info['task_vars'].split('\n') if i.strip() and not i.strip().startswith('#')]
for var in task_vars:
var_list = var.split('=', 1)
if len(var_list) != 2:
continue
self.custom_global_env[var_list[0]] = var_list[1]
self.localhost.init_env(env=self.custom_global_env)
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
self.servers = self.project_info['servers_info']
self.project_name = self.project_info['id']
self.dir_codebase_project = self.local_codebase + str(self.project_name)
# self.init_repo()
# start to deploy
self.console = console
def config(self, console=None):
return {
'task_id': self.task_id,
'user_id': self.user_id,
'stage': self.stage,
'sequence': self.sequence,
'console': console if console is not None else self.console
}
def start(self):
RecordModel().query.filter_by(task_id=self.task_id).delete()
TaskModel().get_by_id(self.task_id).update({'status': TaskModel.status_doing})
self.taskMdl = TaskModel().item(self.task_id)
# ===================== fabric ================
# SocketHandler
def prev_deploy(self):
'''
# TODO
socketio.sleep(0.001)
1.代码检出前要做的基础工作
- 检查 当前用户
- 检查 python 版本
- 检查 git 版本
- 检查 目录是否存在
- 用户自定义命令
:return:
'''
self.stage = self.stage_prev_deploy
self.sequence = 1
# 检查 目录是否存在
self.init_repo()
# 用户自定义命令
commands = self.project_info['prev_deploy']
if commands:
for command in commands.split('\n'):
if command.strip().startswith('#') or not command.strip():
continue
with self.localhost.cd(self.dir_codebase_project):
result = self.localhost.local(command, wenv=self.config())
def deploy(self):
'''
2.检出代码
:param project_name:
:return:
'''
self.stage = self.stage_deploy
self.sequence = 2
#
# # copy to a local version
# self.release_version = '%s_%s_%s' % (
# self.project_name, self.task_id, time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())))
with self.localhost.cd(self.local_codebase):
command = 'cp -rf %s %s' % (self.dir_codebase_project, self.release_version)
current_app.logger.info('cd %s command: %s ', self.dir_codebase_project, command)
result = self.localhost.local(command, wenv=self.config())
# 更新到指定 commit_id
with self.localhost.cd(self.local_codebase + self.release_version):
command = 'git reset -q --hard %s' % (self.taskMdl.get('commit_id'))
result = self.localhost.local(command, wenv=self.config())
if result.exited != Code.Ok:
raise WalleError(Code.shell_git_fail, message=result.stdout)
def post_deploy(self):
'''
3.检出代码后要做的任务
- 用户自定义操作命令
- 代码编译
- 清除日志文件及无用文件
-
- 压缩打包
- 传送到版本库 release
:return:
'''
self.stage = self.stage_post_deploy
self.sequence = 3
# 用户自定义命令
commands = self.project_info['post_deploy']
if commands:
for command in commands.split('\n'):
if command.strip().startswith('#') or not command.strip():
continue
with self.localhost.cd(self.local_codebase + self.release_version):
result = self.localhost.local(command, wenv=self.config())
# 压缩打包
# 排除文件发布
self.release_version_tar = '%s.tgz' % (self.release_version)
with self.localhost.cd(self.local_codebase):
if self.project_info['is_include']:
files = includes_format(self.release_version, self.project_info['excludes'])
else:
files = excludes_format(self.release_version, self.project_info['excludes'])
command = 'tar zcf %s/%s %s' % (self.local_codebase.rstrip('/'), self.release_version_tar, files)
result = self.localhost.local(command, wenv=self.config())
def prev_release(self, waller):
'''
4.部署代码到目标机器前做的任务
- 检查 webroot 父目录是否存在
:return:
'''
self.stage = self.stage_prev_release
self.sequence = 4
# 检查 target_releases 父目录是否存在
command = 'mkdir -p %s' % (self.project_info['target_releases'])
result = waller.run(command, wenv=self.config())
# TODO md5
# 传送到版本库 release
result = waller.put(self.local_codebase + self.release_version_tar,
remote=self.project_info['target_releases'], wenv=self.config())
current_app.logger.info('command: %s', dir(result))
# 解压
self.release_untar(waller)
# 用户自定义命令
self.prev_release_custom(waller)
def prev_release_custom(self, waller):
# 用户自定义命令
commands = self.project_info['prev_release']
if commands:
for command in commands.split('\n'):
if command.strip().startswith('#') or not command.strip():
continue
# TODO
target_release_version = "%s/%s" % (self.project_info['target_releases'], self.release_version)
with waller.cd(target_release_version):
result = waller.run(command, wenv=self.config())
def release(self, waller):
'''
5.部署代码到目标机器做的任务
- 打包代码 local
- scp local => remote
- 解压 remote
:return:
'''
self.stage = self.stage_release
self.sequence = 5
with waller.cd(self.project_info['target_releases']):
# 0. get previous link
command = '[ -L %s ] && readlink %s || echo ""' % (self.project_info['target_root'], self.project_info['target_root'])
result = waller.run(command, wenv=self.config(console=False))
self.previous_release_version = os.path.basename(result.stdout).strip()
# 1. create a tmp link dir
current_link_tmp_dir = '%s/current-tmp-%s' % (self.project_info['target_releases'], self.task_id)
command = 'ln -sfn %s/%s %s' % (
self.project_info['target_releases'], self.release_version, current_link_tmp_dir)
result = waller.run(command, wenv=self.config())
# 2. make a soft link from release to tmp link
# 3. move tmp link to webroot
current_link_tmp_dir = '%s/current-tmp-%s' % (self.project_info['target_releases'], self.task_id)
command = 'mv -fT %s %s' % (current_link_tmp_dir, self.project_info['target_root'])
result = waller.run(command, wenv=self.config())
def rollback(self, waller):
'''
5.部署代码到目标机器做的任务
- 恢复旧版本
:return:
'''
self.stage = self.stage_release
self.sequence = 5
with waller.cd(self.project_info['target_releases']):
# 0. get previous link
command = '[ -L %s ] && readlink %s || echo ""' % (self.project_info['target_root'], self.project_info['target_root'])
result = waller.run(command, wenv=self.config(console=False))
self.previous_release_version = os.path.basename(result.stdout)
# 1. create a tmp link dir
current_link_tmp_dir = '%s/current-tmp-%s' % (self.project_info['target_releases'], self.task_id)
command = 'ln -sfn %s/%s %s' % (
self.project_info['target_releases'], self.release_version, current_link_tmp_dir)
result = waller.run(command, wenv=self.config())
# 2. make a soft link from release to tmp link
# 3. move tmp link to webroot
current_link_tmp_dir = '%s/current-tmp-%s' % (self.project_info['target_releases'], self.task_id)
command = 'mv -fT %s %s' % (current_link_tmp_dir, self.project_info['target_root'])
result = waller.run(command, wenv=self.config())
def release_untar(self, waller):
'''
解压版本包
:return:
'''
with waller.cd(self.project_info['target_releases']):
command = 'tar zxf %s' % (self.release_version_tar)
result = waller.run(command, wenv=self.config())
def post_release(self, waller):
'''
6.部署代码到目标机器后要做的任务
- 切换软链
- 重启 nginx
:return:
'''
self.stage = self.stage_post_release
self.sequence = 6
# 用户自定义命令
commands = self.project_info['post_release']
if commands:
for command in commands.split('\n'):
if command.strip().startswith('#') or not command.strip():
continue
# TODO
with waller.cd(self.project_info['target_root']):
pty = False if command.find('nohup') >= 0 else True
result = waller.run(command, wenv=self.config(), pty=pty)
# 个性化，用户重启的不一定是NGINX，可能是tomcat, apache, php-fpm等
# self.post_release_service(waller)
def post_release_service(self, waller):
'''
代码部署完成后,服务启动工作,如: nginx重启
:param connection:
:return:
'''
with waller.cd(self.project_info['target_root']):
command = 'sudo service nginx restart'
result = waller.run(command, wenv=self.config())
def project_detection(self):
errors = []
# LOCAL_SERVER_USER => git
# LOCAL_SERVER_USER => target_servers
for server_info in self.servers:
waller = Waller(host=server_info['host'], user=server_info['user'], port=server_info['port'])
result = waller.run('id', exception=False, wenv=self.config())
if result.failed:
errors.append({
'title': '远程目标机器免密码登录失败',
'why': '远程目标机器：%s 错误：%s' % (server_info['host'], result.stdout),
'how': '在宿主机中配置免密码登录，把宿主机用户%s的~/.ssh/ssh_rsa.pub添加到远程目标机器用户%s的~/.ssh/authorized_keys。了解更多：http://walle-web.io/docs/troubleshooting.html' % (
pwd.getpwuid(os.getuid())[0], server_info['host']),
})
# maybe this is no webroot's parent dir
command = '[ -d {webroot} ] || mkdir -p {webroot}'.format(webroot=os.path.basename(self.project_info['target_root']))
result = waller.run(command, exception=False, wenv=self.config(console=False))
# 检查 webroot 父目录是否存在,是否为软链
command = '[ -L "%s" ] && echo "true" || echo "false"' % (self.project_info['target_root'])
result = waller.run(command, exception=False, wenv=self.config())
if result.stdout == 'false':
errors.append({
'title': '远程目标机器webroot不能是已建好的目录',
'why': '远程目标机器%s webroot不能是已存在的目录，必须为软链接，你不必新建，walle会自行创建。' % (server_info['host']),
'how': '手工删除远程目标机器：%s webroot目录：%s' % (server_info['host'], self.project_info['target_root']),
})
# remote release directory
return errors
def list_tag(self):
self.init_repo()
with self.localhost.cd(self.dir_codebase_project):
command = 'git tag -l'
result = self.localhost.local(command, pty=False, wenv=self.config())
tags = result.stdout.strip()
tags = tags.split('\n')
return [color_clean(tag.strip()) for tag in tags]
return None
def list_branch(self):
self.init_repo()
with self.localhost.cd(self.dir_codebase_project):
command = 'git pull'
result = self.localhost.local(command, wenv=self.config())
if result.exited != Code.Ok:
raise WalleError(Code.shell_git_pull_fail, message=result.stdout)
current_app.logger.info(self.dir_codebase_project)
command = 'git branch -r'
result = self.localhost.local(command, pty=False, wenv=self.config())
# if result.exited != Code.Ok:
# raise WalleError(Code.shell_run_fail)
# TODO 三种可能: false, error, success
branches = result.stdout.strip()
branches = branches.split('\n')
# 去除 origin/HEAD -> 当前指向
# 去除远端前缀
branches = [branch.strip().lstrip('origin/') for branch in branches if
not branch.strip().startswith('origin/HEAD')]
return branches
return None
def list_commit(self, branch):
self.init_repo()
with self.localhost.cd(self.dir_codebase_project):
command = 'git checkout %s && git pull' % (branch)
self.localhost.local(command, wenv=self.config())
command = 'git log -50 --pretty="%h #@_@# %an #@_@# %s"'
result = self.localhost.local(command, pty=False, wenv=self.config())
current_app.logger.info(result.stdout)
commit_log = result.stdout.strip()
current_app.logger.info(commit_log)
commit_list = commit_log.split('\n')
commits = []
for commit in commit_list:
if not re.search('^.+ #@_@# .+ #@_@# .*$', commit):
continue
commit_dict = commit.split(' #@_@# ')
current_app.logger.info(commit_dict)
commits.append({
'id': commit_dict[0],
'name': commit_dict[1],
'message': commit_dict[2],
})
return commits
# TODO
return None
def init_repo(self):
if not os.path.exists(self.dir_codebase_project):
# 检查 目录是否存在
command = 'mkdir -p %s' % (self.dir_codebase_project)
self.localhost.local(command, wenv=self.config())
with self.localhost.cd(self.dir_codebase_project):
is_git_dir = self.localhost.local('[ -d ".git" ] && git status', exception=False, wenv=self.config())
if is_git_dir.exited != Code.Ok:
# 否则当作新项目检出完整代码
# 检查 目录是否存在
command = 'rm -rf %s' % (self.dir_codebase_project)
self.localhost.local(command, wenv=self.config())
command = 'git clone %s %s' % (self.project_info['repo_url'], self.dir_codebase_project)
current_app.logger.info('cd %s command: %s ' % (self.dir_codebase_project, command))
result = self.localhost.local(command, wenv=self.config())
if result.exited != Code.Ok:
raise WalleError(Code.shell_git_init_fail, message=result.stdout)
def logs(self):
return RecordModel().fetch(task_id=self.task_id)
def end(self, success=True, update_status=True):
if update_status:
status = TaskModel.status_success if success else TaskModel.status_fail
current_app.logger.info('success:%s, status:%s' % (success, status))
TaskModel().get_by_id(self.task_id).update({
'status': status,
'link_id': self.release_version,
'ex_link_id': self.previous_release_version,
})
notice_info = {
'title': '',
'username': current_user.username,
'project_name': self.project_info['name'],
'task_name': '%s ([%s](%s))' % (self.taskMdl.get('name'), self.task_id, Notice.task_url(project_name=self.project_info['name'], task_id=self.task_id)),
'branch': self.taskMdl.get('branch'),
'commit': self.taskMdl.get('commit_id'),
'is_branch': self.project_info['repo_mode'],
}
notice = Notice.create(self.project_info['notice_type'])
if success:
notice_info['title'] = '上线部署成功'
notice.deploy_task(project_info=self.project_info, notice_info=notice_info)
else:
notice_info['title'] = '上线部署失败'
notice.deploy_task(project_info=self.project_info, notice_info=notice_info)
if success:
emit('success', {'event': 'finish', 'data': {'message': '部署完成，辛苦了，为你的努力喝彩！'}}, room=self.task_id)
else:
emit('fail', {'event': 'finish', 'data': {'message': Code.code_msg[Code.deploy_fail]}}, room=self.task_id)
def walle_deploy(self):
self.start()
try:
self.prev_deploy()
self.deploy()
self.post_deploy()
is_all_servers_success = True
for server_info in self.servers:
host = server_info['host']
try:
waller = Waller(host=host, user=server_info['user'], port=server_info['port'], inline_ssh_env=True)
waller.init_env(env=self.custom_global_env)
self.connections[host] = waller
self.prev_release(self.connections[host])
self.release(self.connections[host])
self.post_release(self.connections[host])
RecordModel().save_record(stage=RecordModel.stage_end, sequence=0, user_id=current_user.id,
task_id=self.task_id, status=RecordModel.status_success, host=host,
user=server_info['user'], command='')
emit('success', {'event': 'finish', 'data': {'host': host, 'message': host + ' 部署完成！'}}, room=self.task_id)
except Exception as e:
is_all_servers_success = False
current_app.logger.error(e)
self.errors[host] = e.message
RecordModel().save_record(stage=RecordModel.stage_end, sequence=0, user_id=current_user.id,
task_id=self.task_id, status=RecordModel.status_fail, host=host,
user=server_info['user'], command='')
emit('fail', {'event': 'finish', 'data': {'host': host, 'message': host + Code.code_msg[Code.deploy_fail]}}, room=self.task_id)
self.end(is_all_servers_success)
except Exception as e:
self.end(False)
return {'success': self.success, 'errors': self.errors}
def walle_rollback(self):
self.start()
try:
is_all_servers_success = True
self.release_version = self.taskMdl.get('link_id')
for server_info in self.servers:
host = server_info['host']
try:
waller = Waller(host=host, user=server_info['user'], port=server_info['port'], inline_ssh_env=True)
waller.init_env(env=self.custom_global_env)
self.connections[host] = waller
self.prev_release_custom(self.connections[host])
self.release(self.connections[host])
self.post_release(self.connections[host])
RecordModel().save_record(stage=RecordModel.stage_end, sequence=0, user_id=current_user.id,
task_id=self.task_id, status=RecordModel.status_success, host=host,
user=server_info['user'], command='')
emit('success', {'event': 'finish', 'data': {'host': host, 'message': host + ' 部署完成！'}}, room=self.task_id)
except Exception as e:
is_all_servers_success = False
current_app.logger.error(e)
self.errors[host] = e.message
RecordModel().save_record(stage=RecordModel.stage_end, sequence=0, user_id=current_user.id,
task_id=self.task_id, status=RecordModel.status_fail, host=host,
user=server_info['user'], command='')
emit('fail', {'event': 'finish', 'data': {'host': host, 'message': host + Code.code_msg[Code.deploy_fail]}}, room=self.task_id)
self.end(is_all_servers_success)
except Exception as e:
self.end(False)
return {'success': self.success, 'errors': self.errors}
class Deployer:
'''
序列号
'''
stage = 'init'
sequence = 0
stage_prev_deploy = 'prev_deploy'
stage_deploy = 'deploy'
stage_post_deploy = 'post_deploy'
stage_prev_release = 'prev_release'
stage_release = 'release'
stage_post_release = 'post_release'
task_id = '0'
user_id = '0'
taskMdl = None
TaskRecord = None
console = False
version = datetime.now().strftime('%Y%m%d%H%M%S')
local_codebase, dir_codebase_project, project_name = None, None, None
dir_release, dir_webroot = None, None
connections, success, errors = {}, {}, {}
release_version_tar, previous_release_version, release_version = None, None, None
local = None
def __init__(self, task_id=None, project_id=None, console=False):
self.local_codebase = current_app.config.get('CODE_BASE')
self.localhost = Waller(host='127.0.0.1')
self.TaskRecord = RecordModel()
if task_id:
self.task_id = task_id
# task start
current_app.logger.info(self.task_id)
self.taskMdl = TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.project_info = self.taskMdl.get('project_info')
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
self.servers = self.project_info['servers_info']
self.project_name = self.project_info['id']
self.dir_codebase_project = self.local_codebase + str(self.project_name)
# self.init_repo()
# start to deploy
self.console = console
def config(self, console=None):
return {
'task_id': self.task_id,
'user_id': self.user_id,
'stage': self.stage,
'sequence': self.sequence,
'console': console if console is not None else self.console
}
def start(self):
RecordModel().query.filter_by(task_id=self.task_id).delete()
TaskModel().get_by_id(self.task_id).update({'status': TaskModel.status_doing})
self.taskMdl = TaskModel().item(self.task_id)
# ===================== fabric ================
# SocketHandler
def prev_deploy(self):
'''
# TODO
socketio.sleep(0.001)
1.代码检出前要做的基础工作
- 检查 当前用户
- 检查 python 版本
- 检查 git 版本
- 检查 目录是否存在
- 用户自定义命令
:return:
'''
self.stage = self.stage_prev_deploy
self.sequence = 1
# 检查 目录是否存在
self.init_repo()
# 用户自定义命令
command = self.project_info['prev_deploy']
if command:
current_app.logger.info(command)
with self.localhost.cd(self.dir_codebase_project):
result = self.localhost.local(command, wenv=self.config())
def deploy(self):
'''
2.检出代码
:param project_name:
:return:
'''
self.stage = self.stage_deploy
self.sequence = 2
# copy to a local version
self.release_version = '%s_%s_%s' % (
self.project_name, self.task_id, time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())))
with self.localhost.cd(self.local_codebase):
command = 'cp -rf %s %s' % (self.dir_codebase_project, self.release_version)
current_app.logger.info('cd %s command: %s ', self.dir_codebase_project, command)
result = self.localhost.local(command, wenv=self.config())
# 更新到指定 commit_id
with self.localhost.cd(self.local_codebase + self.release_version):
command = 'git reset -q --hard %s' % (self.taskMdl.get('commit_id'))
result = self.localhost.local(command, wenv=self.config())
if result.exited != Code.Ok:
raise WalleError(Code.shell_git_fail, message=result.stdout)
def post_deploy(self):
'''
3.检出代码后要做的任务
- 用户自定义操作命令
- 代码编译
- 清除日志文件及无用文件
-
- 压缩打包
- 传送到版本库 release
:return:
'''
self.stage = self.stage_post_deploy
self.sequence = 3
# 用户自定义命令
command = self.project_info['post_deploy']
if command:
with self.localhost.cd(self.local_codebase + self.release_version):
result = self.localhost.local(command, wenv=self.config())
# 压缩打包
# 排除文件发布
self.release_version_tar = '%s.tgz' % (self.release_version)
with self.localhost.cd(self.local_codebase):
excludes = excludes_format(self.project_info['excludes'])
command = 'tar zcf %s %s %s' % (self.release_version_tar, excludes, self.release_version)
result = self.localhost.local(command, wenv=self.config())
# # 指定文件发布
# self.release_version_tar = '%s.tgz' % (self.release_version)
# with self.localhost.cd(self.local_codebase):
# excludes = suffix_format(self.dir_codebase_project, self.project_info['excludes'])
# command = 'tar zcf %s %s %s' % (self.release_version_tar, excludes, self.release_version)
# result = self.local.run(command, wenv=self.config())
def prev_release(self, waller):
'''
4.部署代码到目标机器前做的任务
- 检查 webroot 父目录是否存在
:return:
'''
self.stage = self.stage_prev_release
self.sequence = 4
# 检查 target_releases 父目录是否存在
command = 'mkdir -p %s' % (self.project_info['target_releases'])
result = waller.run(command, wenv=self.config())
# TODO md5
# 传送到版本库 release
result = waller.put(self.local_codebase + self.release_version_tar,
remote=self.project_info['target_releases'], wenv=self.config())
current_app.logger.info('command: %s', dir(result))
# 解压
self.release_untar(waller)
# 用户自定义命令
self.prev_release_custom(waller)
def prev_release_custom(self, waller):
# 用户自定义命令
command = self.project_info['prev_release']
if command:
current_app.logger.info(command)
# TODO
target_release_version = "%s/%s" % (self.project_info['target_releases'], self.release_version)
with waller.cd(target_release_version):
result = waller.run(command, wenv=self.config())
def release(self, waller):
'''
5.部署代码到目标机器做的任务
- 打包代码 local
- scp local => remote
- 解压 remote
:return:
'''
self.stage = self.stage_release
self.sequence = 5
with waller.cd(self.project_info['target_releases']):
# 0. get previous link
command = '[ -L %s ] && readlink %s || echo ""' % (self.project_info['target_root'], self.project_info['target_root'])
result = waller.run(command, wenv=self.config(console=False))
self.previous_release_version = os.path.basename(result.stdout).strip()
# 1. create a tmp link dir
current_link_tmp_dir = '%s/current-tmp-%s' % (self.project_info['target_releases'], self.task_id)
command = 'ln -sfn %s/%s %s' % (
self.project_info['target_releases'], self.release_version, current_link_tmp_dir)
result = waller.run(command, wenv=self.config())
# 2. make a soft link from release to tmp link
# 3. move tmp link to webroot
current_link_tmp_dir = '%s/current-tmp-%s' % (self.project_info['target_releases'], self.task_id)
command = 'mv -fT %s %s' % (current_link_tmp_dir, self.project_info['target_root'])
result = waller.run(command, wenv=self.config())
def rollback(self, waller):
'''
5.部署代码到目标机器做的任务
- 恢复旧版本
:return:
'''
self.stage = self.stage_release
self.sequence = 5
with waller.cd(self.project_info['target_releases']):
# 0. get previous link
command = '[ -L %s ] && readlink %s || echo ""' % (self.project_info['target_root'], self.project_info['target_root'])
result = waller.run(command, wenv=self.config(console=False))
self.previous_release_version = os.path.basename(result.stdout)
# 1. create a tmp link dir
current_link_tmp_dir = '%s/current-tmp-%s' % (self.project_info['target_releases'], self.task_id)
command = 'ln -sfn %s/%s %s' % (
self.project_info['target_releases'], self.release_version, current_link_tmp_dir)
result = waller.run(command, wenv=self.config())
# 2. make a soft link from release to tmp link
# 3. move tmp link to webroot
current_link_tmp_dir = '%s/current-tmp-%s' % (self.project_info['target_releases'], self.task_id)
command = 'mv -fT %s %s' % (current_link_tmp_dir, self.project_info['target_root'])
result = waller.run(command, wenv=self.config())
def release_untar(self, waller):
'''
解压版本包
:return:
'''
with waller.cd(self.project_info['target_releases']):
command = 'tar zxf %s' % (self.release_version_tar)
result = waller.run(command, wenv=self.config())
def post_release(self, waller):
'''
6.部署代码到目标机器后要做的任务
- 切换软链
- 重启 nginx
:return:
'''
self.stage = self.stage_post_release
self.sequence = 6
# 用户自定义命令
command = self.project_info['post_release']
if command:
current_app.logger.info(command)
with waller.cd(self.project_info['target_root']):
result = waller.run(command, wenv=self.config())
# 个性化，用户重启的不一定是NGINX，可能是tomcat, apache, php-fpm等
# self.post_release_service(waller)
def post_release_service(self, waller):
'''
代码部署完成后,服务启动工作,如: nginx重启
:param connection:
:return:
'''
with waller.cd(self.project_info['target_root']):
command = 'sudo service nginx restart'
result = waller.run(command, wenv=self.config())
def project_detection(self):
errors = []
# LOCAL_SERVER_USER => git
# LOCAL_SERVER_USER => target_servers
for server_info in self.servers:
waller = Waller(host=server_info['host'], user=server_info['user'], port=server_info['port'])
result = waller.run('id', exception=False, wenv=self.config())
if result.failed:
errors.append({
'title': '远程目标机器免密码登录失败',
'why': '远程目标机器：%s 错误：%s' % (server_info['host'], result.stdout),
'how': '在宿主机中配置免密码登录，把宿主机用户%s的~/.ssh/ssh_rsa.pub添加到远程目标机器用户%s的~/.ssh/authorized_keys。了解更多：http://walle-web.io/docs/troubleshooting.html' % (
pwd.getpwuid(os.getuid())[0], server_info['host']),
})
# maybe this is no webroot's parent dir
command = '[ -d {webroot} ] || mkdir -p {webroot}'.format(webroot=os.path.basename(self.project_info['target_root']))
result = waller.run(command, exception=False, wenv=self.config(console=False))
# 检查 webroot 父目录是否存在,是否为软链
command = '[ -L "%s" ] && echo "true" || echo "false"' % (self.project_info['target_root'])
result = waller.run(command, exception=False, wenv=self.config())
if result.stdout == 'false':
errors.append({
'title': '远程目标机器webroot不能是已建好的目录',
'why': '远程目标机器%s webroot不能是已存在的目录，必须为软链接，你不必新建，walle会自行创建。' % (server_info['host']),
'how': '手工删除远程目标机器：%s webroot目录：%s' % (server_info['host'], self.project_info['target_root']),
})
# remote release directory
return errors
def list_tag(self):
self.init_repo()
with self.localhost.cd(self.dir_codebase_project):
command = 'git tag -l'
result = self.localhost.local(command, pty=False, wenv=self.config())
tags = result.stdout.strip()
tags = tags.split('\n')
return [color_clean(tag.strip()) for tag in tags]
return None
def list_branch(self):
self.init_repo()
with self.localhost.cd(self.dir_codebase_project):
command = 'git pull'
result = self.localhost.local(command, wenv=self.config())
if result.exited != Code.Ok:
raise WalleError(Code.shell_git_pull_fail, message=result.stdout)
current_app.logger.info(self.dir_codebase_project)
command = 'git branch -r'
result = self.localhost.local(command, pty=False, wenv=self.config())
# if result.exited != Code.Ok:
# raise WalleError(Code.shell_run_fail)
# TODO 三种可能: false, error, success
branches = result.stdout.strip()
branches = branches.split('\n')
# 去除 origin/HEAD -> 当前指向
# 去除远端前缀
branches = [branch.strip().lstrip('origin/') for branch in branches if
not branch.strip().startswith('origin/HEAD')]
return branches
return None
def list_commit(self, branch):
self.init_repo()
with self.localhost.cd(self.dir_codebase_project):
command = 'git checkout %s && git pull' % (branch)
self.localhost.local(command, wenv=self.config())
command = 'git log -50 --pretty="%h #@_@# %an #@_@# %s"'
result = self.localhost.local(command, pty=False, wenv=self.config())
current_app.logger.info(result.stdout)
commit_log = result.stdout.strip()
current_app.logger.info(commit_log)
commit_list = commit_log.split('\n')
commits = []
for commit in commit_list:
if not re.search('^.+ #@_@# .+ #@_@# .*$', commit):
continue
commit_dict = commit.split(' #@_@# ')
current_app.logger.info(commit_dict)
commits.append({
'id': commit_dict[0],
'name': commit_dict[1],
'message': commit_dict[2],
})
return commits
# TODO
return None
def init_repo(self):
if not os.path.exists(self.dir_codebase_project):
# 检查 目录是否存在
command = 'mkdir -p %s' % (self.dir_codebase_project)
self.localhost.local(command, wenv=self.config())
with self.localhost.cd(self.dir_codebase_project):
is_git_dir = self.localhost.local('[ -d ".git" ] && git status', exception=False, wenv=self.config())
if is_git_dir.exited != Code.Ok:
# 否则当作新项目检出完整代码
# 检查 目录是否存在
command = 'rm -rf %s' % (self.dir_codebase_project)
self.localhost.local(command, wenv=self.config())
command = 'git clone %s %s' % (self.project_info['repo_url'], self.dir_codebase_project)
current_app.logger.info('cd %s command: %s ' % (self.dir_codebase_project, command))
result = self.localhost.local(command, wenv=self.config())
if result.exited != Code.Ok:
raise WalleError(Code.shell_git_init_fail, message=result.stdout)
def logs(self):
return RecordModel().fetch(task_id=self.task_id)
def end(self, success=True, update_status=True):
if update_status:
status = TaskModel.status_success if success else TaskModel.status_fail
current_app.logger.info('success:%s, status:%s' % (success, status))
TaskModel().get_by_id(self.task_id).update({
'status': status,
'link_id': self.release_version,
'ex_link_id': self.previous_release_version,
})
notice_info = {
'title': '',
'username': current_user.username,
'project_name': self.project_info['name'],
'task_name': '%s ([%s](%s))' % (self.taskMdl.get('name'), self.task_id, Notice.task_url(project_name=self.project_info['name'], task_id=self.task_id)),
'branch': self.taskMdl.get('branch'),
'commit': self.taskMdl.get('commit_id'),
'is_branch': self.project_info['repo_mode'],
}
notice = Notice.create(self.project_info['notice_type'])
if success:
notice_info['title'] = '上线部署成功'
notice.deploy_task(project_info=self.project_info, notice_info=notice_info)
else:
notice_info['title'] = '上线部署失败'
notice.deploy_task(project_info=self.project_info, notice_info=notice_info)
if success:
emit('success', {'event': 'finish', 'data': {'message': '部署完成，辛苦了，为你的努力喝彩！'}}, room=self.task_id)
else:
emit('fail', {'event': 'finish', 'data': {'message': Code.code_msg[Code.deploy_fail]}}, room=self.task_id)
def walle_deploy(self):
self.start()
try:
self.prev_deploy()
self.deploy()
self.post_deploy()
is_all_servers_success = True
for server_info in self.servers:
host = server_info['host']
try:
self.connections[host] = Waller(host=host, user=server_info['user'], port=server_info['port'])
self.prev_release(self.connections[host])
self.release(self.connections[host])
self.post_release(self.connections[host])
RecordModel().save_record(stage=RecordModel.stage_end, sequence=0, user_id=current_user.id,
task_id=self.task_id, status=RecordModel.status_success, host=host,
user=server_info['user'], command='')
emit('success', {'event': 'finish', 'data': {'host': host, 'message': host + ' 部署完成！'}}, room=self.task_id)
except Exception as e:
is_all_servers_success = False
current_app.logger.error(e)
self.errors[host] = e.message
RecordModel().save_record(stage=RecordModel.stage_end, sequence=0, user_id=current_user.id,
task_id=self.task_id, status=RecordModel.status_fail, host=host,
user=server_info['user'], command='')
emit('fail', {'event': 'finish', 'data': {'host': host, 'message': host + Code.code_msg[Code.deploy_fail]}}, room=self.task_id)
self.end(is_all_servers_success)
except Exception as e:
self.end(False)
return {'success': self.success, 'errors': self.errors}
def walle_rollback(self):
self.start()
try:
is_all_servers_success = True
self.release_version = self.taskMdl.get('link_id')
for server_info in self.servers:
host = server_info['host']
try:
self.connections[host] = Waller(host=host, user=server_info['user'], port=server_info['port'])
self.prev_release_custom(self.connections[host])
self.release(self.connections[host])
self.post_release(self.connections[host])
RecordModel().save_record(stage=RecordModel.stage_end, sequence=0, user_id=current_user.id,
task_id=self.task_id, status=RecordModel.status_success, host=host,
user=server_info['user'], command='')
emit('success', {'event': 'finish', 'data': {'host': host, 'message': host + ' 部署完成！'}}, room=self.task_id)
except Exception as e:
is_all_servers_success = False
current_app.logger.error(e)
self.errors[host] = e.message
RecordModel().save_record(stage=RecordModel.stage_end, sequence=0, user_id=current_user.id,
task_id=self.task_id, status=RecordModel.status_fail, host=host,
user=server_info['user'], command='')
emit('fail', {'event': 'finish', 'data': {'host': host, 'message': host + Code.code_msg[Code.deploy_fail]}}, room=self.task_id)
self.end(is_all_servers_success)
except Exception as e:
self.end(False)
return {'success': self.success, 'errors': self.errors}
class Deployer:
'''
序列号
'''
stage = 'init'
sequence = 0
stage_prev_deploy = 'prev_deploy'
stage_deploy = 'deploy'
stage_post_deploy = 'post_deploy'
stage_prev_release = 'prev_release'
stage_release = 'release'
stage_post_release = 'post_release'
task_id = '0'
user_id = '0'
taskMdl = None
TaskRecord = None
console = False
custom_global_env = {}
version = datetime.now().strftime('%Y%m%d%H%M%S')
local_codebase, dir_codebase_project, project_name = None, None, None
dir_release, dir_webroot = None, None
connections, success, errors = {}, {}, {}
release_version_tar, previous_release_version, release_version = None, None, None
local = None
def __init__(self, task_id=None, project_id=None, console=False):
self.local_codebase = current_app.config.get('CODE_BASE').rstrip('/') + '/'
self.localhost = Waller(host='127.0.0.1')
self.TaskRecord = RecordModel()
if task_id:
self.task_id = task_id
# task start
current_app.logger.info(self.task_id)
self.taskMdl = TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.project_info = self.taskMdl.get('project_info')
# copy to a local version
self.release_version = self.taskMdl.get('link_id') if (self.taskMdl.get("is_rollback")) else \
'{project_id}_{task_id}_{timestamp}'.format(
project_id=self.project_info['id'],
task_id=self.task_id,
timestamp=time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())),
)
current_app.logger.info(self.taskMdl)
# 将环境变量包在 "" 里，防止特殊字符报错
format_export = lambda val: '"%s"' % str(val).replace('"', '').replace("'", '')
self.custom_global_env = {
'WEBROOT': str(self.project_info['target_root']),
'VERSION': str(self.release_version),
'CURRENT_RELEASE': str(self.project_info['target_releases']),
'BRANCH': format_export(self.taskMdl.get('branch')),
'TAG': str(self.taskMdl.get('tag')),
'COMMIT_ID': str(self.taskMdl.get('commit_id')),
'PROJECT_NAME': format_export(self.project_info['name']),
'PROJECT_ID': str(self.project_info['id']),
'TASK_NAME': format_export(self.taskMdl.get('name')),
'TASK_ID': str(self.task_id),
'DEPLOY_USER': str(self.taskMdl.get('user_name')),
'DEPLOY_TIME': str(time.strftime('%Y%m%d-%H:%M:%S', time.localtime(time.time()))),
}
if self.project_info['task_vars']:
task_vars = [i.strip() for i in self.project_info['task_vars'].split('\n') if i.strip() and not i.strip().startswith('#')]
for var in task_vars:
var_list = var.split('=', 1)
if len(var_list) != 2:
continue
self.custom_global_env[var_list[0]] = var_list[1]
self.localhost.init_env(env=self.custom_global_env)
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
self.servers = self.project_info['servers_info']
self.project_name = self.project_info['id']
self.dir_codebase_project = self.local_codebase + str(self.project_name)
# self.init_repo()
# start to deploy
self.console = console
def config(self, console=None):
return {
'task_id': self.task_id,
'user_id': self.user_id,
'stage': self.stage,
'sequence': self.sequence,
'console': console if console is not None else self.console
}
def start(self):
RecordModel().query.filter_by(task_id=self.task_id).delete()
TaskModel().get_by_id(self.task_id).update({'status': TaskModel.status_doing})
self.taskMdl = TaskModel().item(self.task_id)
# ===================== fabric ================
# SocketHandler
def prev_deploy(self):
'''
# TODO
socketio.sleep(0.001)
1.代码检出前要做的基础工作
- 检查 当前用户
- 检查 python 版本
- 检查 git 版本
- 检查 目录是否存在
- 用户自定义命令
:return:
'''
self.stage = self.stage_prev_deploy
self.sequence = 1
# 检查 目录是否存在
self.init_repo()
# 用户自定义命令
commands = self.project_info['prev_deploy']
if commands:
for command in commands.split('\n'):
if command.strip().startswith('#') or not command.strip():
continue
with self.localhost.cd(self.dir_codebase_project):
result = self.localhost.local(command, wenv=self.config())
def deploy(self):
'''
2.检出代码
:param project_name:
:return:
'''
self.stage = self.stage_deploy
self.sequence = 2
#
# # copy to a local version
# self.release_version = '%s_%s_%s' % (
# self.project_name, self.task_id, time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())))
with self.localhost.cd(self.local_codebase):
command = 'cp -rf %s %s' % (self.dir_codebase_project, self.release_version)
current_app.logger.info('cd %s command: %s ', self.dir_codebase_project, command)
result = self.localhost.local(command, wenv=self.config())
# 更新到指定 branch/commit_id 或 tag
repo = Repo(self.local_codebase + self.release_version)
if self.project_info['repo_mode'] == ProjectModel.repo_mode_branch:
repo.checkout_2_commit(branch=self.taskMdl['branch'], commit=self.taskMdl['commit_id'])
else:
repo.checkout_2_tag(tag=self.taskMdl['tag'])
def post_deploy(self):
'''
3.检出代码后要做的任务
- 用户自定义操作命令
- 代码编译
- 清除日志文件及无用文件
-
- 压缩打包
- 传送到版本库 release
:return:
'''
self.stage = self.stage_post_deploy
self.sequence = 3
# 用户自定义命令
commands = self.project_info['post_deploy']
if commands:
for command in commands.split('\n'):
if command.strip().startswith('#') or not command.strip():
continue
with self.localhost.cd(self.local_codebase + self.release_version):
result = self.localhost.local(command, wenv=self.config())
# 压缩打包
# 排除文件发布
self.release_version_tar = '%s.tgz' % (self.release_version)
with self.localhost.cd(self.local_codebase):
if self.project_info['is_include']:
files = includes_format(self.release_version, self.project_info['excludes'])
else:
files = excludes_format(self.release_version, self.project_info['excludes'])
command = 'tar zcf %s/%s %s' % (self.local_codebase.rstrip('/'), self.release_version_tar, files)
result = self.localhost.local(command, wenv=self.config())
def prev_release(self, waller):
'''
4.部署代码到目标机器前做的任务
- 检查 webroot 父目录是否存在
:return:
'''
self.stage = self.stage_prev_release
self.sequence = 4
# 检查 target_releases 父目录是否存在
command = 'mkdir -p %s' % (self.project_info['target_releases'])
result = waller.run(command, wenv=self.config())
# TODO md5
# 传送到版本库 release
result = waller.put(self.local_codebase + self.release_version_tar,
remote=self.project_info['target_releases'], wenv=self.config())
current_app.logger.info('command: %s', dir(result))
# 解压
self.release_untar(waller)
# 用户自定义命令
self.prev_release_custom(waller)
def prev_release_custom(self, waller):
# 用户自定义命令
commands = self.project_info['prev_release']
if commands:
for command in commands.split('\n'):
if command.strip().startswith('#') or not command.strip():
continue
# TODO
target_release_version = "%s/%s" % (self.project_info['target_releases'], self.release_version)
with waller.cd(target_release_version):
result = waller.run(command, wenv=self.config())
def release(self, waller):
'''
5.部署代码到目标机器做的任务
- 打包代码 local
- scp local => remote
- 解压 remote
:return:
'''
self.stage = self.stage_release
self.sequence = 5
with waller.cd(self.project_info['target_releases']):
# 0. get previous link
command = '[ -L %s ] && readlink %s || echo ""' % (self.project_info['target_root'], self.project_info['target_root'])
result = waller.run(command, wenv=self.config(console=False))
self.previous_release_version = os.path.basename(result.stdout).strip()
# 1. create a tmp link dir
current_link_tmp_dir = 'current-tmp-%s' % (self.task_id)
command = 'ln -sfn {library}/{version} {library}/{current_tmp}'.format(
library=self.project_info['target_releases'], version=self.release_version,
current_tmp=current_link_tmp_dir)
result = waller.run(command, wenv=self.config())
# 2. make a soft link from release to tmp link
# 3. move tmp link to webroot
current_link_tmp_dir = '%s/current-tmp-%s' % (self.project_info['target_releases'], self.task_id)
command = 'mv -fT %s %s' % (current_link_tmp_dir, self.project_info['target_root'])
result = waller.run(command, wenv=self.config())
def rollback(self, waller):
'''
5.部署代码到目标机器做的任务
- 恢复旧版本
:return:
'''
self.stage = self.stage_release
self.sequence = 5
with waller.cd(self.project_info['target_releases']):
# 0. get previous link
command = '[ -L %s ] && readlink %s || echo ""' % (self.project_info['target_root'], self.project_info['target_root'])
result = waller.run(command, wenv=self.config(console=False))
self.previous_release_version = os.path.basename(result.stdout)
# 1. create a tmp link dir
current_link_tmp_dir = '%s/current-tmp-%s' % (self.project_info['target_releases'], self.task_id)
command = 'ln -sfn %s/%s %s' % (
self.project_info['target_releases'], self.release_version, current_link_tmp_dir)
result = waller.run(command, wenv=self.config())
# 2. make a soft link from release to tmp link
# 3. move tmp link to webroot
current_link_tmp_dir = '%s/current-tmp-%s' % (self.project_info['target_releases'], self.task_id)
command = 'mv -fT %s %s' % (current_link_tmp_dir, self.project_info['target_root'])
result = waller.run(command, wenv=self.config())
def release_untar(self, waller):
'''
解压版本包
:return:
'''
with waller.cd(self.project_info['target_releases']):
command = 'tar zxf %s' % (self.release_version_tar)
result = waller.run(command, wenv=self.config())
def post_release(self, waller):
'''
6.部署代码到目标机器后要做的任务
- 切换软链
- 重启 nginx
:return:
'''
self.stage = self.stage_post_release
self.sequence = 6
# 用户自定义命令
commands = self.project_info['post_release']
if commands:
for command in commands.split('\n'):
if command.strip().startswith('#') or not command.strip():
continue
# TODO
with waller.cd(self.project_info['target_root']):
pty = False if command.find('nohup') >= 0 else True
result = waller.run(command, wenv=self.config(), pty=pty)
# 个性化，用户重启的不一定是NGINX，可能是tomcat, apache, php-fpm等
# self.post_release_service(waller)
# 清理现场
self.cleanup_remote(waller)
def post_release_service(self, waller):
'''
代码部署完成后,服务启动工作,如: nginx重启
:param connection:
:return:
'''
with waller.cd(self.project_info['target_root']):
command = 'sudo service nginx restart'
result = waller.run(command, wenv=self.config())
def project_detection(self):
errors = []
# LOCAL_SERVER_USER => git
# LOCAL_SERVER_USER => target_servers
for server_info in self.servers:
waller = Waller(host=server_info['host'], user=server_info['user'], port=server_info['port'])
result = waller.run('id', exception=False, wenv=self.config())
if result.failed:
errors.append({
'title': '远程目标机器免密码登录失败',
'why': '远程目标机器：%s 错误：%s' % (server_info['host'], result.stdout),
'how': '在宿主机中配置免密码登录，把宿主机用户%s的~/.ssh/ssh_rsa.pub添加到远程目标机器用户%s的~/.ssh/authorized_keys。了解更多：http://walle-web.io/docs/troubleshooting.html' % (
pwd.getpwuid(os.getuid())[0], server_info['host']),
})
# maybe this is no webroot's parent dir
command = '[ -d {webroot} ] || mkdir -p {webroot}'.format(webroot=os.path.basename(self.project_info['target_root']))
result = waller.run(command, exception=False, wenv=self.config(console=False))
# 检查 webroot 父目录是否存在,是否为软链
command = '[ -L "%s" ] && echo "true" || echo "false"' % (self.project_info['target_root'])
result = waller.run(command, exception=False, wenv=self.config())
if result.stdout == 'false':
errors.append({
'title': '远程目标机器webroot不能是已建好的目录',
'why': '远程目标机器%s webroot不能是已存在的目录，必须为软链接，你不必新建，walle会自行创建。' % (server_info['host']),
'how': '手工删除远程目标机器：%s webroot目录：%s' % (server_info['host'], self.project_info['target_root']),
})
# remote release directory
return errors
def list_tag(self):
repo = Repo(self.dir_codebase_project)
repo.init(url=self.project_info['repo_url'])
return repo.tags()
def list_branch(self):
repo = Repo(self.dir_codebase_project)
repo.init(url=self.project_info['repo_url'])
return repo.branches()
def list_commit(self, branch):
repo = Repo(self.dir_codebase_project)
repo.init(url=self.project_info['repo_url'])
return repo.commits(branch)
def init_repo(self):
repo = Repo(self.dir_codebase_project)
repo.init(url=self.project_info['repo_url'])
# @todo 没有做emit
def cleanup_local(self):
# clean local package
command = 'rm -rf {project_id}_{task_id}_*'.format(project_id=self.project_info['id'], task_id=self.task_id)
with self.localhost.cd(self.local_codebase):
result = self.localhost.local(command, wenv=self.config())
def cleanup_remote(self, waller):
command = 'rm -rf {project_id}_{task_id}_*.tgz'.format(project_id=self.project_info['id'], task_id=self.task_id)
with waller.cd(self.project_info['target_releases']):
result = waller.run(command, wenv=self.config())
command = 'find ./ -name "{project_id}_*" -print | ls -t | tail -n +{keep_version_num} | xargs rm -rf'.format(
project_id=self.project_info['id'], keep_version_num=int(self.project_info['keep_version_num']) + 1)
with waller.cd(self.project_info['target_releases']):
result = waller.run(command, wenv=self.config())
def logs(self):
return RecordModel().fetch(task_id=self.task_id)
def end(self, success=True, update_status=True):
if update_status:
status = TaskModel.status_success if success else TaskModel.status_fail
current_app.logger.info('success:%s, status:%s' % (success, status))
TaskModel().get_by_id(self.task_id).update({
'status': status,
'link_id': self.release_version,
'ex_link_id': self.previous_release_version,
})
notice_info = {
'title': '',
'username': current_user.username,
'project_name': self.project_info['name'],
'task_name': '%s ([%s](%s))' % (self.taskMdl.get('name'), self.task_id, Notice.task_url(project_name=self.project_info['name'], task_id=self.task_id)),
'branch': self.taskMdl.get('branch'),
'commit': self.taskMdl.get('commit_id'),
'tag': self.taskMdl.get('tag'),
'repo_mode': self.project_info['repo_mode'],
}
notice = Notice.create(self.project_info['notice_type'])
if success:
notice_info['title'] = '上线部署成功'
notice.deploy_task(project_info=self.project_info, notice_info=notice_info)
else:
notice_info['title'] = '上线部署失败'
notice.deploy_task(project_info=self.project_info, notice_info=notice_info)
# 清理本地
self.cleanup_local()
if success:
emit('success', {'event': 'finish', 'data': {'message': '部署完成，辛苦了，为你的努力喝彩！'}}, room=self.task_id)
else:
emit('fail', {'event': 'finish', 'data': {'message': Code.code_msg[Code.deploy_fail]}}, room=self.task_id)
def walle_deploy(self):
self.start()
try:
self.prev_deploy()
self.deploy()
self.post_deploy()
is_all_servers_success = True
for server_info in self.servers:
host = server_info['host']
try:
waller = Waller(host=host, user=server_info['user'], port=server_info['port'], inline_ssh_env=True)
waller.init_env(env=self.custom_global_env)
self.connections[self.task_id] = waller
self.prev_release(self.connections[self.task_id])
self.release(self.connections[self.task_id])
self.post_release(self.connections[self.task_id])
RecordModel().save_record(stage=RecordModel.stage_end, sequence=0, user_id=current_user.id,
task_id=self.task_id, status=RecordModel.status_success, host=host,
user=server_info['user'], command='')
emit('success', {'event': 'finish', 'data': {'host': host, 'message': host + ' 部署完成！'}}, room=self.task_id)
except Exception as e:
is_all_servers_success = False
current_app.logger.exception(e)
self.errors[host] = e.message
RecordModel().save_record(stage=RecordModel.stage_end, sequence=0, user_id=current_user.id,
task_id=self.task_id, status=RecordModel.status_fail, host=host,
user=server_info['user'], command='')
emit('fail', {'event': 'finish', 'data': {'host': host, 'message': host + Code.code_msg[Code.deploy_fail]}}, room=self.task_id)
self.end(is_all_servers_success)
except Exception as e:
self.end(False)
return {'success': self.success, 'errors': self.errors}
def walle_rollback(self):
self.start()
try:
is_all_servers_success = True
for server_info in self.servers:
host = server_info['host']
try:
waller = Waller(host=host, user=server_info['user'], port=server_info['port'], inline_ssh_env=True)
waller.init_env(env=self.custom_global_env)
self.connections[self.task_id] = waller
self.prev_release_custom(self.connections[self.task_id])
self.release(self.connections[self.task_id])
self.post_release(self.connections[self.task_id])
RecordModel().save_record(stage=RecordModel.stage_end, sequence=0, user_id=current_user.id,
task_id=self.task_id, status=RecordModel.status_success, host=host,
user=server_info['user'], command='')
emit('success', {'event': 'finish', 'data': {'host': host, 'message': host + ' 部署完成！'}}, room=self.task_id)
except Exception as e:
is_all_servers_success = False
current_app.logger.exception(e)
self.errors[host] = e.message
RecordModel().save_record(stage=RecordModel.stage_end, sequence=0, user_id=current_user.id,
task_id=self.task_id, status=RecordModel.status_fail, host=host,
user=server_info['user'], command='')
emit('fail', {'event': 'finish', 'data': {'host': host, 'message': host + Code.code_msg[Code.deploy_fail]}}, room=self.task_id)
self.end(is_all_servers_success)
except Exception as e:
self.end(False)
return {'success': self.success, 'errors': self.errors}
class DeploySocketIO:
'''
序列号
'''
stage = '0'
sequence = 0
stage_prev_deploy = 'prev_deploy'
stage_deploy = 'deploy'
stage_post_deploy = 'post_deploy'
stage_prev_release = 'prev_release'
stage_release = 'release'
stage_post_release = 'post_release'
task_id = '0'
user_id = '0'
taskMdl = None
TaskRecord = None
version = datetime.now().strftime('%Y%m%d%H%M%s')
project_name = 'walden'
dir_codebase = '/tmp/walle/codebase/'
dir_codebase_project = dir_codebase + project_name
# 定义远程机器
# env.hosts = ['172.16.0.231', '172.16.0.177']
dir_release = None
dir_webroot = None
connections, success, errors = {}, {}, {}
release_version_tar, release_version = None, None
local, websocket = None, None
def __init__(self, task_id=None, project_id=None, websocket=None):
self.local = Waller(host=current_app.config.get('LOCAL_SERVER_HOST'),
user=current_app.config.get('LOCAL_SERVER_USER'),
port=current_app.config.get('LOCAL_SERVER_PORT'))
self.TaskRecord = TaskModel.TaskRecordModel()
current_app.logger.info('DeploySocketIO.__init__ before')
emit('message', {'msg': 'DeploySocketIO.__init__'})
current_app.logger.info('DeploySocketIO.__init__ after')
# if websocket:
# websocket.send_updates(__name__)
# self.websocket = websocket
if task_id:
self.task_id = task_id
self.taskMdl = TaskModel.TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.task = self.taskMdl.get('target_user')
self.project_info = self.taskMdl.get('project_info')
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
def config(self):
return {
'task_id': self.task_id,
'user_id': self.user_id,
'stage': self.stage,
'sequence': self.sequence,
'websocket': self.websocket
}
# ===================== fabric ================
# SocketHandler
def deploy(self):
'''
1.代码检出前要做的基础工作
- 检查 当前用户
- 检查 python 版本
- 检查 git 版本
- 检查 目录是否存在
- 用户自定义命令
:return:
'''
current_app.logger.info('deploy ing')
emit('message', {'msg': 'deploy ing'})
current_app.logger.info('deploy end')
self.stage = self.stage_prev_deploy
self.sequence = 1
# TODO remove
# result = self.local.run('sleep 30', wenv=self.config())
# 检查 当前用户
command = 'whoami'
current_app.logger.info(command)
emit('message', {'msg': command})
result = self.local.run(command, wenv=self.config())
# 检查 python 版本
command = 'python --version'
result = self.local.run(command, wenv=self.config())
current_app.logger.info(command)
# 检查 git 版本
command = 'git --version'
result = self.local.run(command, wenv=self.config())
current_app.logger.info(command)
# 检查 目录是否存在
command = 'mkdir -p %s' % (self.dir_codebase_project)
# TODO remove
current_app.logger.info(command)
result = self.local.run(command, wenv=self.config())
# 用户自定义命令
command = self.project_info['prev_deploy']
current_app.logger.info(command)
with self.local.cd(self.dir_codebase_project):
result = self.local.run(command, wenv=self.config())
class Deployer:
'''
序列号
'''
stage = '0'
sequence = 0
stage_prev_deploy = 'prev_deploy'
stage_deploy = 'deploy'
stage_post_deploy = 'post_deploy'
stage_prev_release = 'prev_release'
stage_release = 'release'
stage_post_release = 'post_release'
task_id = '0'
user_id = '0'
taskMdl = None
TaskRecord = None
version = datetime.now().strftime('%Y%m%d%H%M%s')
project_name = 'walden'
dir_codebase = '/tmp/walle/codebase/'
dir_codebase_project = dir_codebase + project_name
# 定义远程机器
# env.hosts = ['172.16.0.231', '172.16.0.177']
dir_release = None
dir_webroot = None
connections, success, errors = {}, {}, {}
release_version_tar, release_version = None, None
local, websocket = None, None
def __init__(self, task_id=None, project_id=None, websocket=None):
self.local = Waller(host=current_app.config.get('LOCAL_SERVER_HOST'),
user=current_app.config.get('LOCAL_SERVER_USER'),
port=current_app.config.get('LOCAL_SERVER_PORT'))
self.TaskRecord = TaskModel.TaskRecordModel()
if websocket:
websocket.send_updates(__name__)
self.websocket = websocket
if task_id:
self.task_id = task_id
self.taskMdl = TaskModel.TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.task = self.taskMdl.get('target_user')
self.project_info = self.taskMdl.get('project_info')
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
def config(self):
return {
'task_id': self.task_id,
'user_id': self.user_id,
'stage': self.stage,
'sequence': self.sequence,
'websocket': self.websocket
}
# ===================== fabric ================
# SocketHandler
def prev_deploy(self):
'''
1.代码检出前要做的基础工作
- 检查 当前用户
- 检查 python 版本
- 检查 git 版本
- 检查 目录是否存在
- 用户自定义命令
:return:
'''
self.stage = self.stage_prev_deploy
self.sequence = 1
# TODO remove
# result = self.local.run('sleep 30', wenv=self.config())
# 检查 当前用户
command = 'whoami'
self.websocket.send_updates(command)
current_app.logger.info(command)
result = self.local.run(command, wenv=self.config())
# 检查 python 版本
command = 'python --version'
result = self.local.run(command, wenv=self.config())
current_app.logger.info(command)
# 检查 git 版本
command = 'git --version'
result = self.local.run(command, wenv=self.config())
current_app.logger.info(command)
# 检查 目录是否存在
command = 'mkdir -p %s' % (self.dir_codebase_project)
# TODO remove
current_app.logger.info(command)
result = self.local.run(command, wenv=self.config())
# 用户自定义命令
command = self.project_info['prev_deploy']
current_app.logger.info(command)
with self.local.cd(self.dir_codebase_project):
result = self.local.run(command, wenv=self.config())
# SocketHandler.send_to_all({
# 'type': 'user',
# 'id': 33,
# 'host': env.host_string,
# 'command': command,
# 'message': result.stdout,
# })
def deploy(self):
'''
2.检出代码
:param project_name:
:return:
'''
self.stage = self.stage_deploy
self.sequence = 2
current_app.logger.info('git dir: %s',
self.dir_codebase_project + '/.git')
# 如果项目底下有 .git 目录则认为项目完整,可以直接检出代码
# TODO 不标准
if os.path.exists(self.dir_codebase_project + '/.git'):
with self.local.cd(self.dir_codebase_project):
command = 'pwd && git pull'
result = self.local.run(command, wenv=self.config())
else:
# 否则当作新项目检出完整代码
with self.local.cd(self.dir_codebase_project):
command = 'pwd && git clone %s .' % (
self.project_info['repo_url'])
current_app.logger.info('cd %s command: %s ',
self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
# copy to a local version
self.release_version = '%s_%s_%s' % (
self.project_name, self.task_id,
time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())))
with self.local.cd(self.dir_codebase):
command = 'cp -rf %s %s' % (self.dir_codebase_project,
self.release_version)
current_app.logger.info('cd %s command: %s ',
self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
# 更新到指定 commit_id
with self.local.cd(self.dir_codebase + self.release_version):
command = 'git reset -q --hard %s' % (
self.taskMdl.get('commit_id'))
result = self.local.run(command, wenv=self.config())
# SocketHandler.send_to_all({
# 'type': 'user',
# 'id': 33,
# 'host': env.host_string,
# 'command': command,
# 'message': result.stdout,
# })
# 用户自定义命令
# command = self.project_info['deploy']
# current_app.logger.info(command)
# with self.local.cd(self.dir_codebase):
# result = self.local.run(command)
pass
def post_deploy(self):
'''
3.检出代码后要做的任务
- 用户自定义操作命令
- 代码编译
- 清除日志文件及无用文件
-
- 压缩打包
- 传送到版本库 release
:return:
'''
self.stage = self.stage_post_deploy
self.sequence = 3
# 用户自定义命令
command = self.project_info['post_deploy']
current_app.logger.info(command)
with self.local.cd(self.dir_codebase + self.release_version):
result = self.local.run(command, wenv=self.config())
# 压缩打包
self.release_version_tar = '%s.tgz' % (self.release_version)
with self.local.cd(self.dir_codebase):
command = 'tar zcvf %s %s' % (self.release_version_tar,
self.release_version)
result = self.local.run(command, wenv=self.config())
def prev_release(self, waller):
'''
4.部署代码到目标机器前做的任务
- 检查 webroot 父目录是否存在
:return:
'''
self.stage = self.stage_prev_release
self.sequence = 4
# 检查 target_releases 父目录是否存在
command = 'mkdir -p %s' % (self.project_info['target_releases'])
result = waller.run(command, wenv=self.config())
# TODO 检查 webroot 父目录是否存在,是否为软链
# command = 'mkdir -p %s' % (self.project_info['target_root'])
# result = waller.run(command)
# current_app.logger.info('command: %s', dir(result))
# TODO md5
# 传送到版本库 release
current_app.logger.info('/tmp/walle/codebase/' +
self.release_version_tar)
result = waller.put('/tmp/walle/codebase/' + self.release_version_tar,
remote=self.project_info['target_releases'])
current_app.logger.info('command: %s', dir(result))
# 解压
self.release_untar(waller)
def release(self, waller):
'''
5.部署代码到目标机器做的任务
- 打包代码 local
- scp local => remote
- 解压 remote
:return:
'''
self.stage = self.stage_release
self.sequence = 5
with waller.cd(self.project_info['target_releases']):
# 1. create a tmp link dir
current_link_tmp_dir = '%s/current-tmp-%s' % (
self.project_info['target_releases'], self.task_id)
command = 'ln -sfn %s/%s %s' % (
self.project_info['target_releases'], self.release_version,
current_link_tmp_dir)
result = waller.run(command, wenv=self.config())
# 2. make a soft link from release to tmp link
# 3. move tmp link to webroot
current_link_tmp_dir = '%s/current-tmp-%s' % (
self.project_info['target_releases'], self.task_id)
command = 'mv -fT %s %s' % (current_link_tmp_dir,
self.project_info['target_root'])
result = waller.run(command, wenv=self.config())
def release_untar(self, waller):
'''
解压版本包
:return:
'''
with waller.cd(self.project_info['target_releases']):
command = 'tar zxf %s' % (self.release_version_tar)
result = waller.run(command, wenv=self.config())
def post_release(self, waller):
'''
6.部署代码到目标机器后要做的任务
- 切换软链
- 重启 nginx
:return:
'''
self.stage = self.stage_post_release
self.sequence = 6
self.post_release_service(waller)
def post_release_service(self, waller):
'''
代码部署完成后,服务启动工作,如: nginx重启
:param connection:
:return:
'''
with waller.cd(self.project_info['target_root']):
command = 'sudo service nginx restart'
result = waller.run(command, wenv=self.config())
def list_tag(self):
with self.local.cd(self.dir_codebase_project):
command = 'git tag -l'
current_app.logger.info('cd %s command: %s ',
self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
current_app.logger.info(dir(result))
return result
return None
def list_branch(self):
with self.local.cd(self.dir_codebase_project):
command = 'git pull'
# result = self.local.run(command, wenv=self.config())
current_app.logger.info(self.dir_codebase_project)
command = 'git branch -r'
result = self.local.run(command, wenv=self.config())
# TODO 三种可能: false, error, success
branches = result.stdout.strip().split('\n')
# 去除 origin/HEAD -> 当前指向
# 去除远端前缀
branches = [
branch.strip().lstrip('origin/') for branch in branches
if not branch.startswith('origin/HEAD')
]
return branches
return None
def list_commit(self, branch):
with self.local.cd(self.dir_codebase_project):
command = 'git checkout %s && git pull' % (branch)
result = self.local.run(command, wenv=self.config())
# TODO 10是需要前端传的
command = 'git log -10 --pretty="%h #_# %an #_# %s"'
result = self.local.run(command, wenv=self.config())
commit_list = result.stdout.strip().split('\n')
commits = []
for commit in commit_list:
commit_dict = commit.split(' #_# ')
commits.append({
'id': commit_dict[0],
'name': commit_dict[1],
'message': commit_dict[2],
})
return commits
return None
def walle_deploy(self):
self.prev_deploy()
self.deploy()
self.post_deploy()
server = '172.16.0.231'
try:
self.connections[server] = Waller(
host=server, user=self.project_info['target_user'])
self.prev_release(self.connections[server])
self.release(self.connections[server])
self.post_release(self.connections[server])
except Exception, e:
current_app.logger.exception(e)
self.errors[server] = e.message
# for server_info in self.servers:
# server = server_info.host
# try:
# self.connections[server] = Waller(host=server, user=self.project_info['target_user'])
# self.prev_release(self.connections[server])
# self.release(self.connections[server])
# self.post_release(self.connections[server])
# except Exception, e:
# self.errors[server] = e.message
return {'success': self.success, 'errors': self.errors}
class Deployer:
'''
序列号
'''
stage = '0'
sequence = 0
stage_prev_deploy = 'prev_deploy'
stage_deploy = 'deploy'
stage_post_deploy = 'post_deploy'
stage_prev_release = 'prev_release'
stage_release = 'release'
stage_post_release = 'post_release'
task_id = '0'
user_id = '0'
taskMdl = None
TaskRecord = None
console = False
version = datetime.now().strftime('%Y%m%d%H%M%s')
project_name = None
dir_codebase = '/tmp/walle/codebase/'
dir_codebase_project = ''
# 定义远程机器
# env.hosts = ['172.16.0.231', '172.16.0.177']
dir_release = None
dir_webroot = None
connections, success, errors = {}, {}, {}
release_version_tar, release_version = None, None
local = None
def __init__(self, task_id=None, project_id=None, console=False):
self.local = Waller(host=current_app.config.get('LOCAL_SERVER_HOST'),
user=current_app.config.get('LOCAL_SERVER_USER'),
port=current_app.config.get('LOCAL_SERVER_PORT'))
self.TaskRecord = RecordModel()
if task_id:
self.task_id = task_id
# task start
TaskModel(id=self.task_id).update(status=TaskModel.status_doing)
self.taskMdl = TaskModel().item(self.task_id)
self.user_id = self.taskMdl.get('user_id')
self.servers = self.taskMdl.get('servers_info')
self.task = self.taskMdl.get('target_user')
self.project_info = self.taskMdl.get('project_info')
if project_id:
self.project_id = project_id
self.project_info = ProjectModel(id=project_id).item()
self.project_name = self.project_info['id']
self.dir_codebase_project = self.dir_codebase + str(self.project_name)
# start to deploy
def config(self):
return {'task_id': self.task_id, 'user_id': self.user_id, 'stage': self.stage, 'sequence': self.sequence, 'console': self.console}
# ===================== fabric ================
# SocketHandler
def prev_deploy(self):
'''
# TODO
socketio.sleep(0.001)
1.代码检出前要做的基础工作
- 检查 当前用户
- 检查 python 版本
- 检查 git 版本
- 检查 目录是否存在
- 用户自定义命令
:return:
'''
self.stage = self.stage_prev_deploy
self.sequence = 1
# TODO remove
# result = self.local.run('sleep 30', wenv=self.config())
# 检查 当前用户
command = 'whoami'
current_app.logger.info(command)
result = self.local.run(command, wenv=self.config())
# 检查 python 版本
command = 'python --version'
result = self.local.run(command, wenv=self.config())
current_app.logger.info(command)
# 检查 git 版本
command = 'git --version'
result = self.local.run(command, wenv=self.config())
current_app.logger.info(command)
# 检查 目录是否存在
self.init_repo()
# TODO to be removed
command = 'mkdir -p %s' % (self.dir_codebase_project)
# TODO remove
current_app.logger.info(command)
result = self.local.run(command, wenv=self.config())
# 用户自定义命令
command = self.project_info['prev_deploy']
current_app.logger.info(command)
with self.local.cd(self.dir_codebase_project):
result = self.local.run(command, wenv=self.config())
# SocketHandler.send_to_all({
# 'type': 'user',
# 'id': 33,
# 'host': env.host_string,
# 'command': command,
# 'message': result.stdout,
# })
def deploy(self):
'''
2.检出代码
:param project_name:
:return:
'''
# TODO
socketio.sleep(0.001)
self.stage = self.stage_deploy
self.sequence = 2
current_app.logger.info('git dir: %s', self.dir_codebase_project + '/.git')
# 如果项目底下有 .git 目录则认为项目完整,可以直接检出代码
# TODO 不标准
if os.path.exists(self.dir_codebase_project + '/.git'):
with self.local.cd(self.dir_codebase_project):
command = 'pwd && git pull'
result = self.local.run(command, wenv=self.config())
else:
# 否则当作新项目检出完整代码
with self.local.cd(self.dir_codebase_project):
command = 'pwd && git clone %s .' % (self.project_info['repo_url'])
current_app.logger.info('cd %s command: %s ', self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
# copy to a local version
self.release_version = '%s_%s_%s' % (
self.project_name, self.task_id, time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())))
with self.local.cd(self.dir_codebase):
command = 'cp -rf %s %s' % (self.dir_codebase_project, self.release_version)
current_app.logger.info('cd %s command: %s ', self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
# 更新到指定 commit_id
with self.local.cd(self.dir_codebase + self.release_version):
command = 'git reset -q --hard %s' % (self.taskMdl.get('commit_id'))
result = self.local.run(command, wenv=self.config())
# SocketHandler.send_to_all({
# 'type': 'user',
# 'id': 33,
# 'host': env.host_string,
# 'command': command,
# 'message': result.stdout,
# })
# 用户自定义命令
# command = self.project_info['deploy']
# current_app.logger.info(command)
# with self.local.cd(self.dir_codebase):
# result = self.local.run(command)
pass
def post_deploy(self):
'''
3.检出代码后要做的任务
- 用户自定义操作命令
- 代码编译
- 清除日志文件及无用文件
-
- 压缩打包
- 传送到版本库 release
:return:
'''
# TODO
socketio.sleep(0.001)
self.stage = self.stage_post_deploy
self.sequence = 3
# 用户自定义命令
command = self.project_info['post_deploy']
current_app.logger.info(command)
current_app.logger.info(self.dir_codebase)
current_app.logger.info(self.release_version)
with self.local.cd(self.dir_codebase + self.release_version):
result = self.local.run(command, wenv=self.config())
# 压缩打包
self.release_version_tar = '%s.tgz' % (self.release_version)
with self.local.cd(self.dir_codebase):
command = 'tar zcvf %s %s' % (self.release_version_tar, self.release_version)
result = self.local.run(command, wenv=self.config())
def prev_release(self, waller):
'''
4.部署代码到目标机器前做的任务
- 检查 webroot 父目录是否存在
:return:
'''
# TODO
socketio.sleep(0.001)
self.stage = self.stage_prev_release
self.sequence = 4
# 检查 target_releases 父目录是否存在
command = 'mkdir -p %s' % (self.project_info['target_releases'])
result = waller.run(command, wenv=self.config())
# TODO 检查 webroot 父目录是否存在,是否为软链
# command = 'mkdir -p %s' % (self.project_info['target_root'])
# result = waller.run(command)
# current_app.logger.info('command: %s', dir(result))
# TODO md5
# 传送到版本库 release
current_app.logger.info('/tmp/walle/codebase/' + self.release_version_tar)
result = waller.put('/tmp/walle/codebase/' + self.release_version_tar,
remote=self.project_info['target_releases'], wenv=self.config())
current_app.logger.info('command: %s', dir(result))
# 解压
self.release_untar(waller)
def release(self, waller):
'''
5.部署代码到目标机器做的任务
- 打包代码 local
- scp local => remote
- 解压 remote
:return:
'''
# TODO
socketio.sleep(0.001)
self.stage = self.stage_release
self.sequence = 5
with waller.cd(self.project_info['target_releases']):
# 1. create a tmp link dir
current_link_tmp_dir = '%s/current-tmp-%s' % (self.project_info['target_releases'], self.task_id)
command = 'ln -sfn %s/%s %s' % (
self.project_info['target_releases'], self.release_version, current_link_tmp_dir)
result = waller.run(command, wenv=self.config())
# 2. make a soft link from release to tmp link
# 3. move tmp link to webroot
current_link_tmp_dir = '%s/current-tmp-%s' % (self.project_info['target_releases'], self.task_id)
command = 'mv -fT %s %s' % (current_link_tmp_dir, self.project_info['target_root'])
result = waller.run(command, wenv=self.config())
def release_untar(self, waller):
'''
解压版本包
:return:
'''
# TODO
socketio.sleep(0.001)
with waller.cd(self.project_info['target_releases']):
command = 'tar zxf %s' % (self.release_version_tar)
result = waller.run(command, wenv=self.config())
def post_release(self, waller):
'''
6.部署代码到目标机器后要做的任务
- 切换软链
- 重启 nginx
:return:
'''
self.stage = self.stage_post_release
self.sequence = 6
self.post_release_service(waller)
def post_release_service(self, waller):
'''
代码部署完成后,服务启动工作,如: nginx重启
:param connection:
:return:
'''
with waller.cd(self.project_info['target_root']):
command = 'sudo service nginx restart'
result = waller.run(command, wenv=self.config())
def list_tag(self):
self.init_repo()
with self.local.cd(self.dir_codebase_project):
command = 'git tag -l'
current_app.logger.info('cd %s command: %s ', self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
current_app.logger.info(dir(result))
return result
return None
def list_branch(self):
self.init_repo()
with self.local.cd(self.dir_codebase_project):
command = 'git pull'
result = self.local.run(command, wenv=self.config())
current_app.logger.info(self.dir_codebase_project)
command = 'git branch -r'
result = self.local.run(command, wenv=self.config())
# TODO 三种可能: false, error, success
branches = result.stdout.strip().split('\n')
# 去除 origin/HEAD -> 当前指向
# 去除远端前缀
branches = [branch.strip().lstrip('origin/') for branch in branches if not branch.startswith('origin/HEAD')]
return branches
return None
def list_commit(self, branch):
self.init_repo()
with self.local.cd(self.dir_codebase_project):
command = 'git checkout %s && git pull' % (branch)
result = self.local.run(command, wenv=self.config())
# TODO 10是需要前端传的
command = 'git log -10 --pretty="%h #_# %an #_# %s"'
result = self.local.run(command, wenv=self.config())
commit_list = result.stdout.strip().split('\n')
commits = []
for commit in commit_list:
commit_dict = commit.split(' #_# ')
commits.append({
'id': commit_dict[0],
'name': commit_dict[1],
'message': commit_dict[2],
})
return commits
return None
def init_repo(self):
current_app.logger.info('git dir: %s', self.dir_codebase_project + '/.git')
# 如果项目底下有 .git 目录则认为项目完整,可以直接检出代码
# TODO 不标准
if not os.path.exists(self.dir_codebase_project):
# 检查 目录是否存在
command = 'mkdir -p %s' % (self.dir_codebase_project)
# TODO remove
current_app.logger.info(command)
result = self.local.run(command, wenv=self.config())
if not os.path.exists(self.dir_codebase_project + '/.git'):
# 否则当作新项目检出完整代码
with self.local.cd(self.dir_codebase_project):
command = 'pwd && git clone %s .' % (self.project_info['repo_url'])
current_app.logger.info('cd %s command: %s ', self.dir_codebase_project, command)
result = self.local.run(command, wenv=self.config())
def walle_deploy(self):
self.prev_deploy()
self.deploy()
self.post_deploy()
server = '172.16.0.231'
try:
self.connections[server] = Waller(host=server, user=self.project_info['target_user'])
self.prev_release(self.connections[server])
self.release(self.connections[server])
self.post_release(self.connections[server])
except Exception as e:
current_app.logger.exception(e)
self.errors[server] = e.message
# for server_info in self.servers:
# server = server_info.host
# try:
# self.connections[server] = Waller(host=server, user=self.project_info['target_user'])
# self.prev_release(self.connections[server])
# self.release(self.connections[server])
# self.post_release(self.connections[server])
# except Exception, e:
# self.errors[server] = e.message
return {'success': self.success, 'errors': self.errors}