def checkout(self, branch=None, version=None, tag=None):
logger.debug("checkout:")
if branch is None or version is None:
LocalShell.check_call(
"cd {0} && git checkout {1}".format(self.dest, tag),
shell=True)
else:
LocalShell.check_call(
"cd {0} && git checkout -B {1} -t origin/{1} && git pull -q origin {1} && git reset --hard {2}".format(
self.dest, branch, version),
shell=True)
def checkout_branch(self, branch, version=""):
logger.debug("checkout branch:")
if branch in self.local_branch():
LocalShell.check_call(
"cd {0} && git checkout -q {1} && git pull -q origin {1} && git reset --hard {2}".format(
self.dest, branch, version),
shell=True)
else:
LocalShell.check_call(
"cd {0} && git checkout -q -b {1} -t origin/{1} && git pull -q origin {1} && git reset --hard {2}".format(
self.dest, branch, version),
shell=True)
def checkout_branch(self, branch, version=""):
logger.debug("checkout branch:")
if branch in self.local_branch():
LocalShell.check_call("cd {0} && git checkout -q {1} && git pull "
"-q origin {1} && git reset --hard {2}"
.format(self.dest, branch, version),
shell=True)
else:
LocalShell.check_call("cd {0} && git checkout -q -b {1} -t "
"origin/{1} && git pull -q origin {1} && "
"git reset --hard {2}"
.format(self.dest, branch, version),
shell=True)
def clone(self):
logger.debug("clone repo:")
shell = ("mkdir -p {0} && cd {0} && git clone -q {1} .").format(self.dest, self.url)
rc = LocalShell.call(shell, shell=True)
# destination path '.' already exists and is not an empty directory.
if rc == 128:
shell = ("cd {0} && git clean -xdfq && git reset -q --hard && git remote update && git checkout -q master && git remote prune origin && git pull -q --all && git branch | grep -v \\* | xargs git branch -D").format(self.dest)
rc = LocalShell.call(shell, shell=True)
# branch name required
if rc == 123:
return
if rc != 0:
raise Error(12000)
def tag(self):
shell = "cd {0} && git fetch -q -a && git tag".format(self.dest)
stdout = LocalShell.check_output(shell, shell=True)
if stdout:
return stdout.strip().split("\n")
else:
return []
def remote_branch(self):
shell = "cd {0} && git fetch -q -a && git branch -r".format(self.dest)
stdout = LocalShell.check_output(shell, shell=True)
stdout = stdout.strip().split("\n")
stdout = [s.strip(" ").split("/", 1)[1] for s in stdout if "->" not in
s]
return stdout
def tag(self):
shell = "cd {0} && git fetch -q -a && git tag".format(self.dest)
stdout = LocalShell.check_output(shell, shell=True)
if stdout:
return stdout.strip().split("\n")
else:
return []
def clone(self):
logger.debug("clone repo:")
shell = ("mkdir -p {0} && cd {0} && git clone -q {1} .").format(
self.dest, self.url)
rc = LocalShell.call(shell, shell=True)
# destination path '.' already exists and is not an empty directory.
if rc == 128:
shell = (
"cd {0} && git clean -xdfq && git reset -q --hard && git remote update && git checkout -q master && git remote prune origin && git pull -q --all && git branch | grep -v \\* | xargs git branch -D"
).format(self.dest)
rc = LocalShell.call(shell, shell=True)
# branch name required
if rc == 123:
return
if rc != 0:
raise Error(12000)
def log(self):
shell = ("cd {0} && git log -20 --pretty=\"%h %an %s\"").format(self.dest)
stdout = LocalShell.check_output(shell, shell=True)
stdout = stdout.strip().split("\n")
stdout = [s.split(" ", 2) for s in stdout]
return [{"abbreviated_commit": s[0],
"author_name": s[1],
"subject": s[2]}
for s in stdout]
def log(self):
shell = ("cd {0} && git log -20 --pretty=\"%h %an %s\""
).format(self.dest)
stdout = LocalShell.check_output(shell, shell=True)
stdout = stdout.strip().split("\n")
stdout = [s.split(" ", 2) for s in stdout]
return [{"abbreviated_commit": s[0],
"author_name": s[1],
"subject": s[2]}
for s in stdout]
def build_thread(deploy):
# before checkout
git = Git(deploy.project.checkout_dir, deploy.project.repo_url)
before_checkout = deploy.project.before_checkout.replace("\r", "").replace("\n", " && ")
logger.debug("before_checkout"+before_checkout)
if before_checkout:
LocalShell.check_call(
"WORKSPACE='{0}' && mkdir -p $WORKSPACE && cd $WORKSPACE && {1}".format(
deploy.project.checkout_dir, before_checkout),
shell=True)
# checkout
git.clone()
if deploy.mode == 0:
git.checkout_branch(deploy.branch, deploy.version)
else:
git.checkout_tag(deploy.version)
# after checkout
after_checkout = deploy.project.after_checkout.replace("\r", "").replace("\n", " && ")
if after_checkout:
LocalShell.check_call(
"WORKSPACE='{0}' && cd $WORKSPACE && {1}".format(
deploy.project.checkout_dir, after_checkout),
shell=True)
def deploy_thread(project_id):
deploys = DeploysService()
deploy = deploys.first(project_id=project_id, status=3)
if not deploy:
logger.info("no deploy wait in quene.")
return
logger.info("deploy thread start: {}".format(deploy.id))
ssh = RemoteShell(host=deploy.host.ssh_host,
port=deploy.host.ssh_port,
user=deploy.host.ssh_user,
passwd=deploy.host.ssh_pass)
try:
deploys.update(deploy, progress=0, status=2)
# before checkout
git = Git(deploy.project.checkout_dir, deploy.project.repo_url)
before_checkout = deploy.project.before_checkout.replace(
"\r", "").replace("\n", " && ")
logger.debug("before_checkout" + before_checkout)
deploys.append_comment(deploy, "before checkout:\n")
cmd = "mkdir -p {0} && rm -rf {1}/*".format(deploy.project.target_dir,
deploy.project.target_dir)
LocalShell.check_call(cmd, shell=True)
if before_checkout:
cmd = "WORKSPACE='{0}' && cd $WORKSPACE && {1}".format(
deploy.project.checkout_dir, before_checkout)
LocalShell.check_call(cmd, shell=True)
deploys.append_comment(deploy, "OK!\n")
deploys.update(deploy, progress=17)
# checkout
deploys.append_comment(deploy, "checkout:\n")
git.clone()
if deploy.mode == 0:
git.checkout_branch(deploy.branch, deploy.version)
else:
git.checkout_tag(deploy.version)
deploys.append_comment(deploy, "OK!\n")
deploys.update(deploy, progress=33)
# after checkout
after_checkout = deploy.project.after_checkout.replace(
"\r", "").replace("\n", " && ")
deploys.append_comment(deploy, "after checkout:\n")
if after_checkout:
cmd = "WORKSPACE='{0}' && cd $WORKSPACE && {1}".format(
deploy.project.checkout_dir, after_checkout)
LocalShell.check_call(cmd, shell=True)
deploys.append_comment(deploy, "OK!\n")
deploys.update(deploy, progress=50)
# before deploy
deploys.append_comment(deploy, "before deploy:\n")
ssh.check_call("mkdir -p {0}".format(
os.path.join(deploy.project.deploy_history_dir,
deploy.softln_filename)))
logger.debug("before deploy:")
ssh.check_call(
("WORKSPACE='{0}' && cd $WORKSPACE && ls -1t | tail -n +{1} | "
"xargs rm -rf").format(deploy.project.deploy_history_dir,
config.MAX_DEPLOY_HISTORY))
before_deploy = deploy.project.before_deploy.replace("\r", "").replace(
"\n", " && ")
if before_deploy:
ssh.check_call("WORKSPACE='{0}' && cd $WORKSPACE && {1}".format(
deploy.project.deploy_dir, before_deploy))
deploys.append_comment(deploy, "OK!\n")
deploys.update(deploy, progress=67)
# deploy
deploys.append_comment(deploy, "deploy:\n")
logger.debug("deploy:")
logger.debug("rsync:")
if deploy.host.ssh_method == 0:
cmd = ("rsync -avzq "
"--rsh=\"sshpass -p {ssh_pass} ssh -p {ssh_port}\" "
"--exclude='.git' {local_dest}/ {ssh_user}@{ssh_host}:"
"{remote_dest}/").format(
local_dest=deploy.project.target_dir,
remote_dest=os.path.join(
deploy.project.deploy_history_dir,
deploy.softln_filename),
ssh_user=deploy.host.ssh_user,
ssh_host=deploy.host.ssh_host,
ssh_port=deploy.host.ssh_port,
ssh_pass=deploy.host.ssh_pass)
else:
cmd = ("rsync -avzq --exclude='.git' {local_dest}/ "
"{ssh_user}@{ssh_host}:{remote_dest}/").format(
local_dest=deploy.project.target_dir,
remote_dest=os.path.join(
deploy.project.deploy_history_dir,
deploy.softln_filename),
ssh_user=deploy.host.ssh_user,
ssh_host=deploy.host.ssh_host,
ssh_port=deploy.host.ssh_port,
ssh_pass=deploy.host.ssh_pass)
LocalShell.check_call(cmd, shell=True)
ssh.check_call("ln -snf {0} {1}".format(
os.path.join(deploy.project.deploy_history_dir,
deploy.softln_filename), deploy.project.deploy_dir))
deploys.append_comment(deploy, "OK!\n")
deploys.update(deploy, progress=83)
# after deploy
deploys.append_comment(deploy, "after deploy:\n")
logger.debug("after deploy:")
after_deploy = deploy.project.after_deploy.replace("\r", "").replace(
"\n", " && ")
if after_deploy:
ssh.check_call("WORKSPACE='{0}' && cd $WORKSPACE && {1}".format(
deploy.project.deploy_dir, after_deploy))
deploys.append_comment(deploy, "OK!\n")
except Exception as err:
traceback.print_exc()
logger.error(err)
deploys.append_comment(deploy, repr(err))
deploys.update(deploy, status=0)
else:
deploys.update(deploy, progress=100, status=1)
finally:
logger.info("deploy thread end: %d" % deploy.id)
ssh.close()
deploy = deploys.first(project_id=project_id, status=3)
if deploy:
logger.info("deploy thread fetch from wait: {}".format(deploy))
deploys.deploy(deploy)
def local_branch(self):
shell = "cd {0} && git fetch -q -a && git branch".format(self.dest)
stdout = LocalShell.check_output(shell, shell=True)
stdout = stdout.strip().split("\n")
stdout = [s.strip("* ") for s in stdout]
return stdout
def checkout_tag(self, tag):
logger.debug("checkout to tag: %s" % tag)
LocalShell.check_call(
"cd {0} && git checkout -q {1}".format(self.dest, tag),
shell=True)
def checkout(self, branch, version):
logger.debug("checkout:")
LocalShell.check_call(
"cd {0} && git checkout -B {1} -t origin/{1} && git pull -q origin {1} && git reset --hard {2}"
.format(self.dest, branch, version),
shell=True)
def checkout_tag(self, tag):
logger.debug("checkout to tag: %s" % tag)
LocalShell.check_call(
"cd {0} && git checkout -q {1}".format(self.dest, tag),
shell=True)
def local_branch(self):
shell = "cd {0} && git fetch -q -a && git branch".format(self.dest)
stdout = LocalShell.check_output(shell, shell=True)
stdout = stdout.strip().split("\n")
stdout = [s.strip("* ") for s in stdout]
return stdout
def deploy_thread(service, deploy):
ssh = RemoteShell(host=deploy.host.ssh_host,
port=deploy.host.ssh_port,
user=deploy.host.ssh_user,
passwd=deploy.host.ssh_pass)
try:
service.update(deploy, progress=0, status=2)
# before checkout
git = Git(deploy.project.checkout_dir, deploy.project.repo_url)
before_checkout = deploy.project.before_checkout.replace("\r", "").replace("\n", " && ")
logger.debug("before_checkout"+before_checkout)
if before_checkout:
LocalShell.check_call(
"WORKSPACE='{0}' && mkdir -p $WORKSPACE && cd $WORKSPACE && {1}".format(
deploy.project.checkout_dir, before_checkout),
shell=True)
service.update(deploy, progress=17)
# checkout
git.clone()
if deploy.mode == 0:
git.checkout(deploy.branch, deploy.version)
else:
git.checkout(tag=deploy.version)
service.update(deploy, progress=33)
# after checkout
after_checkout = deploy.project.after_checkout.replace("\r", "").replace("\n", " && ")
if after_checkout:
LocalShell.check_call(
"WORKSPACE='{0}' && cd $WORKSPACE && {1}".format(
deploy.project.checkout_dir, after_checkout),
shell=True)
service.update(deploy, progress=50)
# before deploy
rc, stdout, stderr = ssh.exec_command(
"mkdir -p {0}".format(
os.path.join(deploy.project.deploy_history_dir, deploy.softln_filename)))
if rc:
raise Error(11003)
logger.debug("before deploy:")
rc, stdout, stderr = ssh.exec_command(
"WORKSPACE='{0}' && cd $WORKSPACE && ls -1t | tail -n +20 | xargs rm -rf".format(
deploy.project.deploy_history_dir))
if rc:
raise Error(11000)
before_deploy = deploy.project.before_deploy.replace("\r", "").replace("\n", " && ")
if before_deploy:
rc, stdout, stderr = ssh.exec_command(
"WORKSPACE='{0}' && cd $WORKSPACE && {1}".format(
deploy.project.deploy_dir, before_deploy))
if rc:
raise Error(11000)
service.update(deploy, progress=67)
# deploy
logger.debug("deploy:")
logger.debug("rsync:")
shell = ("rsync -avzq --rsh=\"sshpass -p {ssh_pass} ssh -p {ssh_port}\" --exclude='.git' {local_dest}/ {ssh_user}@{ssh_host}:{remote_dest}/").format(
local_dest=deploy.project.checkout_dir,
remote_dest=os.path.join(deploy.project.deploy_history_dir, deploy.softln_filename),
ssh_user=deploy.host.ssh_user,
ssh_host=deploy.host.ssh_host,
ssh_port=deploy.host.ssh_port,
ssh_pass=deploy.host.ssh_pass)
LocalShell.check_call(shell, shell=True)
rc,stdout, stderr = ssh.exec_command("ln -snf {0} {1}".format(
os.path.join(deploy.project.deploy_history_dir, deploy.softln_filename), deploy.project.deploy_dir))
if rc:
raise Error(11001)
service.update(deploy, progress=83)
# after deploy
logger.debug("after deploy:")
after_deploy = deploy.project.after_deploy.replace("\r", "").replace("\n", " && ")
if after_deploy:
rc, stdout, stderr = ssh.exec_command(
"WORKSPACE='{0}' && cd $WORKSPACE && {1}".format(
deploy.project.deploy_dir, after_deploy))
if rc:
raise Error(11002)
except Exception:
service.update(deploy, status=0)
else:
service.update(deploy, progress=100, status=1)
finally:
ssh.close()
def deploy_thread(service, deploy):
ssh = RemoteShell(host=deploy.host.ssh_host,
port=deploy.host.ssh_port,
user=deploy.host.ssh_user,
passwd=deploy.host.ssh_pass)
try:
service.update(deploy, progress=0, status=2)
# before checkout
git = Git(deploy.project.checkout_dir, deploy.project.repo_url)
before_checkout = deploy.project.before_checkout.replace("\r", "").replace("\n", " && ")
logger.debug("before_checkout"+before_checkout)
service.append_comment(deploy, "before checkout:\n")
if before_checkout:
cmd = "WORKSPACE='{0}' && mkdir -p $WORKSPACE && cd $WORKSPACE && {1}".format(
deploy.project.checkout_dir, before_checkout)
LocalShell.check_call(cmd, shell=True)
service.append_comment(deploy, "OK!\n")
service.update(deploy, progress=17)
# checkout
service.append_comment(deploy, "checkout:\n")
git.clone()
if deploy.mode == 0:
git.checkout_branch(deploy.branch, deploy.version)
else:
git.checkout_tag(deploy.version)
service.append_comment(deploy, "OK!\n")
service.update(deploy, progress=33)
# after checkout
after_checkout = deploy.project.after_checkout.replace("\r", "").replace("\n", " && ")
service.append_comment(deploy, "after checkout:\n")
if after_checkout:
cmd = "WORKSPACE='{0}' && cd $WORKSPACE && {1}".format(
deploy.project.checkout_dir, after_checkout)
LocalShell.check_call(cmd, shell=True)
service.append_comment(deploy, "OK!\n")
service.update(deploy, progress=50)
# before deploy
service.append_comment(deploy, "before deploy:\n")
ssh.check_call(
"mkdir -p {0}".format(
os.path.join(deploy.project.deploy_history_dir, deploy.softln_filename)))
logger.debug("before deploy:")
ssh.check_call(
"WORKSPACE='{0}' && cd $WORKSPACE && ls -1t | tail -n +{1} | xargs rm -rf".format(
deploy.project.deploy_history_dir, config.MAX_DEPLOY_HISTORY))
before_deploy = deploy.project.before_deploy.replace("\r", "").replace("\n", " && ")
if before_deploy:
ssh.check_call(
"WORKSPACE='{0}' && cd $WORKSPACE && {1}".format(
deploy.project.deploy_dir, before_deploy))
service.append_comment(deploy, "OK!\n")
service.update(deploy, progress=67)
# deploy
service.append_comment(deploy, "deploy:\n")
logger.debug("deploy:")
logger.debug("rsync:")
cmd = ("rsync -avzq --rsh=\"sshpass -p {ssh_pass} ssh -p {ssh_port}\" --exclude='.git' {local_dest}/ {ssh_user}@{ssh_host}:{remote_dest}/").format(
local_dest=deploy.project.checkout_dir,
remote_dest=os.path.join(deploy.project.deploy_history_dir, deploy.softln_filename),
ssh_user=deploy.host.ssh_user,
ssh_host=deploy.host.ssh_host,
ssh_port=deploy.host.ssh_port,
ssh_pass=deploy.host.ssh_pass)
LocalShell.check_call(cmd, shell=True)
ssh.check_call("ln -snf {0} {1}".format(
os.path.join(deploy.project.deploy_history_dir, deploy.softln_filename), deploy.project.deploy_dir))
service.append_comment(deploy, "OK!\n")
service.update(deploy, progress=83)
# after deploy
service.append_comment(deploy, "after deploy:\n")
logger.debug("after deploy:")
after_deploy = deploy.project.after_deploy.replace("\r", "").replace("\n", " && ")
if after_deploy:
ssh.check_call(
"WORKSPACE='{0}' && cd $WORKSPACE && {1}".format(
deploy.project.deploy_dir, after_deploy))
service.append_comment(deploy, "OK!\n")
except Exception as err:
logger.error(err)
service.append_comment(deploy, "Command: "+err.cmd+"\nReturn code: "+str(err.returncode)+"\nOutput: "+err.output)
service.update(deploy, status=0)
else:
service.update(deploy, progress=100, status=1)
finally:
ssh.close()
def deploy_thread(project_id):
deploys = DeploysService()
deploy = deploys.first(project_id=project_id, status=3)
if not deploy:
logger.info("no deploy wait in quene.")
return
logger.info("deploy thread start: {}".format(deploy.id))
ssh = RemoteShell(host=deploy.host.ssh_host,
port=deploy.host.ssh_port,
user=deploy.host.ssh_user,
passwd=deploy.host.ssh_pass)
try:
deploys.update(deploy, progress=0, status=2)
# before checkout
git = Git(deploy.project.checkout_dir, deploy.project.repo_url)
before_checkout = deploy.project.before_checkout.replace(
"\r", "").replace("\n", " && ")
logger.debug("before_checkout"+before_checkout)
deploys.append_comment(deploy, "before checkout:\n")
cmd = "mkdir -p {0} && rm -rf {1}/*".format(
deploy.project.target_dir, deploy.project.target_dir)
LocalShell.check_call(cmd, shell=True)
if before_checkout:
cmd = "WORKSPACE='{0}' && cd $WORKSPACE && {1}".format(
deploy.project.checkout_dir, before_checkout)
LocalShell.check_call(cmd, shell=True)
deploys.append_comment(deploy, "OK!\n")
deploys.update(deploy, progress=17)
# checkout
deploys.append_comment(deploy, "checkout:\n")
git.clone()
if deploy.mode == 0:
git.checkout_branch(deploy.branch, deploy.version)
else:
git.checkout_tag(deploy.version)
deploys.append_comment(deploy, "OK!\n")
deploys.update(deploy, progress=33)
# after checkout
after_checkout = deploy.project.after_checkout.replace(
"\r", "").replace("\n", " && ")
deploys.append_comment(deploy, "after checkout:\n")
if after_checkout:
cmd = "WORKSPACE='{0}' && cd $WORKSPACE && {1}".format(
deploy.project.checkout_dir, after_checkout)
LocalShell.check_call(cmd, shell=True)
deploys.append_comment(deploy, "OK!\n")
deploys.update(deploy, progress=50)
# before deploy
deploys.append_comment(deploy, "before deploy:\n")
ssh.check_call(
"mkdir -p {0}".format(
os.path.join(deploy.project.deploy_history_dir,
deploy.softln_filename)))
logger.debug("before deploy:")
ssh.check_call(
("WORKSPACE='{0}' && cd $WORKSPACE && ls -1t | tail -n +{1} | "
"xargs rm -rf").format(deploy.project.deploy_history_dir,
config.MAX_DEPLOY_HISTORY))
before_deploy = deploy.project.before_deploy.replace("\r", "").replace(
"\n", " && ")
if before_deploy:
ssh.check_call(
"WORKSPACE='{0}' && cd $WORKSPACE && {1}".format(
deploy.project.deploy_dir, before_deploy))
deploys.append_comment(deploy, "OK!\n")
deploys.update(deploy, progress=67)
# deploy
deploys.append_comment(deploy, "deploy:\n")
logger.debug("deploy:")
logger.debug("rsync:")
if deploy.host.ssh_method == 0:
cmd = ("rsync -avzq "
"--rsh=\"sshpass -p {ssh_pass} ssh -p {ssh_port}\" "
"--exclude='.git' {local_dest}/ {ssh_user}@{ssh_host}:"
"{remote_dest}/"
).format(local_dest=deploy.project.target_dir,
remote_dest=os.path.join(
deploy.project.deploy_history_dir,
deploy.softln_filename),
ssh_user=deploy.host.ssh_user,
ssh_host=deploy.host.ssh_host,
ssh_port=deploy.host.ssh_port,
ssh_pass=deploy.host.ssh_pass)
else:
cmd = ("rsync -avzq --exclude='.git' {local_dest}/ "
"{ssh_user}@{ssh_host}:{remote_dest}/"
).format(local_dest=deploy.project.target_dir,
remote_dest=os.path.join(
deploy.project.deploy_history_dir,
deploy.softln_filename),
ssh_user=deploy.host.ssh_user,
ssh_host=deploy.host.ssh_host,
ssh_port=deploy.host.ssh_port,
ssh_pass=deploy.host.ssh_pass)
LocalShell.check_call(cmd, shell=True)
ssh.check_call("ln -snf {0} {1}".format(
os.path.join(deploy.project.deploy_history_dir,
deploy.softln_filename),
deploy.project.deploy_dir))
deploys.append_comment(deploy, "OK!\n")
deploys.update(deploy, progress=83)
# after deploy
deploys.append_comment(deploy, "after deploy:\n")
logger.debug("after deploy:")
after_deploy = deploy.project.after_deploy.replace("\r", "").replace(
"\n", " && ")
if after_deploy:
ssh.check_call(
"WORKSPACE='{0}' && cd $WORKSPACE && {1}".format(
deploy.project.deploy_dir, after_deploy))
deploys.append_comment(deploy, "OK!\n")
except Exception as err:
traceback.print_exc()
logger.error(err)
deploys.append_comment(deploy, repr(err))
deploys.update(deploy, status=0)
else:
deploys.update(deploy, progress=100, status=1)
finally:
logger.info("deploy thread end: %d" % deploy.id)
ssh.close()
deploy = deploys.first(project_id=project_id, status=3)
if deploy:
logger.info("deploy thread fetch from wait: {}".format(deploy))
deploys.deploy(deploy)
def checkout(self, branch, version):
logger.debug("checkout:")
LocalShell.check_call(
"cd {0} && git checkout -B {1} -t origin/{1} && git pull -q origin {1} && git reset --hard {2}".format(
self.dest, branch, version),
shell=True)