def getRegisteredMachines():
result = helper.execCommand("VBoxManage list vms")
lines = result.stdout.decode("utf-8").strip().split("\n")
machines = {}
for line in lines:
if len(line) > 0:
(name, vid) = line.split(" ")
machines[vid] = name.strip("\"")
return machines
def destroyMachine(vid):
vmCleaned = False
if vid != None:
machines = getRegisteredMachines()
if vid in machines:
name = machines[vid]
log.info(u"VM - vid: '{}', name: '{}' - destroyed.".format(
vid, name))
helper.execCommand(
"VBoxManage controlvm \"{}\" poweroff".format(vid),
exitstatus_check=False)
helper.execCommand(
"VBoxManage unregistervm --delete \"{}\"".format(vid),
exitstatus_check=False)
vmCleaned = True
else:
log.error(u"VM - vid: '{}' not found.".format(vid))
return vmCleaned
def initRepository(repository_dir, repository_url, build_dir):
if not os.path.isdir(repository_dir):
log.info("Clone repository: {} ... ".format(repository_url),
end='',
flush=True)
cloneResult = helper.execCommand(
u"git clone {}".format(repository_url), build_dir)
if cloneResult.returncode == 0:
log.info(u"done", flush=True)
else:
log.error(u"error: {}".format(cloneResult.stdout.decode("utf-8")),
flush=True)
def showRunningJobs(lib_dir):
pid = getPid()
if pid != None:
log.info(u"Main process is running with pid '{}'.".format(pid))
else:
log.info(u"Main process is not running.")
processes = []
ci_result = helper.execCommand("ps -alx | grep 'ci_job_handler'")
ci_lines = ci_result.stdout.decode("utf-8").split(u"\n")
formatProcesses(ci_lines,processes)
vm_result = helper.execCommand("ps -alx | grep virtualbox")
vm_lines = vm_result.stdout.decode("utf-8").split(u"\n")
formatProcesses(vm_lines,processes)
if len(processes) > 0:
log.info(u"Following sub processes are running.")
log.info(u"\n".join(processes))
else:
log.info(u"No sub processes are running.")
virtualbox.checkMachines(lib_dir, True)
def getLog(repository_dir, git_hash):
result = helper.execCommand(u"git show --quiet {}".format(git_hash),
repository_dir)
lines = result.stdout.decode("utf-8").split("\n")
author = ""
subject = ""
is_subject = False
for line in lines:
line = line.strip()
if line.startswith("Author"):
author = re.findall(r'Author: ([^<]+).*', line)[0].strip()
elif line.startswith("Date"):
is_subject = True
elif is_subject and line != "":
subject = line
break
return {"author": author, "subject": subject}
def getHash(repository_dir):
checkResult = helper.execCommand(u"git rev-parse @", repository_dir)
return checkResult.stdout.decode("utf-8").strip()
def updateRepository(repository_dir, branch):
# git ls-remote {{vault_deployment_config_git}} HEAD
helper.execCommand(u"git pull", repository_dir)
def startCheck(self, config_name, os_name, args):
argsStr = ""
if args != None:
argsStr = u" --args={}".format(args)
#output = pexpect.run("sleep 30", timeout=1, cwd=self.repository_dir, withexitstatus=True, events={pexpect.TIMEOUT:self.searchDeploymentVID})
#return
author = re.sub('\\W+|\\s+', "_", self.commit['author'])
subject = re.sub('\\W+|\\s+', "_", self.commit['subject'])
if len(subject) > max_subject_length:
subject = subject[0:max_subject_length]
pos = subject.rfind("_")
subject = u"{}_".format(subject[0:pos])
i = 0
while True:
self.registered_machines = {}
self.active_machine = None
self.cancel_reason = None
self.deploy_exit_status = 1
self.start_time = datetime.now()
self.start_time_str = self.start_time.strftime(
START_TIME_STR_FORMAT)
deployment_log_file = getLogFilename(
self.log_dir, self.start_time_str, 0, "running", config_name,
os_name, self.branch, self.git_hash, author, subject)
vagrant_path = pathlib.Path(
__file__).parent.absolute().as_posix() + "/../vagrant"
# force VBox folder
helper.execCommand(
"VBoxManage setproperty machinefolder {}VirtualMachines".
format(self.lib_dir))
env = {"VAGRANT_HOME": self.lib_dir}
with open(deployment_log_file, 'w') as f:
try:
# Always test with the latest version
helper.log(u"Check for new images")
update_cmd = u"{} --config={} --os={} box update".format(
vagrant_path, config_name, os_name)
(update_output,
update_exit_status) = pexpect.run(update_cmd,
timeout=1800,
logfile=LogFile(f),
cwd=self.repository_dir,
env=env,
withexitstatus=True)
except ValueError:
#helper.log( update_output )
#helper.log( traceback.format_exc(), "err" )
pass
self.registered_machines = virtualbox.getRegisteredMachines()
#helper.log( u"{}".format("VAGRANT_HOME={}".format(self.lib_dir)))
#helper.log( u"{}".format(update_exit_status) )
#helper.log( u"{}".format(update_output) )
# Deployment start
deploy_output = ""
cmd = u"{} --config={} --os={}{} up".format(
vagrant_path, config_name, os_name, argsStr)
#cmd = u"echo '\033[31mtest1\033[0m' && echo '\033[200mtest2' && echo 1 && sleep 5 && echo 2 && sleep 5 && echo 3 2>&1"
helper.log(u"Deployment for commit '{}' ('{}') started".format(
self.git_hash, cmd))
with open(deployment_log_file, 'a') as f:
try:
(deploy_output, self.deploy_exit_status) = pexpect.run(
cmd,
timeout=1,
logfile=LogFile(f),
cwd=self.repository_dir,
env=env,
withexitstatus=True,
events={pexpect.TIMEOUT: self.searchMachineVID})
except ValueError:
#helper.log( deploy_output )
#helper.log( traceback.format_exc(), "err" )
pass
# Deployment done
retry = False
if self.deploy_exit_status == 0:
helper.log(u"Deployment for commit '{}' successful".format(
self.git_hash))
self.cancel_reason = None
else:
if self.cancel_reason == None:
log_lines = deploy_output.split(b"\n")
log_lines_to_check = log_lines[-100:] if len(
log_lines) > 100 else log_lines
i = i + 1
# Check if retry is possible
if self.checkForRetry(log_lines_to_check):
if i < max_retries:
self.cancel_reason = "retry"
retry = True
else:
self.cancel_reason = "max_retries"
reason = u" ({})".format(
self.cancel_reason) if self.cancel_reason != None else ""
helper.log(
u"Deployment for commit '{}' unsuccessful {}".format(
self.git_hash, reason))
# Final logfile preperation
duration = int(
round(datetime.now().timestamp() -
self.start_time.timestamp()))
status_msg = ""
with open(deployment_log_file, 'a') as f:
f.write("\n")
lf = LogFile(f)
if self.deploy_exit_status == 0:
lf.write(
"The command '{}' exited with 0 (successful) after {}.\n"
.format(cmd, timedelta(seconds=duration)))
status_msg = 'success'
else:
if retry:
status_msg = 'retry'
else:
status_msg = 'failed'
if self.cancel_reason != None:
lf.write("{}\n".format(
retry_messages[self.cancel_reason]))
lf.write(
"The command '{}' exited with {} (unsuccessful) after {}.\n"
.format(cmd, self.deploy_exit_status,
timedelta(seconds=duration)))
# Rename logfile
finished_log_file = getLogFilename(self.log_dir,
self.start_time_str, duration,
status_msg, config_name,
os_name, self.branch,
self.git_hash, author, subject)
os.rename(deployment_log_file, finished_log_file)
# Cleanup start
helper.log(u"Cleaning for commit '{}' started".format(
self.git_hash))
cmd = u"{} --config={} --os={} destroy --force".format(
vagrant_path, config_name, os_name)
(destroy_output,
destroy_exit_status) = pexpect.run(cmd,
timeout=max_cleanup_time,
cwd=self.repository_dir,
env=env,
withexitstatus=True)
status.setVID(self.status_file, None)
# Cleanup done
if destroy_exit_status == 0:
helper.log(u"Cleaning for commit '{}' successful".format(
self.git_hash))
else:
helper.log(u"Cleaning for commit '{}' unsuccessful".format(
self.git_hash))
if retry:
helper.log(u"Retry deployment for commit '{}'".format(
self.git_hash))
continue
break
return self.deploy_exit_status == 0, self.start_time_str, self.cancel_reason if self.cancel_reason != None else "deployment"