def set_host_meta(args, xmt_mgr):
hostname = socket.gethostname()
try:
pmt_entry = xmt_mgr.get_entry(xmt.XMT_TYPE_PMT, hostname)
with open(args.host, 'w') as host_file:
yaml.dump(pmt_entry, host_file, explicit_start=True,
default_flow_style=False)
except KeyError:
log.info("No %s defined in the PMT", hostname)
def set_hosts(args, xmt_mgr):
"""Set /etc/hosts for host/ip mapping on
the local host
"""
log.info("refresh hosts file [%s]", args.hosts)
with open(args.template, 'r') as tplfile:
contents = tplfile.readlines()
contents += read_hosts(xmt_mgr)
with open(args.hosts, 'w') as hostsfile:
hostsfile.writelines(contents)
def set_hosts(args, xmt_mgr):
"""Set /etc/hosts for host/ip mapping on
the local host
"""
log.info("refresh hosts file [%s]", args.hosts)
with open(args.template, 'r') as tplfile:
contents = tplfile.readlines()
contents += read_hosts(xmt_mgr)
with open(args.hosts, 'w') as hostsfile:
hostsfile.writelines(contents)
def set_host_meta(args, xmt_mgr):
hostname = socket.gethostname()
try:
pmt_entry = xmt_mgr.get_entry(xmt.XMT_TYPE_PMT, hostname)
with open(args.host, 'w') as host_file:
yaml.dump(pmt_entry,
host_file,
explicit_start=True,
default_flow_style=False)
except KeyError:
log.info("No %s defined in the PMT", hostname)
def check_pid(pidfile):
if not os.path.isfile(pidfile):
log.info("pid file %s does not exist.", pidfile)
return False
with open(pidfile, 'r') as pidfile_hd:
pid = int(pidfile_hd.readline())
try:
os.kill(pid, 0)
except OSError:
log.info("process %d in file %s not exists.", pid, pidfile)
return False
else:
return True
def deploy_workflow_rollback(self):
"""Method to start rollback action.
When rollback completes, it moves to exit state.
"""
log.info("deploy workflow ==> enter [%s] state",
self.current_context['state'])
# rollback to the beginning state
utils.run_command_silient("git reset --hard deploy_start")
log.info("wait for rollback completion...")
time.sleep(self.opts['wait'] * 5)
self.current_context['state'] = STATE_EXIT
self.current_context['prestate'] = STATE_ROLLBACK
def get_file_contents(self, owner, repo, file_path, ref=None):
"""Return the contents of the specified file in string
"""
url = GithubRepo._get_url_base(self.repo_meta['contents_url'])
url += file_path
log.info("github: get file contents (%s)", url)
params = {}
params['ref'] = (None if ref is None else 'master')
resp = self._request_get(url, params=params)
resp_json = resp.json()
file_data = base64.b64decode(resp_json['content'])
return { 'contents': file_data,
'sha': resp_json['sha'],
}
def set_hostname(args, xmt_mgr):
"""setup the hostname in boot mode
"""
try:
pmt_entries = xmt_mgr.get_entries(xmt.XMT_TYPE_PMT)
pmt_entry = [x for x in pmt_entries if args.ecs_name.find(x['ecs_name']) != -1][0]
# pmt_entry = xmt_mgr.get_entries(xmt.XMT_TYPE_PMT, filter={'ecs_name': args.ecs_name})[0]
except ValueError:
log.error("Didn't find the host give ecs_name [%s]!", args.ecs_name)
raise
hostname = pmt_entry['host']
cmd = "echo " + hostname + " > /etc/hostname"
utils.run_command(cmd)
utils.run_command('hostname ' + hostname)
log.info("set hostname to [%s]", hostname)
def deploy_workflow_exit(self):
"""Method to complete the deployment workflow and
update BMT and LSB on master branch accordingly
"""
log.info("deploy workflow ==> enter [%s] state",
self.current_context['state'])
if self.current_context['prestate'] == STATE_ROLLBACK:
# there is nothing to do if it moves from rollback
# state to exit state. LSB and BMT is always rolled
# back
return
lsb_file = self.xmt_mgr.get_xmt_file(xmt.XMT_TYPE_LSB)
# update lsb table
lsb_entries = self.xmt_mgr.construct_lsb_entries()
self.xmt_mgr.save(xmt.XMT_TYPE_LSB, lsb_entries)
msg = "jungar: update lsb on deploy branch"
cmds = []
cmds.append("git add " + lsb_file)
cmds.append("git commit -m \'" + msg + "\'")
utils.run_commands(cmds)
# update the lsb and bmt table on master branch
msg = "deploy: update bmt and lsb at the end of workflow"
cmds = []
cmds.append('git checkout master')
cmds.append('git pull')
cmds.append('git merge ' + self.opts['branch'] +
" -m \"jungar: merge lsb and bmt to master\"")
utils.run_commands(cmds)
retries = 5
while retries:
try:
utils.run_command("git push origin master")
break
except exp.CommandFailure:
log.warn("Push changes to remote failed. Retry [%d/5]", retries)
retries -= 1
time.sleep(5)
return
def set_hostname(args, xmt_mgr):
"""setup the hostname in boot mode
"""
try:
pmt_entries = xmt_mgr.get_entries(xmt.XMT_TYPE_PMT)
pmt_entry = [
x for x in pmt_entries if args.ecs_name.find(x['ecs_name']) != -1
][0]
# pmt_entry = xmt_mgr.get_entries(xmt.XMT_TYPE_PMT, filter={'ecs_name': args.ecs_name})[0]
except ValueError:
log.error("Didn't find the host give ecs_name [%s]!", args.ecs_name)
raise
hostname = pmt_entry['host']
cmd = "echo " + hostname + " > /etc/hostname"
utils.run_command(cmd)
utils.run_command('hostname ' + hostname)
log.info("set hostname to [%s]", hostname)
def deploy_workflow_start(self):
"""Method to perform actions in start state:
- checkout the deploy branch
- prepare for deploying state
"""
log.info("deploy workflow ==> enter [%s] state",
self.current_context['state'])
deploy_branch = self.opts['branch']
cmds = []
cmds.append("git checkout -B " + deploy_branch)
cmds.append("git tag -a deploy_start -m \"jungar: start deploy workflow\"")
cmds.append("git merge master -m \'jungar: merge from master branch\'")
utils.run_commands(cmds)
# prepare to move into next state: deploying
self.current_context['state'] = STATE_DEPLOYING
self.current_context['bmt_phase'] = 0
return self.current_context
def deploy_workflow_deploying(self):
"""Method to perform deploying at given phase
"""
log.info("deploy workflow ==> enter [%s] state",
self.current_context['state'])
bmt_phase = self.current_context['bmt_phase']
assert self.current_context['mlb_phase'] == -1 or \
bmt_phase <= self.current_context['mlb_phase']
# construct BMT table
bmt_entries = self.xmt_mgr.construct_bmt_entries(bmt_phase)
self.xmt_mgr.save(xmt.XMT_TYPE_BMT, bmt_entries)
msg = "deploy: update bmt to phase %d" % (bmt_phase)
bmt_file = self.xmt_mgr.get_xmt_file(xmt.XMT_TYPE_BMT)
output = utils.run_command('git status')
if output.find("nothing to commit") == -1:
cmds = []
cmds.append("git add " + bmt_file)
cmds.append("git commit -m \'" + msg + "\'")
utils.run_commands(cmds)
# TODO: interact with Tarim system to ensure the deployment
# is successfully on the sichuan machines
time.sleep(self.opts['wait'])
# prepare to move into next state
if self.current_context['mlb_phase'] == -1 and bmt_phase < self.max_phase:
self.current_context['state'] = STATE_DEPLOYING
self.current_context['bmt_phase'] = bmt_phase + 1
return self.current_context
if bmt_phase < self.max_phase and \
bmt_phase < self.current_context['mlb_phase']:
self.current_context['state'] = STATE_DEPLOYING
self.current_context['bmt_phase'] = bmt_phase + 1
return self.current_context
self.current_context['state'] = STATE_EXIT
self.current_context['prestate'] = STATE_DEPLOYING
return self.current_context
def test_deploy_worfkflow(self):
log.info("Enter test_deploy_workflow")
opts = {'branch': 'deploy', 'wait': 1}
deploy_mgr = deploy_workflow.DeployWorkflow(self.xmt_mgr, opts)
deploy_mgr.run()
new_xmt_mgr = xmt.XmtManager(pmt_path=self.pmt_file,
bmt_path=self.bmt_file,
lsb_path=self.lsb_file,
mlb_path=self.mlb_file)
new_xmt_mgr.load(xmt.XMT_TYPE_PMT)
new_xmt_mgr.load(xmt.XMT_TYPE_BMT)
new_xmt_mgr.load(xmt.XMT_TYPE_LSB)
new_xmt_mgr.load(xmt.XMT_TYPE_MLB)
# validate that bmt and lsb files are updated successfully
for i in range(2):
lsb_entry = new_xmt_mgr.get_entry(xmt.XMT_TYPE_LSB, i)
self.assertEqual(lsb_entry['build_url'], 'logstream-bits-3')
bmt_entry = new_xmt_mgr.get_entry(xmt.XMT_TYPE_BMT, i)
self.assertEqual(bmt_entry['build_url'], 'logstream-bits-3')
def setUp(self):
super(TestDeployWorkflow, self).setUp()
log.info("enter setup")
self.cur_dir = os.getcwd()
# clone the gittesting repo and make the recovery tag
create_git_test_repo('/tmp', 'git_test')
self.work_dir = "/tmp/git_test"
os.chdir(self.work_dir)
cmds = []
cmds.append("git tag -a test_snapshot -m \"test start snapshot\"")
cmds.append("git config user.email [email protected]")
cmds.append("git config user.name logstream-ci")
cmds.append("git checkout master")
utils.run_commands(cmds)
self.file_dir = os.path.join(self.work_dir)
shutil.copy(lsb_file, self.file_dir)
shutil.copy(mlb_file, self.file_dir)
shutil.copy(pmt_file, self.file_dir)
shutil.copy(bmt_file, self.file_dir)
cmds = []
cmds.append("git add " + self.file_dir + "/*.yaml")
cmds.append("git commit -m \'add test yaml files\'")
utils.run_commands(cmds)
self.pmt_file = os.path.join(self.file_dir, "pmt.yaml")
self.bmt_file = os.path.join(self.file_dir, "bmt.yaml")
self.lsb_file = os.path.join(self.file_dir, "lsb.yaml")
self.mlb_file = os.path.join(self.file_dir, "mlb.yaml")
self.xmt_mgr = xmt.XmtManager(pmt_path = self.pmt_file,
bmt_path = self.bmt_file,
lsb_path = self.lsb_file,
mlb_path = self.mlb_file)
self.xmt_mgr.load(xmt.XMT_TYPE_PMT)
self.xmt_mgr.load(xmt.XMT_TYPE_BMT)
self.xmt_mgr.load(xmt.XMT_TYPE_LSB)
self.xmt_mgr.load(xmt.XMT_TYPE_MLB)
def setUp(self):
super(TestDeployWorkflow, self).setUp()
log.info("enter setup")
self.cur_dir = os.getcwd()
# clone the gittesting repo and make the recovery tag
create_git_test_repo('/tmp', 'git_test')
self.work_dir = "/tmp/git_test"
os.chdir(self.work_dir)
cmds = []
cmds.append("git tag -a test_snapshot -m \"test start snapshot\"")
cmds.append("git config user.email [email protected]")
cmds.append("git config user.name logstream-ci")
cmds.append("git checkout master")
utils.run_commands(cmds)
self.file_dir = os.path.join(self.work_dir)
shutil.copy(lsb_file, self.file_dir)
shutil.copy(mlb_file, self.file_dir)
shutil.copy(pmt_file, self.file_dir)
shutil.copy(bmt_file, self.file_dir)
cmds = []
cmds.append("git add " + self.file_dir + "/*.yaml")
cmds.append("git commit -m \'add test yaml files\'")
utils.run_commands(cmds)
self.pmt_file = os.path.join(self.file_dir, "pmt.yaml")
self.bmt_file = os.path.join(self.file_dir, "bmt.yaml")
self.lsb_file = os.path.join(self.file_dir, "lsb.yaml")
self.mlb_file = os.path.join(self.file_dir, "mlb.yaml")
self.xmt_mgr = xmt.XmtManager(pmt_path=self.pmt_file,
bmt_path=self.bmt_file,
lsb_path=self.lsb_file,
mlb_path=self.mlb_file)
self.xmt_mgr.load(xmt.XMT_TYPE_PMT)
self.xmt_mgr.load(xmt.XMT_TYPE_BMT)
self.xmt_mgr.load(xmt.XMT_TYPE_LSB)
self.xmt_mgr.load(xmt.XMT_TYPE_MLB)
def test_deploy_worfkflow(self):
log.info("Enter test_deploy_workflow")
opts = { 'branch': 'deploy',
'wait': 1
}
deploy_mgr = deploy_workflow.DeployWorkflow(self.xmt_mgr, opts)
deploy_mgr.run()
new_xmt_mgr = xmt.XmtManager(pmt_path = self.pmt_file,
bmt_path = self.bmt_file,
lsb_path = self.lsb_file,
mlb_path = self.mlb_file)
new_xmt_mgr.load(xmt.XMT_TYPE_PMT)
new_xmt_mgr.load(xmt.XMT_TYPE_BMT)
new_xmt_mgr.load(xmt.XMT_TYPE_LSB)
new_xmt_mgr.load(xmt.XMT_TYPE_MLB)
# validate that bmt and lsb files are updated successfully
for i in range(2):
lsb_entry = new_xmt_mgr.get_entry(xmt.XMT_TYPE_LSB, i)
self.assertEqual(lsb_entry['build_url'], 'logstream-bits-3')
bmt_entry = new_xmt_mgr.get_entry(xmt.XMT_TYPE_BMT, i)
self.assertEqual(bmt_entry['build_url'], 'logstream-bits-3')
def run(self):
"""This method is the main loop for deployment
state machine
"""
log.info(">== deployment workflow starts.")
deploy_state = self.current_context['state']
log.debug("deploy_state: %s", deploy_state)
try:
while deploy_state != STATE_EXIT:
action = DeployWorkflow.deploy_state_machine[deploy_state]
action(self)
deploy_state = self.current_context['state']
# go through "exit" state to complete the state machine
DeployWorkflow.deploy_state_machine[deploy_state](self)
except Exception:
# rollback to the beginning state
utils.run_command_silient("git reset --hard deploy_start")
finally:
utils.run_command_silient("git tag -d deploy_start")
utils.run_command_silient("git checkout master")
log.info("<== deployment workflow completed.")
def update_file_contents(self, owner, repo, file_path,
file_contents, message, sha, branch=None):
"""Update the contents of a exisiting file in repo
"""
url = GithubRepo._get_url_base(self.repo_meta['contents_url'])
url += file_path
log.info("github: update file contents (%s)", url)
params = { 'message': message,
'path': file_path,
'content': base64.b64encode(file_contents),
'sha': sha,
"committer": {
"name": self.committer_name,
"email": self.committer_email
},
'branch': ('master' if branch is None else branch)
}
resp = self._request_put(url, params=params)
resp_json = resp.json()
return { 'path': resp_json['content']['path'],
'sha': resp_json['content']['sha'],
'commit_sha': resp_json['commit']['sha'],
}
def deploy_build(args, pmt_entry, pbt):
"""
Deploy the build specified in pbt table
"""
host = pmt_entry['host']
phase = pmt_entry['phase']
log.info(" == start deploying the build for host %s at phase %s", host, phase)
# checkout the branch
d_branch = pbt[phase]['branch']
utils.run_command("cd " + args.shoowo_dir + ";git checkout " + d_branch)
utils.run_command("cd " + args.shoowo_dir + ";git pull")
# if tag is speicifed, checkout that tag
if 'tag' in pbt[phase]:
utils.run_command("cd " + args.shoowo_dir + ";git reset --hard " + pbt[phase]['tag'])
log.info(" -- deployed the build at TAG %s", pbt[phase]['tag'])
else:
utils.run_command('cd ' + args.shoowo_dir + ';git pull')
log.info(" -- deployed the lastest build")
def deploy_build(args, pmt_entry, pbt):
"""
Deploy the build specified in pbt table
"""
host = pmt_entry['host']
phase = pmt_entry['phase']
log.info(" == start deploying the build for host %s at phase %s", host,
phase)
# checkout the branch
d_branch = pbt[phase]['branch']
utils.run_command("cd " + args.shoowo_dir + ";git checkout " + d_branch)
utils.run_command("cd " + args.shoowo_dir + ";git pull")
# if tag is speicifed, checkout that tag
if 'tag' in pbt[phase]:
utils.run_command("cd " + args.shoowo_dir + ";git reset --hard " +
pbt[phase]['tag'])
log.info(" -- deployed the build at TAG %s", pbt[phase]['tag'])
else:
utils.run_command('cd ' + args.shoowo_dir + ';git pull')
log.info(" -- deployed the lastest build")
def send_stats_data(args, metric_name, metric_value, metric_type):
log.info("%s send metric [%s] with value [%s] to %s:%d",
args.source, metric_name, metric_value,
args.server, args.port)
if metric_type == 'gauge':
statsd_client.gauge(metric_name, int(metric_value))
# prepare the working environment
args = prepare()
# load PMT/BMT table
xmt_mgr = xmt.XmtManager()
xmt_mgr.load(xmt.XMT_TYPE_PMT, args.pmt)
hostname = socket.gethostname()
pmt_entry = xmt_mgr.get_entry(xmt.XMT_TYPE_PMT, hostname)
log.debug("Get the PMT entry: %s", pmt_entry)
# modify puppet config with proper environment value
cmd = "sed -i \'s/environment=production$/environment=" + \
pmt_entry['environment'] + "/g\' /etc/puppet/puppet.conf"
utils.run_command(cmd)
# start deployment with the specified build
pbt = load_yaml(args.pbt)
deploy_build(args, pmt_entry, pbt)
except Exception as e:
log.error("Error happens during post boot: %s", e)
raise
return 0
if __name__ == '__main__':
start_time = time.time()
main()
log.info("--- Finished in %.4f seconds ---", (time.time() - start_time))
args = prepare()
# load PMT/BMT table
xmt_mgr = xmt.XmtManager()
xmt_mgr.load(xmt.XMT_TYPE_PMT, args.pmt)
hostname = socket.gethostname()
pmt_entry = xmt_mgr.get_entry(xmt.XMT_TYPE_PMT, hostname)
log.debug("Get the PMT entry: %s", pmt_entry)
# modify puppet config with proper environment value
cmd = "sed -i \'s/environment=production$/environment=" + \
pmt_entry['environment'] + "/g\' /etc/puppet/puppet.conf"
utils.run_command(cmd)
# start deployment with the specified build
pbt = load_yaml(args.pbt)
deploy_build(args, pmt_entry, pbt)
except Exception as e:
log.error("Error happens during post boot: %s", e)
raise
return 0
if __name__ == '__main__':
start_time = time.time()
main()
log.info("--- Finished in %.4f seconds ---", (time.time() - start_time))