示例#1
0
def set_host_meta(args, xmt_mgr):
  hostname = socket.gethostname()
  try:
    pmt_entry = xmt_mgr.get_entry(xmt.XMT_TYPE_PMT, hostname)
    with open(args.host, 'w') as host_file:
        yaml.dump(pmt_entry, host_file, explicit_start=True,
                default_flow_style=False)
  except KeyError:
    log.info("No %s defined in the PMT", hostname)
示例#2
0
def set_hosts(args, xmt_mgr):
    """Set /etc/hosts for host/ip mapping on
     the local host
  """
    log.info("refresh hosts file [%s]", args.hosts)
    with open(args.template, 'r') as tplfile:
        contents = tplfile.readlines()
        contents += read_hosts(xmt_mgr)
        with open(args.hosts, 'w') as hostsfile:
            hostsfile.writelines(contents)
示例#3
0
def set_hosts(args, xmt_mgr):
  """Set /etc/hosts for host/ip mapping on
     the local host
  """
  log.info("refresh hosts file [%s]", args.hosts)
  with open(args.template, 'r') as tplfile:
    contents = tplfile.readlines()
    contents += read_hosts(xmt_mgr)
    with open(args.hosts, 'w') as hostsfile:
      hostsfile.writelines(contents)
示例#4
0
def set_host_meta(args, xmt_mgr):
    hostname = socket.gethostname()
    try:
        pmt_entry = xmt_mgr.get_entry(xmt.XMT_TYPE_PMT, hostname)
        with open(args.host, 'w') as host_file:
            yaml.dump(pmt_entry,
                      host_file,
                      explicit_start=True,
                      default_flow_style=False)
    except KeyError:
        log.info("No %s defined in the PMT", hostname)
示例#5
0
def check_pid(pidfile):
  if not os.path.isfile(pidfile):
    log.info("pid file %s does not exist.", pidfile)
    return False
  with open(pidfile, 'r') as pidfile_hd:
    pid = int(pidfile_hd.readline())
    try:
      os.kill(pid, 0)
    except OSError:
      log.info("process %d in file %s not exists.", pid, pidfile)
      return False
    else:
      return True
示例#6
0
  def deploy_workflow_rollback(self):
    """Method to start rollback action.
      When rollback completes, it moves to exit state.
    """
    log.info("deploy workflow ==> enter [%s] state",
             self.current_context['state'])
    # rollback to the beginning state
    utils.run_command_silient("git reset --hard deploy_start")

    log.info("wait for rollback completion...")
    time.sleep(self.opts['wait'] * 5)

    self.current_context['state'] = STATE_EXIT
    self.current_context['prestate'] = STATE_ROLLBACK
示例#7
0
  def get_file_contents(self, owner, repo, file_path, ref=None):
    """Return the contents of the specified file in string
    """
    url = GithubRepo._get_url_base(self.repo_meta['contents_url'])
    url += file_path
    log.info("github: get file contents (%s)", url)

    params = {}
    params['ref'] = (None if ref is None else 'master')

    resp = self._request_get(url, params=params)
    resp_json = resp.json()
    file_data = base64.b64decode(resp_json['content'])
    return { 'contents': file_data,
             'sha': resp_json['sha'],
           }
示例#8
0
def set_hostname(args, xmt_mgr):
  """setup the hostname in boot mode
  """
  try:
    pmt_entries = xmt_mgr.get_entries(xmt.XMT_TYPE_PMT)
    pmt_entry = [x for x in pmt_entries if args.ecs_name.find(x['ecs_name']) != -1][0]
#    pmt_entry = xmt_mgr.get_entries(xmt.XMT_TYPE_PMT, filter={'ecs_name': args.ecs_name})[0]
  except ValueError:
    log.error("Didn't find the host give ecs_name [%s]!", args.ecs_name)
    raise

  hostname = pmt_entry['host']
  cmd = "echo " + hostname + " > /etc/hostname"
  utils.run_command(cmd)
  utils.run_command('hostname ' + hostname)
  log.info("set hostname to [%s]", hostname)
示例#9
0
  def deploy_workflow_exit(self):
    """Method to complete the deployment workflow and
      update BMT and LSB on master branch accordingly
    """
    log.info("deploy workflow ==> enter [%s] state",
             self.current_context['state'])

    if self.current_context['prestate'] == STATE_ROLLBACK:
      # there is nothing to do if it moves from rollback
      # state to exit state. LSB and BMT is always rolled
      # back
      return

    lsb_file = self.xmt_mgr.get_xmt_file(xmt.XMT_TYPE_LSB)

    # update lsb table
    lsb_entries = self.xmt_mgr.construct_lsb_entries()
    self.xmt_mgr.save(xmt.XMT_TYPE_LSB, lsb_entries)
    msg = "jungar: update lsb on deploy branch"
    cmds = []
    cmds.append("git add " + lsb_file)
    cmds.append("git commit -m \'" + msg + "\'")
    utils.run_commands(cmds)

    # update the lsb and bmt table on master branch
    msg = "deploy: update bmt and lsb at the end of workflow"
    cmds = []
    cmds.append('git checkout master')
    cmds.append('git pull')
    cmds.append('git merge ' + self.opts['branch'] +
                " -m \"jungar: merge lsb and bmt to master\"")
    utils.run_commands(cmds)

    retries = 5
    while retries:
      try:
        utils.run_command("git push origin master")
        break
      except exp.CommandFailure:
        log.warn("Push changes to remote failed. Retry [%d/5]", retries)
        retries -= 1
        time.sleep(5)
    return
示例#10
0
def set_hostname(args, xmt_mgr):
    """setup the hostname in boot mode
  """
    try:
        pmt_entries = xmt_mgr.get_entries(xmt.XMT_TYPE_PMT)
        pmt_entry = [
            x for x in pmt_entries if args.ecs_name.find(x['ecs_name']) != -1
        ][0]


#    pmt_entry = xmt_mgr.get_entries(xmt.XMT_TYPE_PMT, filter={'ecs_name': args.ecs_name})[0]
    except ValueError:
        log.error("Didn't find the host give ecs_name [%s]!", args.ecs_name)
        raise

    hostname = pmt_entry['host']
    cmd = "echo " + hostname + " > /etc/hostname"
    utils.run_command(cmd)
    utils.run_command('hostname ' + hostname)
    log.info("set hostname to [%s]", hostname)
示例#11
0
  def deploy_workflow_start(self):
    """Method to perform actions in start state:
      - checkout the deploy branch
      - prepare for deploying state
    """
    log.info("deploy workflow ==> enter [%s] state",
             self.current_context['state'])
    deploy_branch = self.opts['branch']

    cmds = []
    cmds.append("git checkout -B " + deploy_branch)
    cmds.append("git tag -a deploy_start -m \"jungar: start deploy workflow\"")
    cmds.append("git merge master -m \'jungar: merge from master branch\'")
    utils.run_commands(cmds)

    # prepare to move into next state: deploying
    self.current_context['state'] = STATE_DEPLOYING
    self.current_context['bmt_phase'] = 0

    return self.current_context
示例#12
0
  def deploy_workflow_deploying(self):
    """Method to perform deploying at given phase
    """
    log.info("deploy workflow ==> enter [%s] state",
             self.current_context['state'])
    bmt_phase = self.current_context['bmt_phase']
    assert self.current_context['mlb_phase'] == -1 or \
        bmt_phase <= self.current_context['mlb_phase']

    # construct BMT table
    bmt_entries = self.xmt_mgr.construct_bmt_entries(bmt_phase)
    self.xmt_mgr.save(xmt.XMT_TYPE_BMT, bmt_entries)
    msg = "deploy: update bmt to phase %d" % (bmt_phase)

    bmt_file = self.xmt_mgr.get_xmt_file(xmt.XMT_TYPE_BMT)
    output = utils.run_command('git status')
    if output.find("nothing to commit") == -1:
      cmds = []
      cmds.append("git add " + bmt_file)
      cmds.append("git commit -m \'" + msg + "\'")
      utils.run_commands(cmds)

    # TODO: interact with Tarim system to ensure the deployment
    # is successfully on the sichuan machines
    time.sleep(self.opts['wait'])

    # prepare to move into next state
    if self.current_context['mlb_phase'] == -1 and bmt_phase < self.max_phase:
      self.current_context['state'] = STATE_DEPLOYING
      self.current_context['bmt_phase'] = bmt_phase + 1
      return self.current_context

    if bmt_phase < self.max_phase and \
       bmt_phase < self.current_context['mlb_phase']:
      self.current_context['state'] = STATE_DEPLOYING
      self.current_context['bmt_phase'] = bmt_phase + 1
      return self.current_context

    self.current_context['state'] = STATE_EXIT
    self.current_context['prestate'] = STATE_DEPLOYING
    return self.current_context
示例#13
0
    def test_deploy_worfkflow(self):
        log.info("Enter test_deploy_workflow")
        opts = {'branch': 'deploy', 'wait': 1}
        deploy_mgr = deploy_workflow.DeployWorkflow(self.xmt_mgr, opts)
        deploy_mgr.run()

        new_xmt_mgr = xmt.XmtManager(pmt_path=self.pmt_file,
                                     bmt_path=self.bmt_file,
                                     lsb_path=self.lsb_file,
                                     mlb_path=self.mlb_file)
        new_xmt_mgr.load(xmt.XMT_TYPE_PMT)
        new_xmt_mgr.load(xmt.XMT_TYPE_BMT)
        new_xmt_mgr.load(xmt.XMT_TYPE_LSB)
        new_xmt_mgr.load(xmt.XMT_TYPE_MLB)

        # validate that bmt and lsb files are updated successfully
        for i in range(2):
            lsb_entry = new_xmt_mgr.get_entry(xmt.XMT_TYPE_LSB, i)
            self.assertEqual(lsb_entry['build_url'], 'logstream-bits-3')
            bmt_entry = new_xmt_mgr.get_entry(xmt.XMT_TYPE_BMT, i)
            self.assertEqual(bmt_entry['build_url'], 'logstream-bits-3')
示例#14
0
  def setUp(self):
    super(TestDeployWorkflow, self).setUp()
    log.info("enter setup")

    self.cur_dir = os.getcwd()

    # clone the gittesting repo and make the recovery tag
    create_git_test_repo('/tmp', 'git_test')
    self.work_dir = "/tmp/git_test"
    os.chdir(self.work_dir)
    cmds = []
    cmds.append("git tag -a test_snapshot -m \"test start snapshot\"")
    cmds.append("git config user.email [email protected]")
    cmds.append("git config user.name logstream-ci")
    cmds.append("git checkout master")
    utils.run_commands(cmds)

    self.file_dir = os.path.join(self.work_dir)
    shutil.copy(lsb_file, self.file_dir)
    shutil.copy(mlb_file, self.file_dir)
    shutil.copy(pmt_file, self.file_dir)
    shutil.copy(bmt_file, self.file_dir)

    cmds = []
    cmds.append("git add " + self.file_dir + "/*.yaml")
    cmds.append("git commit -m \'add test yaml files\'")
    utils.run_commands(cmds)

    self.pmt_file = os.path.join(self.file_dir, "pmt.yaml")
    self.bmt_file = os.path.join(self.file_dir, "bmt.yaml")
    self.lsb_file = os.path.join(self.file_dir, "lsb.yaml")
    self.mlb_file = os.path.join(self.file_dir, "mlb.yaml")
    self.xmt_mgr = xmt.XmtManager(pmt_path = self.pmt_file,
                                  bmt_path = self.bmt_file,
                                  lsb_path = self.lsb_file,
                                  mlb_path = self.mlb_file)
    self.xmt_mgr.load(xmt.XMT_TYPE_PMT)
    self.xmt_mgr.load(xmt.XMT_TYPE_BMT)
    self.xmt_mgr.load(xmt.XMT_TYPE_LSB)
    self.xmt_mgr.load(xmt.XMT_TYPE_MLB)
示例#15
0
    def setUp(self):
        super(TestDeployWorkflow, self).setUp()
        log.info("enter setup")

        self.cur_dir = os.getcwd()

        # clone the gittesting repo and make the recovery tag
        create_git_test_repo('/tmp', 'git_test')
        self.work_dir = "/tmp/git_test"
        os.chdir(self.work_dir)
        cmds = []
        cmds.append("git tag -a test_snapshot -m \"test start snapshot\"")
        cmds.append("git config user.email [email protected]")
        cmds.append("git config user.name logstream-ci")
        cmds.append("git checkout master")
        utils.run_commands(cmds)

        self.file_dir = os.path.join(self.work_dir)
        shutil.copy(lsb_file, self.file_dir)
        shutil.copy(mlb_file, self.file_dir)
        shutil.copy(pmt_file, self.file_dir)
        shutil.copy(bmt_file, self.file_dir)

        cmds = []
        cmds.append("git add " + self.file_dir + "/*.yaml")
        cmds.append("git commit -m \'add test yaml files\'")
        utils.run_commands(cmds)

        self.pmt_file = os.path.join(self.file_dir, "pmt.yaml")
        self.bmt_file = os.path.join(self.file_dir, "bmt.yaml")
        self.lsb_file = os.path.join(self.file_dir, "lsb.yaml")
        self.mlb_file = os.path.join(self.file_dir, "mlb.yaml")
        self.xmt_mgr = xmt.XmtManager(pmt_path=self.pmt_file,
                                      bmt_path=self.bmt_file,
                                      lsb_path=self.lsb_file,
                                      mlb_path=self.mlb_file)
        self.xmt_mgr.load(xmt.XMT_TYPE_PMT)
        self.xmt_mgr.load(xmt.XMT_TYPE_BMT)
        self.xmt_mgr.load(xmt.XMT_TYPE_LSB)
        self.xmt_mgr.load(xmt.XMT_TYPE_MLB)
示例#16
0
  def test_deploy_worfkflow(self):
    log.info("Enter test_deploy_workflow")
    opts = { 'branch': 'deploy',
             'wait': 1
           }
    deploy_mgr = deploy_workflow.DeployWorkflow(self.xmt_mgr, opts)
    deploy_mgr.run()

    new_xmt_mgr = xmt.XmtManager(pmt_path = self.pmt_file,
                                 bmt_path = self.bmt_file,
                                 lsb_path = self.lsb_file,
                                 mlb_path = self.mlb_file)
    new_xmt_mgr.load(xmt.XMT_TYPE_PMT)
    new_xmt_mgr.load(xmt.XMT_TYPE_BMT)
    new_xmt_mgr.load(xmt.XMT_TYPE_LSB)
    new_xmt_mgr.load(xmt.XMT_TYPE_MLB)

    # validate that bmt and lsb files are updated successfully
    for i in range(2):
      lsb_entry = new_xmt_mgr.get_entry(xmt.XMT_TYPE_LSB, i)
      self.assertEqual(lsb_entry['build_url'], 'logstream-bits-3')
      bmt_entry = new_xmt_mgr.get_entry(xmt.XMT_TYPE_BMT, i)
      self.assertEqual(bmt_entry['build_url'], 'logstream-bits-3')
示例#17
0
  def run(self):
    """This method is the main loop for deployment
      state machine
    """
    log.info(">== deployment workflow starts.")

    deploy_state = self.current_context['state']
    log.debug("deploy_state: %s", deploy_state)
    try:
      while deploy_state != STATE_EXIT:
        action = DeployWorkflow.deploy_state_machine[deploy_state]
        action(self)
        deploy_state = self.current_context['state']
      # go through "exit" state to complete the state machine
      DeployWorkflow.deploy_state_machine[deploy_state](self)
    except Exception:
      # rollback to the beginning state
      utils.run_command_silient("git reset --hard deploy_start")
    finally:
      utils.run_command_silient("git tag -d deploy_start")
      utils.run_command_silient("git checkout master")

    log.info("<== deployment workflow completed.")
示例#18
0
  def update_file_contents(self, owner, repo, file_path,
                          file_contents, message, sha, branch=None):
    """Update the contents of a exisiting file in repo
    """
    url = GithubRepo._get_url_base(self.repo_meta['contents_url'])
    url += file_path
    log.info("github: update file contents (%s)", url)

    params = { 'message': message,
               'path': file_path,
               'content': base64.b64encode(file_contents),
               'sha': sha,
               "committer": {
                 "name": self.committer_name,
                 "email": self.committer_email
               },
               'branch': ('master' if branch is None else branch)
             }
    resp = self._request_put(url, params=params)
    resp_json = resp.json()
    return { 'path': resp_json['content']['path'],
             'sha': resp_json['content']['sha'],
             'commit_sha': resp_json['commit']['sha'],
           }
示例#19
0
def deploy_build(args, pmt_entry, pbt):
    """
    Deploy the build specified in pbt table
    """
    host = pmt_entry['host']
    phase = pmt_entry['phase']

    log.info(" == start deploying the build for host %s at phase %s", host, phase)

    # checkout the branch
    d_branch = pbt[phase]['branch']
    utils.run_command("cd " + args.shoowo_dir + ";git checkout " + d_branch)
    utils.run_command("cd " + args.shoowo_dir + ";git pull")

    # if tag is speicifed, checkout that tag
    if 'tag' in pbt[phase]:
        utils.run_command("cd " + args.shoowo_dir + ";git reset --hard " + pbt[phase]['tag'])
        log.info(" -- deployed the build at TAG %s", pbt[phase]['tag'])
    else:
        utils.run_command('cd ' + args.shoowo_dir + ';git pull')
        log.info(" -- deployed the lastest build")
示例#20
0
def deploy_build(args, pmt_entry, pbt):
    """
    Deploy the build specified in pbt table
    """
    host = pmt_entry['host']
    phase = pmt_entry['phase']

    log.info(" == start deploying the build for host %s at phase %s", host,
             phase)

    # checkout the branch
    d_branch = pbt[phase]['branch']
    utils.run_command("cd " + args.shoowo_dir + ";git checkout " + d_branch)
    utils.run_command("cd " + args.shoowo_dir + ";git pull")

    # if tag is speicifed, checkout that tag
    if 'tag' in pbt[phase]:
        utils.run_command("cd " + args.shoowo_dir + ";git reset --hard " +
                          pbt[phase]['tag'])
        log.info(" -- deployed the build at TAG %s", pbt[phase]['tag'])
    else:
        utils.run_command('cd ' + args.shoowo_dir + ';git pull')
        log.info(" -- deployed the lastest build")
示例#21
0
def send_stats_data(args, metric_name, metric_value, metric_type):
  log.info("%s send metric [%s] with value [%s] to %s:%d",
           args.source, metric_name, metric_value,
           args.server, args.port)
  if metric_type == 'gauge':
    statsd_client.gauge(metric_name, int(metric_value))
示例#22
0
        # prepare the working environment
        args = prepare()

        # load PMT/BMT table
        xmt_mgr = xmt.XmtManager()
        xmt_mgr.load(xmt.XMT_TYPE_PMT, args.pmt)

        hostname = socket.gethostname()
        pmt_entry = xmt_mgr.get_entry(xmt.XMT_TYPE_PMT, hostname)
        log.debug("Get the PMT entry: %s", pmt_entry)

        # modify puppet config with proper environment value
        cmd = "sed -i \'s/environment=production$/environment=" + \
              pmt_entry['environment'] + "/g\' /etc/puppet/puppet.conf"
        utils.run_command(cmd)

        # start deployment with the specified build
        pbt = load_yaml(args.pbt)
        deploy_build(args, pmt_entry, pbt)

    except Exception as e:
        log.error("Error happens during post boot: %s", e)
        raise

    return 0

if __name__ == '__main__':
  start_time = time.time()
  main()
  log.info("--- Finished in %.4f seconds ---", (time.time() - start_time))
示例#23
0
        args = prepare()

        # load PMT/BMT table
        xmt_mgr = xmt.XmtManager()
        xmt_mgr.load(xmt.XMT_TYPE_PMT, args.pmt)

        hostname = socket.gethostname()
        pmt_entry = xmt_mgr.get_entry(xmt.XMT_TYPE_PMT, hostname)
        log.debug("Get the PMT entry: %s", pmt_entry)

        # modify puppet config with proper environment value
        cmd = "sed -i \'s/environment=production$/environment=" + \
              pmt_entry['environment'] + "/g\' /etc/puppet/puppet.conf"
        utils.run_command(cmd)

        # start deployment with the specified build
        pbt = load_yaml(args.pbt)
        deploy_build(args, pmt_entry, pbt)

    except Exception as e:
        log.error("Error happens during post boot: %s", e)
        raise

    return 0


if __name__ == '__main__':
    start_time = time.time()
    main()
    log.info("--- Finished in %.4f seconds ---", (time.time() - start_time))