예제 #1
0
def main():
    """ db_uninstaller main loop """

    # handle parser option
    options = get_options()

    notify = lambda n: raw_input('Uninstall Trafodion on [%s] [N]: ' % n)

    format_output('Trafodion Uninstall Start')

    if options.pwd:
        pwd = getpass.getpass('Input remote host SSH Password: '******''

    node_list = ''
    # parse node list from trafodion_config
    if os.path.exists(TRAF_CFG_FILE):
        with open(TRAF_CFG_FILE, 'r') as f:
            traf_cfgs = f.readlines()
        try:
            line = [l for l in traf_cfgs if 'NODE_LIST' in l][0]
            node_list = re.search(r'NODE_LIST="(.*)"', line).groups()[0]
        except Exception as e:
            err_m('Cannot find node list info from %s: %s' % (TRAF_CFG_FILE, e))
    # parse node list from installation config file
    elif options.cfgfile:
        if not os.path.exists(options.cfgfile):
            err_m('Cannot find config file \'%s\'' % options.cfgfile)
        config_file = options.cfgfile
        p = ParseInI(config_file, 'dbconfigs')
        cfgs = p.load()
        node_list = cfgs['node_list']
    # user input
    else:
        node_lists = raw_input('Enter Trafodion node list to uninstall(separated by comma): ')
        if not node_lists: err_m('Empty value')
        node_list = ' '.join(expNumRe(node_lists))

    if not options.silent:
        rc = notify(node_list)
        if rc.lower() != 'y': sys.exit(1)

    nodes = node_list.split()
    first_node = nodes[0]

    remotes = [Remote(node, pwd=pwd) for node in nodes]
    sudo_prefix = get_sudo_prefix()

    # remove trafodion userid and group on all trafodion nodes, together with folders
    for remote in remotes:
        info('Remove Trafodion on node [%s] ...' % remote.host)
        remote.execute('ps -f -u %s|awk \'{print $2}\'|xargs %s kill -9' % (TRAF_USER, sudo_prefix), chkerr=False)
        remote.execute('trafid=`getent passwd %s|awk -F: \'{print $3}\'`; if [[ -n $trafid ]]; then ps -f -u $trafid|awk \'{print $2}\'|xargs %s kill -9; fi' % (TRAF_USER, sudo_prefix), chkerr=False)
        remote.execute('%s /usr/sbin/userdel -rf %s' % (sudo_prefix, TRAF_USER), chkerr=False)
        remote.execute('%s /usr/sbin/groupdel %s' % (sudo_prefix, TRAF_USER), chkerr=False)
        remote.execute('%s rm -rf /etc/security/limits.d/%s.conf %s /tmp/hsperfdata_%s 2>/dev/null' % (sudo_prefix, TRAF_USER, TRAF_CFG_DIR, TRAF_USER), chkerr=False)

    run_cmd('rm -f %s/*.status %s' % (INSTALLER_LOC, DBCFG_FILE))
    format_output('Trafodion Uninstall Completed')
예제 #2
0
def main():
    """ db_uninstaller main loop """

    # handle parser option
    options = get_options()

    notify = lambda n: raw_input('Uninstall Trafodion on [%s] [N]: ' % n)

    format_output('Trafodion Uninstall Start')

    if options.pwd:
        pwd = getpass.getpass('Input remote host SSH Password: '******''

    node_list = ''
    # parse node list from trafodion_config
    if os.path.exists(TRAFODION_CFG_FILE):
        with open(TRAFODION_CFG_FILE, 'r') as f:
            traf_cfgs = f.readlines()
        try:
            line = [l for l in traf_cfgs if 'NODE_LIST' in l][0]
            node_list = re.search(r'NODE_LIST="(.*)"', line).groups()[0]
        except Exception as e:
            err_m('Cannot find node list info from %s: %s' % (TRAFODION_CFG_FILE, e))
    # parse node list from installation config file
    elif options.cfgfile:
        if not os.path.exists(options.cfgfile):
            err_m('Cannot find config file \'%s\'' % options.cfgfile)
        config_file = options.cfgfile
        p = ParseInI(config_file, 'dbconfigs')
        cfgs = p.load()
        node_list = cfgs['node_list']
    # user input
    else:
        node_lists = raw_input('Enter Trafodion node list to uninstall(separated by comma): ')
        if not node_lists: err_m('Empty value')
        node_list = ' '.join(expNumRe(node_lists))

    if not options.silent:
        rc = notify(node_list)
        if rc.lower() != 'y': sys.exit(1)

    nodes = node_list.split()
    first_node = nodes[0]

    remotes = [Remote(node, pwd=pwd) for node in nodes]
    # stop trafodion on the first node
    remotes[0].execute('sudo su %s -l -c ckillall' % TRAF_USER, chkerr=False)

    # remove trafodion userid and group on all trafodion nodes, together with folders
    for remote in remotes:
        info('Remove Trafodion on node [%s] ...' % remote.host)
        remote.execute('ps -ef|grep ^%s|awk \'{print $2}\'|xargs sudo kill -9' % TRAF_USER, chkerr=False)
        remote.execute('sudo -n /usr/sbin/userdel -rf %s' % TRAF_USER, chkerr=False)
        remote.execute('sudo -n /usr/sbin/groupdel %s' % TRAF_USER, chkerr=False)
        remote.execute('sudo -n rm -rf /etc/security/limits.d/trafodion.conf /etc/trafodion /tmp/hsperfdata_%s 2>/dev/null' % TRAF_USER, chkerr=False)

    format_output('Trafodion Uninstall Completed')
예제 #3
0
def main():
    """ db_uninstaller main loop """

    # handle parser option
    options = get_options()

    notify = lambda n: raw_input('Uninstall Trafodion on [%s] [N]: ' % n)

    format_output('Trafodion Uninstall Start')

    if options.pwd:
        pwd = getpass.getpass('Input remote host SSH Password: '******''

    try:
      traf_var = run_cmd_as_user(TRAF_USER,"echo $TRAF_VAR")
      node_list = run_cmd_as_user(TRAF_USER,"trafconf -name")
    except:
      if options.cfgfile:
        if not os.path.exists(options.cfgfile):
            err_m('Cannot find config file \'%s\'' % options.cfgfile)
        config_file = options.cfgfile
        p = ParseInI(config_file, 'dbconfigs')
        cfgs = p.load()
        traf_var = cfgs['traf_var']
        node_list = cfgs['node_list']
      # user input
      else:
        traf_var = '/var/lib/trafodion'
        node_lists = raw_input('Enter Trafodion node list to uninstall(separated by comma): ')
        if not node_lists: err_m('Empty value')
        node_list = ' '.join(expNumRe(node_lists))

    if not options.silent:
        rc = notify(node_list)
        if rc.lower() != 'y': sys.exit(1)

    nodes = node_list.split()
    first_node = nodes[0]

    remotes = [Remote(node, pwd=pwd) for node in nodes]
    sudo_prefix = get_sudo_prefix()

    # remove trafodion userid and group on all trafodion nodes, together with folders
    for remote in remotes:
        info('Remove Trafodion on node [%s] ...' % remote.host)
        remote.execute('ps -f -u %s|awk \'{print $2}\'|xargs %s kill -9' % (TRAF_USER, sudo_prefix), chkerr=False)
        remote.execute('trafid=`getent passwd %s|awk -F: \'{print $3}\'`; if [[ -n $trafid ]]; then ps -f -u $trafid|awk \'{print $2}\'|xargs %s kill -9; fi' % (TRAF_USER, sudo_prefix), chkerr=False)
        remote.execute('%s /usr/sbin/userdel -rf %s' % (sudo_prefix, TRAF_USER), chkerr=False)
        remote.execute('%s /usr/sbin/groupdel %s' % (sudo_prefix, TRAF_USER), chkerr=False)
        remote.execute('%s rm -rf /etc/security/limits.d/%s.conf %s %s /tmp/hsperfdata_%s 2>/dev/null' % (sudo_prefix, TRAF_USER, TRAF_CFG_DIR, traf_var, TRAF_USER), chkerr=False)

    run_cmd('rm -f %s/*.status %s' % (INSTALLER_LOC, DBCFG_FILE))
    format_output('Trafodion Uninstall Completed')
예제 #4
0
def main():
    options = get_options()

    cfgs = defaultdict(str)

    if options.cfgfile:
        if not os.path.exists(options.cfgfile):
            err_m('Cannot find config file \'%s\'' % options.cfgfile)
        config_file = options.cfgfile
    else:
        config_file = DBCFG_FILE

    if options.pwd:
        pwd = getpass.getpass('Input remote host SSH Password: '******''

    if os.path.exists(config_file):
        cfgs = ParseInI(config_file, 'dbconfigs').load()
    else:
        node_lists = expNumRe(
            raw_input(
                'Enter list of Nodes separated by comma, support numeric RE, i.e. n[01-12]: '
            ))

        # check if node list is expanded successfully
        if len([1 for node in node_lists if '[' in node]):
            err('Failed to expand node list, please check your input.')

        cfgs['node_list'] = ','.join(node_lists)

    results = wrapper.run(cfgs, options, mode='discover', pwd=pwd)

    format_output('Discover results')

    if len(results) > 4:
        output = output_row(results)
    else:
        output = output_column(results)

    print output
    with open('discover_result', 'w') as f:
        f.write('Discover Date: %s\n' % time.strftime('%Y-%m-%d %H:%M'))
        f.write(output)
예제 #5
0
    def notify_user(self):
        """ show the final configs to user """
        format_output('Final Configs')
        title = ['config type', 'value']
        pt = PrettyTable(title)
        for item in title:
            pt.align[item] = 'l'

        for key, value in sorted(cfgs.items()):
            # only notify user input value
            if self.in_data.has_key(key) and value:
                if self.in_data[key].has_key('ispasswd'): continue
                pt.add_row([key, value])
        print pt
        confirm = self.get_confirm()
        if confirm != 'Y':
            if os.path.exists(DBCFG_FILE): os.remove(DBCFG_FILE)
            run_cmd('rm -rf %s/*.status' % INSTALLER_LOC)
            log_err('User quit')
예제 #6
0
    def notify_user(self):
        """ show the final configs to user """
        format_output('Final Configs')
        title = ['config type', 'value']
        pt = PrettyTable(title)
        for item in title:
            pt.align[item] = 'l'

        for key, value in sorted(cfgs.items()):
            # only notify user input value
            if self.in_data.has_key(key) and value:
                if self.in_data[key].has_key('ispasswd'): continue
                pt.add_row([key, value])
        print pt
        confirm = self.get_confirm()
        if confirm != 'Y':
            if os.path.exists(DBCFG_FILE): os.remove(DBCFG_FILE)
            run_cmd('rm -rf %s/*.status' % INSTALLER_LOC)
            log_err('User quit')
예제 #7
0
def main():
    """ db_installer main loop """
    global cfgs
    format_output('Trafodion Installation ToolKit')

    # handle parser option
    options = get_options()

    if options.build and options.cfgfile:
        log_err(
            'Wrong parameter, cannot specify both --build and --config-file')

    if options.build and options.offline:
        log_err('Wrong parameter, cannot specify both --build and --offline')

    if options.cfgfile:
        if not os.path.exists(options.cfgfile):
            log_err('Cannot find config file \'%s\'' % options.cfgfile)
        config_file = options.cfgfile
    else:
        config_file = DBCFG_FILE

    if options.pwd:
        pwd = getpass.getpass('Input remote host SSH Password: '******''

    # not specified config file and default config file doesn't exist either
    p = ParseInI(config_file, 'dbconfigs')
    if options.build or (not os.path.exists(config_file)):
        if options.build: format_output('DryRun Start')
        user_input(options, prompt_mode=True, pwd=pwd)

        # save config file as json format
        print '\n** Generating config file to save configs ... \n'
        p.save(cfgs)
    # config file exists
    else:
        print '\n** Loading configs from config file ... \n'
        cfgs = p.load()
        if options.offline and cfgs['offline_mode'] != 'Y':
            log_err(
                'To enable offline mode, must set "offline_mode = Y" in config file'
            )
        user_input(options, prompt_mode=False, pwd=pwd)

    if options.reinstall:
        cfgs['reinstall'] = 'Y'

    if options.offline:
        http_start(cfgs['local_repo_dir'], cfgs['repo_http_port'])
    else:
        cfgs['offline_mode'] = 'N'

    if not options.build:
        format_output('Installation Start')

        ### perform actual installation ###
        wrapper.run(cfgs, options, pwd=pwd)

        format_output('Installation Complete')

        if options.offline: http_stop()

        # rename default config file when successfully installed
        # so next time user can input new variables for a new install
        # or specify the backup config file to install again
        try:
            # only rename default config file
            ts = time.strftime('%y%m%d_%H%M')
            if config_file == DBCFG_FILE and os.path.exists(config_file):
                os.rename(config_file, config_file + '.bak' + ts)
        except OSError:
            log_err('Cannot rename config file')
    else:
        format_output('DryRun Complete')

    # remove temp config file
    if os.path.exists(DBCFG_TMP_FILE): os.remove(DBCFG_TMP_FILE)
예제 #8
0
def main():
    """ add_nodes main loop """
    cfgs = defaultdict(str)

    # handle parser option
    options = get_options()
    if not options.nodes:
        err_m('Must specifiy the node names using \'--nodes\' option')

    # get node list from user input
    new_nodes = expNumRe(options.nodes)
    if not new_nodes:
        err_m('Incorrect format')

    if options.pwd:
        pwd = getpass.getpass('Input remote host SSH Password: '******''

    u = UserInput(options, pwd)
    g = lambda n: u.get_input(n, cfgs[n], prompt_mode=prompt_mode)

    format_output('Trafodion Elastic Add Nodes Script')

    ### read configs from current trafodion_config and save it to cfgs
    if os.path.exists(TRAF_CFG_FILE):
        with open(TRAF_CFG_FILE, 'r') as f:
            traf_cfgs = f.readlines()
        for traf_cfg in traf_cfgs:
            if not traf_cfg.strip(): continue
            key, value = traf_cfg.replace('export ', '').split('=')
            value = value.replace('"', '')
            value = value.replace('\n', '')
            cfgs[key.lower()] = value
    else:
        err_m(
            'Cannot find %s, be sure to run this script on one of trafodion nodes'
            % TRAF_CFG_FILE)

    ### config check
    if not cfgs['hbase_lib_path'] or not cfgs['traf_version']:
        err_m('Missing parameters in Trafodion config file')

    if not cfgs['traf_home'] or not cmd_output(
            '%s ls %s' % (get_sudo_prefix(), cfgs['traf_home'])):
        err_m('Cannot find trafodion binary folder')
    # get trafodion user from traf_home path
    cfgs['traf_user'] = cfgs['traf_home'].split('/')[-2]
    if not cfgs['traf_user']:
        err_m('Cannot detect trafodion user')

    ### parse trafodion user's password
    cfgs['traf_shadow'] = cmd_output(
        "%s grep %s /etc/shadow |awk -F: '{print $2}'" %
        (get_sudo_prefix(), cfgs['traf_user']))

    def copy_files():
        # package trafodion binary into a tar file
        if not os.path.exists(TRAF_PKG_FILE):
            info(
                'Creating trafodion packages of %s, this will take a while ...'
                % cfgs['traf_home'])
            run_cmd_as_user(
                cfgs['traf_user'],
                'cd %s; tar czf %s ./* --exclude logs/* --exclude core.* --exclude tmp/*'
                % (cfgs['traf_home'], TRAF_PKG_FILE))
        else:
            info('Using existing trafodion package %s' % TRAF_PKG_FILE)

        info(
            'Copying trafodion files to new nodes, this will take a while ...')
        run_cmd('%s cp -rf %s/../.ssh /tmp' %
                (get_sudo_prefix(), cfgs['traf_home']))
        run_cmd('%s chmod -R 755 /tmp/.ssh' % get_sudo_prefix())
        traf_ssh_folder = '/tmp/.ssh'

        hbase_trx_file = cmd_output('ls %s/hbase-trx-*' %
                                    cfgs['hbase_lib_path'])
        trafodion_utility_file = cmd_output('ls %s/trafodion-utility-*' %
                                            cfgs['hbase_lib_path'])

        files = [
            TRAF_CFG_FILE, TRAF_PKG_FILE, traf_ssh_folder, hbase_trx_file,
            trafodion_utility_file
        ]

        remote_insts = [Remote(h, pwd=pwd) for h in new_nodes]
        threads = [
            Thread(target=r.copy, args=(files, '/tmp')) for r in remote_insts
        ]
        for thread in threads:
            thread.start()
        for thread in threads:
            thread.join()

        for r in remote_insts:
            if r.rc != 0: err_m('Failed to copy files to %s' % r.host)

    ### copy trafodion_config/trafodion-package/hbase-trx to the new nodes
    copy_files()

    ### set parameters
    if cfgs['enable_ha'].upper() == 'true':
        g('dcs_backup_nodes')
        cfgs['dcs_ha'] = 'Y'
    else:
        cfgs['dcs_ha'] = 'N'

    if cfgs['trafodion_enable_authentication'] == 'YES':
        cfgs['ldap_security'] = 'Y'
    else:
        cfgs['ldap_security'] = 'N'

    if cfgs['secure_hadoop'].upper() == 'Y':
        g('kdc_server')
        g('admin_principal')
        g('kdcadmin_pwd')

    #TODO: offline support
    cfgs['offline_mode'] = 'N'

    format_output('AddNode sub scripts Start')

    ### run addNode script on new nodes ###
    cfgs['node_list'] = ','.join(new_nodes)
    info('Running add node setup on new node(s) [%s] ...' % cfgs['node_list'])
    wrapper.run(cfgs, options, mode='addnodes_new', pwd=pwd)

    ### run dcs setup script on all nodes ###
    # get current trafodion node list
    current_nodes = cmd_output('%s su - %s -c "trafconf -name 2>/dev/null"' %
                               (get_sudo_prefix(), cfgs['traf_user'])).split()
    all_nodes = list(set(new_nodes + current_nodes))
    cfgs['node_list'] = ','.join(all_nodes)
    info('Running dcs setup on all node(s) [%s] ...' % cfgs['node_list'])
    wrapper.run(cfgs, options, mode='addnodes_all', pwd=pwd)

    ### do sqshell node add/up, sqregen
    # check if trafodion is running
    mon_process = cmd_output('ps -ef|grep -v grep|grep -c "monitor COLD"')
    if int(mon_process) > 0:
        info('Trafodion instance is up, adding node in sqshell ...')

        # cores=0-1;processors=2;roles=connection,aggregation,storage
        sqconfig_ptr = cmd_output(
            '%s su - %s -c "trafconf -node|sed -n 2p|cut -d\\\";\\\" -f3-5"' %
            (get_sudo_prefix(), cfgs['traf_user']))
        for node in new_nodes:
            info('adding node [%s] in sqshell ...' % node)
            run_cmd_as_user(
                cfgs['traf_user'],
                'echo "node add {node-name %s,%s}" | sqshell -a' %
                (node, sqconfig_ptr))
            run_cmd_as_user(cfgs['traf_user'],
                            'echo "node up %s" | sqshell -a' % node)
            ok('Node [%s] added!' % node)

        info('Starting DCS on new nodes ...')
        run_cmd_as_user(cfgs['traf_user'], 'dcsstart')
    else:
        info('Trafodion instance is not up, do sqgen ...')
        run_cmd_as_user(cfgs['traf_user'],
                        'rm %s/sqconfig.db' % cfgs['traf_var'])
        run_cmd_as_user(cfgs['traf_user'], 'sqgen')
        ok('Setup completed. You need to start trafodion manually')

    ### clean up
    run_cmd('%s rm -rf /tmp/.ssh' % get_sudo_prefix())
    run_cmd('%s rm -rf %s' % (get_sudo_prefix(), TRAF_PKG_FILE))

    format_output('AddNode Complete')
    info(
        'NOTICE: You need to manually restart RegionServer on newly added nodes to take effect'
    )
예제 #9
0
def main():
    """ db_installer main loop """
    global cfgs
    format_output('Trafodion Installation ToolKit')

    # handle parser option
    options = get_options()

    if options.build and options.cfgfile:
        log_err('Wrong parameter, cannot specify both --build and --config-file')

    if options.build and options.offline:
        log_err('Wrong parameter, cannot specify both --build and --offline')

    if options.cfgfile:
        if not os.path.exists(options.cfgfile):
            log_err('Cannot find config file \'%s\'' % options.cfgfile)
        config_file = options.cfgfile
    else:
        config_file = DBCFG_FILE

    if options.pwd:
        pwd = getpass.getpass('Input remote host SSH Password: '******''

    # not specified config file and default config file doesn't exist either
    p = ParseInI(config_file, 'dbconfigs')
    if options.build or (not os.path.exists(config_file)):
        if options.build: format_output('DryRun Start')
        user_input(options, prompt_mode=True, pwd=pwd)

        # save config file as json format
        print '\n** Generating config file to save configs ... \n'
        p.save(cfgs)
    # config file exists
    else:
        print '\n** Loading configs from config file ... \n'
        cfgs = p.load()
        if options.offline and cfgs['offline_mode'] != 'Y':
            log_err('To enable offline mode, must set "offline_mode = Y" in config file')
        user_input(options, prompt_mode=False, pwd=pwd)

    if options.reinstall:
        cfgs['reinstall'] = 'Y'

    if options.offline:
        http_start(cfgs['local_repo_dir'], cfgs['repo_http_port'])
    else:
        cfgs['offline_mode'] = 'N'

    if not options.build:
        format_output('Installation Start')

        ### perform actual installation ###
        wrapper.run(cfgs, options, pwd=pwd)

        format_output('Installation Complete')

        if options.offline: http_stop()

        # rename default config file when successfully installed
        # so next time user can input new variables for a new install
        # or specify the backup config file to install again
        try:
            # only rename default config file
            ts = time.strftime('%y%m%d_%H%M')
            if config_file == DBCFG_FILE and os.path.exists(config_file):
                os.rename(config_file, config_file + '.bak' + ts)
        except OSError:
            log_err('Cannot rename config file')
    else:
        format_output('DryRun Complete')

    # remove temp config file
    if os.path.exists(DBCFG_TMP_FILE): os.remove(DBCFG_TMP_FILE)
예제 #10
0
def main():
    """ add_nodes main loop """
    cfgs = defaultdict(str)

    # handle parser option
    options = get_options()
    if not options.nodes:
        err_m('Must specifiy the node names using \'--nodes\' option')

    # get node list from user input
    new_nodes = expNumRe(options.nodes)
    if not new_nodes:
        err_m('Incorrect format')

    if options.pwd:
        pwd = getpass.getpass('Input remote host SSH Password: '******''

    u = UserInput(options, pwd)
    g = lambda n: u.get_input(n, cfgs[n], prompt_mode=prompt_mode)

    format_output('Trafodion Elastic Add Nodes Script')

    ### read configs from current trafodion_config and save it to cfgs
    if os.path.exists(TRAF_CFG_FILE):
        with open(TRAF_CFG_FILE, 'r') as f:
            traf_cfgs = f.readlines()
        for traf_cfg in traf_cfgs:
            if not traf_cfg.strip(): continue
            key, value = traf_cfg.replace('export ', '').split('=')
            value = value.replace('"','')
            value = value.replace('\n','')
            cfgs[key.lower()] = value
    else:
        err_m('Cannot find %s, be sure to run this script on one of trafodion nodes' % TRAF_CFG_FILE)

    ### config check
    if not cfgs['hbase_lib_path'] or not cfgs['traf_version']:
        err_m('Missing parameters in Trafodion config file')

    if not cfgs['traf_home'] or not cmd_output('%s ls %s' % (get_sudo_prefix(), cfgs['traf_home'])):
        err_m('Cannot find trafodion binary folder')
    # get trafodion user from traf_home path
    cfgs['traf_user'] = cfgs['traf_home'].split('/')[-2]
    if not cfgs['traf_user']:
        err_m('Cannot detect trafodion user')

    ### parse trafodion user's password
    cfgs['traf_shadow'] = cmd_output("%s grep %s /etc/shadow |awk -F: '{print $2}'" % (get_sudo_prefix(), cfgs['traf_user']))

    def copy_files():
        # package trafodion binary into a tar file
        if not os.path.exists(TRAF_PKG_FILE):
            info('Creating trafodion packages of %s, this will take a while ...' % cfgs['traf_home'])
            run_cmd_as_user(cfgs['traf_user'], 'cd %s; tar czf %s ./* --exclude logs/* --exclude core.* --exclude tmp/*' % (cfgs['traf_home'], TRAF_PKG_FILE))
        else:
            info('Using existing trafodion package %s' % TRAF_PKG_FILE)

        info('Copying trafodion files to new nodes, this will take a while ...')
        run_cmd('%s cp -rf %s/../.ssh /tmp' % (get_sudo_prefix(), cfgs['traf_home']))
        run_cmd('%s chmod -R 755 /tmp/.ssh' % get_sudo_prefix())
        traf_ssh_folder = '/tmp/.ssh'

        hbase_trx_file = cmd_output('ls %s/hbase-trx-*' % cfgs['hbase_lib_path'])
        trafodion_utility_file = cmd_output('ls %s/trafodion-utility-*' % cfgs['hbase_lib_path'])

        files = [TRAF_CFG_FILE, TRAF_PKG_FILE, traf_ssh_folder, hbase_trx_file, trafodion_utility_file]

        remote_insts = [Remote(h, pwd=pwd) for h in new_nodes]
        threads = [Thread(target=r.copy, args=(files, '/tmp')) for r in remote_insts]
        for thread in threads: thread.start()
        for thread in threads: thread.join()

        for r in remote_insts:
            if r.rc != 0: err_m('Failed to copy files to %s' % r.host)

    ### copy trafodion_config/trafodion-package/hbase-trx to the new nodes
    copy_files()

    ### set parameters
    if cfgs['enable_ha'].upper() == 'true':
        g('dcs_backup_nodes')
        cfgs['dcs_ha'] = 'Y'
    else:
        cfgs['dcs_ha'] = 'N'

    if cfgs['trafodion_enable_authentication'] == 'YES':
        cfgs['ldap_security'] = 'Y'
    else:
        cfgs['ldap_security'] = 'N'

    if cfgs['secure_hadoop'].upper() == 'Y':
        g('kdc_server')
        g('admin_principal')
        g('kdcadmin_pwd')

    #TODO: offline support
    cfgs['offline_mode'] = 'N'


    format_output('AddNode sub scripts Start')

    ### run addNode script on new nodes ###
    cfgs['node_list'] = ','.join(new_nodes)
    info('Running add node setup on new node(s) [%s] ...' % cfgs['node_list'])
    wrapper.run(cfgs, options, mode='addnodes_new', pwd=pwd)

    ### run dcs setup script on all nodes ###
    # get current trafodion node list
    current_nodes = cmd_output('%s su - %s -c "trafconf -name 2>/dev/null"' % (get_sudo_prefix(), cfgs['traf_user'])).split()
    all_nodes = list(set(new_nodes + current_nodes))
    cfgs['node_list'] = ','.join(all_nodes)
    info('Running dcs setup on all node(s) [%s] ...' % cfgs['node_list'])
    wrapper.run(cfgs, options, mode='addnodes_all', pwd=pwd)

    ### do sqshell node add/up, sqregen
    # check if trafodion is running
    mon_process = cmd_output('ps -ef|grep -v grep|grep -c "monitor COLD"')
    if int(mon_process) > 0:
        info('Trafodion instance is up, adding node in sqshell ...')

        # cores=0-1;processors=2;roles=connection,aggregation,storage
        sqconfig_ptr = cmd_output('%s su - %s -c "trafconf -node|sed -n 2p|cut -d\\\";\\\" -f3-5"' % (get_sudo_prefix(), cfgs['traf_user']))
        for node in new_nodes:
            info('adding node [%s] in sqshell ...' % node)
            run_cmd_as_user(cfgs['traf_user'], 'echo "node add {node-name %s,%s}" | sqshell -a' % (node, sqconfig_ptr))
            run_cmd_as_user(cfgs['traf_user'], 'echo "node up %s" | sqshell -a' % node)
            ok('Node [%s] added!' % node)

        info('Starting DCS on new nodes ...')
        run_cmd_as_user(cfgs['traf_user'], 'dcsstart')
    else:
        info('Trafodion instance is not up, do sqgen ...')
        run_cmd_as_user(cfgs['traf_user'], 'rm %s/sql/scripts/sqconfig.db' % cfgs['traf_home'])
        run_cmd_as_user(cfgs['traf_user'], 'sqgen')
        ok('Setup completed. You need to start trafodion manually')

    ### clean up
    run_cmd('%s rm -rf /tmp/.ssh' % get_sudo_prefix())
    run_cmd('%s rm -rf %s' % (get_sudo_prefix(), TRAF_PKG_FILE))

    format_output('AddNode Complete')
    info('NOTICE: You need to manually restart RegionServer on newly added nodes to take effect')