def run():
    dbcfgs = json.loads(dbcfgs_json)
    if 'APACHE' in dbcfgs['distro']:
        modcfgs = ParseJson(MODCFG_FILE).load()
        MOD_CFGS = modcfgs['MOD_CFGS']

        hdfs_xml_file = dbcfgs['hdfs_xml_file']
        hbase_xml_file = dbcfgs['hbase_xml_file']

        hbasexml = ParseXML(hbase_xml_file)
        for key, value in MOD_CFGS['hbase-site'].items():
            hbasexml.add_property(key, value)
        hbasexml.write_xml()

        hdfsxml = ParseXML(hdfs_xml_file)
        for key, value in MOD_CFGS['hdfs-site'].items():
            hdfsxml.add_property(key, value)
        hdfsxml.write_xml()

        print 'Apache Hadoop modification completed'
        first_node = dbcfgs['first_rsnode']
        local_host = socket.gethostname()
        if first_node in local_host:
            hadoop_home = dbcfgs['hadoop_home']
            hbase_home = dbcfgs['hbase_home']
            # stop
            run_cmd(hbase_home + '/bin/stop-hbase.sh')
            run_cmd(hadoop_home + '/sbin/stop-dfs.sh')
            # start
            run_cmd(hadoop_home + '/sbin/start-dfs.sh')
            run_cmd(hbase_home + '/bin/start-hbase.sh')

            print 'Apache Hadoop restart completed'
    else:
        print 'no apache distribution found, skipping'
 def get_secure_hadoop(self):
     if self.dbcfgs.has_key('hadoop_home'): # apache distro
         CORE_SITE_XML = '%s/etc/hadoop/core-site.xml' % self.dbcfgs['hadoop_home']
     else:
         CORE_SITE_XML = '/etc/hadoop/conf/core-site.xml'
     p = ParseXML(CORE_SITE_XML)
     return p.get_property('hadoop.security.authentication')
示例#3
0
 def get_secure_hadoop(self):
     if self.dbcfgs.has_key('hadoop_home'):  # apache distro
         CORE_SITE_XML = '%s/etc/hadoop/core-site.xml' % self.dbcfgs[
             'hadoop_home']
     else:
         CORE_SITE_XML = '/etc/hadoop/conf/core-site.xml'
     p = ParseXML(CORE_SITE_XML)
     return p.get_property('hadoop.security.authentication')
    def _get_core_site_info(self, name):
        if self.dbcfgs.has_key('hadoop_home'): # apache distro
            core_site_xml = '%s/etc/hadoop/core-site.xml' % self.dbcfgs['hadoop_home']
        else:
            core_site_xml = DEF_CORE_SITE_XML

        if os.path.exists(core_site_xml):
            p = ParseXML(core_site_xml)
            return p.get_property(name)
        else:
            return NA
    def _get_core_site_info(self, name):
        if self.dbcfgs.has_key('hadoop_home'):  # apache distro
            CORE_SITE_XML = '%s/etc/hadoop/core-site.xml' % self.dbcfgs[
                'hadoop_home']
        else:
            CORE_SITE_XML = '/etc/hadoop/conf/core-site.xml'

        if os.path.exists(CORE_SITE_XML):
            p = ParseXML(CORE_SITE_XML)
            return p.get_property(name)
        else:
            return NA
示例#6
0
def run():
    """ create trafodion user, bashrc, setup passwordless SSH """
    dbcfgs = json.loads(dbcfgs_json)

    distro = dbcfgs['distro']
    if 'CDH' in distro:
        hadoop_type = 'cloudera'
    elif 'HDP' in distro:
        hadoop_type = 'hortonworks'
    elif 'APACHE' in distro:
        hadoop_type = 'apache'

    home_dir = get_default_home()
    # customize trafodion home dir
    if dbcfgs.has_key('home_dir') and dbcfgs['home_dir']:
        home_dir = dbcfgs['home_dir']

    traf_user = dbcfgs['traf_user']
    traf_user_dir = '%s/%s' % (home_dir, traf_user)
    traf_dirname = dbcfgs['traf_dirname']
    traf_home = '%s/%s' % (traf_user_dir, traf_dirname)
    traf_log = dbcfgs['traf_log']
    traf_var = dbcfgs['traf_var']

    hbase_xml_file = dbcfgs['hbase_xml_file']
    auth_key_file = '%s/.ssh/authorized_keys' % traf_user_dir
    ssh_cfg_file = '%s/.ssh/config' % traf_user_dir
    ulimits_file = '/etc/security/limits.d/%s.conf' % traf_user

    # create trafodion user and group
    if cmd_output('getent passwd %s' % traf_user):
        # trafodion user exists, set actual trafodion group
        traf_group = cmd_output('id -ng %s' % traf_user)
    else:
        # default trafodion group
        traf_group = traf_user
        if not cmd_output('getent group %s' % traf_group):
            run_cmd('groupadd %s > /dev/null 2>&1' % traf_group)
        traf_pwd = dbcfgs['traf_pwd']
        run_cmd('useradd --shell /bin/bash -m %s -g %s --home %s --password "$(openssl passwd %s)"' % (traf_user, traf_group, traf_user_dir, traf_pwd))
    # hbase group is generally either hbase or hadoop, depending on distro
    if cmd_output('getent group hbase'):
        cmd_output('/usr/sbin/usermod -a -G hbase %s' % traf_user)
    if cmd_output('getent group hadoop'):
        cmd_output('/usr/sbin/usermod -a -G hadoop %s' % traf_user)
    if cmd_output('getent group hive'):
        cmd_output('/usr/sbin/usermod -a -G hive %s' % traf_user)

    if not os.path.exists(traf_user_dir):
        run_cmd('mkdir -p %s' % traf_user_dir)
        run_cmd('chmod 700 %s' % traf_user_dir)

    # set ssh key
    run_cmd_as_user(traf_user, 'echo -e "y" | ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa')
    # the key is generated in copy_file script running on the installer node
    run_cmd('cp %s{,.pub} %s/.ssh/' % (SSHKEY_FILE, traf_user_dir))

    run_cmd_as_user(traf_user, 'cat ~/.ssh/id_rsa.pub > %s' % auth_key_file)
    run_cmd('chmod 644 %s' % auth_key_file)

    ssh_cfg = 'StrictHostKeyChecking=no\nNoHostAuthenticationForLocalhost=yes\n'
    with open(ssh_cfg_file, 'w') as f:
        f.write(ssh_cfg)
    run_cmd('chmod 600 %s' % ssh_cfg_file)

    run_cmd('chown -R %s:%s %s/.ssh/' % (traf_user, traf_group, traf_user_dir))

    hb = ParseXML(hbase_xml_file)
    zk_nodes = hb.get_property('hbase.zookeeper.quorum')
    zk_port = hb.get_property('hbase.zookeeper.property.clientPort')
    # set trafodion_config
    nodes = dbcfgs['node_list'].split(',')
    trafodion_config = """
export TRAF_HOME="%s"
export TRAF_VAR="%s"
export TRAF_CONF="%s"
export TRAF_LOG="%s"
export JAVA_HOME="%s"
export node_count="%s"
export HADOOP_TYPE="%s"
export ENABLE_HA="%s"
export ZOOKEEPER_NODES="%s"
export ZOOKEEPER_PORT="%s"
export SECURE_HADOOP="%s"
export CLUSTERNAME="%s"
""" % (traf_home, traf_var, TRAF_CFG_DIR, traf_log, dbcfgs['java_home'], str(len(nodes)), hadoop_type, dbcfgs['enable_ha'],
       zk_nodes, zk_port, dbcfgs['secure_hadoop'], socket.gethostname())

    # save additonal configs for elastic
    trafodion_config += """
export hbase_xml_file="%s"
export hbase_lib_path="%s"
export traf_user="******"
export traf_version="%s"
export dcs_cnt_per_node="%s"
""" % (dbcfgs['hbase_xml_file'], dbcfgs['hbase_lib_path'], dbcfgs['traf_user'], dbcfgs['traf_version'], dbcfgs['dcs_cnt_per_node'])

    # save additonal configs for multi instance support
    trafodion_config += """
export TRAF_CLUSTER_NAME="%s"
export TRAF_INSTANCE_NAME="%s"
export TRAF_CLUSTER_ID="%s"
export TRAF_INSTANCE_ID="%s"
export TRAF_ROOT_ZNODE="/%s"
""" % (dbcfgs['cluster_name'], dbcfgs['traf_instance_name'], dbcfgs['traf_cluster_id'], dbcfgs['traf_instance_id'], dbcfgs['traf_user'])

    run_cmd('mkdir -p %s' % TRAF_CFG_DIR)
    write_file(TRAF_CFG_FILE, trafodion_config)

    if 'APACHE' in distro:
        extra_config = """
export HADOOP_PREFIX=%s
export HBASE_HOME=%s
export HIVE_HOME=%s
export PATH=$PATH:$HADOOP_PREFIX/bin:$HADOOP_PREFIX/sbin:$HBASE_HOME/bin
        """ % (dbcfgs['hadoop_home'], dbcfgs['hbase_home'], dbcfgs['hive_home'])
        append_file(TRAFODION_CFG_FILE, extra_config)

    # set permission
    run_cmd('chown -R %s:%s %s*' % (traf_user, traf_group, TRAF_CFG_DIR))

    # set ulimits for trafodion user
    ulimits_config = '''
# Trafodion settings
%s   soft   core unlimited
%s   hard   core unlimited
%s   soft   memlock unlimited
%s   hard   memlock unlimited
%s   soft   nofile 32768
%s   hard   nofile 65536
%s   soft   nproc 100000
%s   hard   nproc 100000
%s   soft nofile 8192
%s   hard nofile 65535
''' % ((traf_user,) * 10)

    write_file(ulimits_file, ulimits_config)

    # change permission for hsperfdata
    if os.path.exists(TRAF_HSPERFDATA_FILE):
        run_cmd('chown -R %s:%s %s' % (traf_user, traf_group, TRAF_HSPERFDATA_FILE))

    # clean up unused key file at the last step
    run_cmd('rm -rf %s{,.pub}' % SSHKEY_FILE)

    print 'Setup trafodion user successfully!'
def run():
    dbcfgs = json.loads(dbcfgs_json)
    if 'APACHE' in dbcfgs['distro']:
        modcfgs = ParseJson(MODCFG_FILE).load()
        MOD_CFGS = modcfgs['MOD_CFGS']

        hdfs_xml_file = dbcfgs['hdfs_xml_file']
        hbase_xml_file = dbcfgs['hbase_xml_file']

        hbasexml = ParseXML(hbase_xml_file)
        for key, value in MOD_CFGS['hbase-site'].items():
            hbasexml.add_property(key, value)
        hbasexml.write_xml()

        hdfsxml = ParseXML(hdfs_xml_file)
        for key, value in MOD_CFGS['hdfs-site'].items():
            hdfsxml.add_property(key, value)
        hdfsxml.write_xml()

        print 'Apache Hadoop modification completed'
        first_node = dbcfgs['first_rsnode']
        local_host = socket.gethostname()
        if first_node in local_host:
            hadoop_home = dbcfgs['hadoop_home']
            hbase_home = dbcfgs['hbase_home']
            # stop
            run_cmd(hbase_home + '/bin/stop-hbase.sh')
            run_cmd(hadoop_home + '/sbin/stop-dfs.sh')
            # start
            run_cmd(hadoop_home + '/sbin/start-dfs.sh')
            run_cmd(hbase_home + '/bin/start-hbase.sh')

            print 'Apache Hadoop restart completed'
    else:
        print 'no apache distribution found, skipping'
示例#8
0
def run():
    """ create trafodion user, bashrc, setup passwordless SSH """
    dbcfgs = json.loads(dbcfgs_json)

    DISTRO = dbcfgs['distro']
    if 'CDH' in DISTRO:
        hadoop_type = 'cloudera'
    elif 'HDP' in DISTRO:
        hadoop_type = 'hortonworks'
    elif 'APACHE' in DISTRO:
        hadoop_type = 'apache'

    TRAF_USER = dbcfgs['traf_user']
    TRAF_GROUP = TRAF_USER
    HOME_DIR = cmd_output('cat /etc/default/useradd |grep HOME |cut -d "=" -f 2').strip()
    # customize trafodion home dir
    if dbcfgs.has_key('home_dir') and dbcfgs['home_dir']:
        HOME_DIR = dbcfgs['home_dir']

    TRAF_USER_DIR = '%s/%s' % (HOME_DIR, TRAF_USER)
    TRAF_DIRNAME = dbcfgs['traf_dirname']
    TRAF_HOME = '%s/%s' % (TRAF_USER_DIR, TRAF_DIRNAME)

    TRAFODION_CFG_DIR = '/etc/trafodion/'
    TRAFODION_CFG_FILE = '/etc/trafodion/trafodion_config'
    HBASE_XML_FILE = dbcfgs['hbase_xml_file']
    KEY_FILE = '/tmp/id_rsa'
    AUTH_KEY_FILE = '%s/.ssh/authorized_keys' % TRAF_USER_DIR
    SSH_CFG_FILE = '%s/.ssh/config' % TRAF_USER_DIR
    ULIMITS_FILE = '/etc/security/limits.d/%s.conf' % TRAF_USER
    HSPERFDATA_FILE = '/tmp/hsperfdata_trafodion'

    # create trafodion user and group
    if not cmd_output('getent group %s' % TRAF_GROUP):
        run_cmd('groupadd %s > /dev/null 2>&1' % TRAF_GROUP)

    if not cmd_output('getent passwd %s' % TRAF_USER):
        TRAF_PWD = dbcfgs['traf_pwd']
        run_cmd('useradd --shell /bin/bash -m %s -g %s --home %s --password "$(openssl passwd %s)"' % (TRAF_USER, TRAF_GROUP, TRAF_USER_DIR, TRAF_PWD))
    elif not os.path.exists(TRAF_USER_DIR):
        run_cmd('mkdir -p %s' % TRAF_USER_DIR)
        run_cmd('chmod 700 %s' % TRAF_USER_DIR)

    # set ssh key
    run_cmd_as_user(TRAF_USER, 'echo -e "y" | ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa')
    # the key is generated in copy_file script running on the installer node
    run_cmd('cp %s{,.pub} %s/.ssh/' % (KEY_FILE, TRAF_USER_DIR))

    run_cmd_as_user(TRAF_USER, 'cat ~/.ssh/id_rsa.pub > %s' % AUTH_KEY_FILE)
    run_cmd('chmod 644 %s' % AUTH_KEY_FILE)

    ssh_cfg = 'StrictHostKeyChecking=no\nNoHostAuthenticationForLocalhost=yes\n'
    with open(SSH_CFG_FILE, 'w') as f:
        f.write(ssh_cfg)
    run_cmd('chmod 600 %s' % SSH_CFG_FILE)

    run_cmd('chown -R %s:%s %s/.ssh/' % (TRAF_USER, TRAF_GROUP, TRAF_USER_DIR))

    hb = ParseXML(HBASE_XML_FILE)
    zk_nodes = hb.get_property('hbase.zookeeper.quorum')
    zk_port = hb.get_property('hbase.zookeeper.property.clientPort')
    # set trafodion_config
    nodes = dbcfgs['node_list'].split(',')
    trafodion_config = """
export TRAF_HOME="%s"
export MY_SQROOT=$TRAF_HOME # for compatibility
export JAVA_HOME="%s"
export NODE_LIST="%s"
export MY_NODES="%s"
export node_count="%s"
export HADOOP_TYPE="%s"
export ENABLE_HA="%s"
export ZOOKEEPER_NODES="%s"
export ZOOKEEPER_PORT="%s"
export SECURE_HADOOP="%s"
""" % (TRAF_HOME, dbcfgs['java_home'], ' '.join(nodes), ' -w ' + ' -w '.join(nodes),
       str(len(nodes)), hadoop_type, dbcfgs['enable_ha'], zk_nodes, zk_port, dbcfgs['secure_hadoop'])

    run_cmd('mkdir -p %s' % TRAFODION_CFG_DIR)
    write_file(TRAFODION_CFG_FILE, trafodion_config)

    if 'APACHE' in DISTRO:
        extra_config = """
export HADOOP_PREFIX=%s
export HBASE_HOME=%s
export PATH=$PATH:$HADOOP_PREFIX/bin:$HADOOP_PREFIX/sbin:$HBASE_HOME/bin
        """ % (dbcfgs['hadoop_home'], dbcfgs['hbase_home'])
        append_file(TRAFODION_CFG_FILE, extra_config)

    # set permission
    run_cmd('chown -R %s:%s %s*' % (TRAF_USER, TRAF_GROUP, TRAFODION_CFG_DIR))


    # set ulimits for trafodion user
    ulimits_config = '''
# Trafodion settings
%s   soft   core unlimited
%s   hard   core unlimited
%s   soft   memlock unlimited
%s   hard   memlock unlimited
%s   soft   nofile 32768
%s   hard   nofile 65536
%s   soft   nproc 100000
%s   hard   nproc 100000
%s   soft nofile 8192
%s   hard nofile 65535
''' % ((TRAF_USER,) * 10)

    write_file(ULIMITS_FILE, ulimits_config)

    # change permission for hsperfdata
    if os.path.exists(HSPERFDATA_FILE):
        run_cmd('chown -R %s:%s %s' % (TRAF_USER, TRAF_GROUP, HSPERFDATA_FILE))

    # clean up unused key file at the last step
    run_cmd('rm -rf %s{,.pub}' % KEY_FILE)

    print 'Setup trafodion user successfully!'
示例#9
0
def run():
    dbcfgs = json.loads(dbcfgs_json)

    hbase_xml_file = dbcfgs['hbase_xml_file']

    dcs_conf_dir = '%s/dcs' % (TRAF_CFG_DIR)
    dcs_srv_file = dcs_conf_dir + '/servers'
    dcs_master_file = dcs_conf_dir + '/masters'
    dcs_site_file = dcs_conf_dir + '/dcs-site.xml'
    rest_site_file = '%s/rest/rest-site.xml' % (TRAF_CFG_DIR)

    ### dcs setting ###
    # servers
    nodes = dbcfgs['node_list'].split(',')
    dcs_cnt = dbcfgs['dcs_cnt_per_node']
    dcs_servers = ''
    for node in nodes:
        dcs_servers += '%s %s\n' % (node, dcs_cnt)

    write_file(dcs_srv_file, dcs_servers)

    ### modify dcs config files ###
    # modify master
    dcs_master = nodes[0]
    append_file(dcs_master_file, dcs_master+'\n')

    # modify dcs-site.xml
    net_interface = run_cmd('ip route |grep default|awk \'{print $5}\'')
    hb = ParseXML(hbase_xml_file)
    zk_hosts = hb.get_property('hbase.zookeeper.quorum')
    zk_port = hb.get_property('hbase.zookeeper.property.clientPort')

    p = ParseXML(dcs_site_file)
    p.add_property('dcs.zookeeper.property.clientPort', zk_port)
    p.add_property('dcs.zookeeper.quorum', zk_hosts)
    p.add_property('dcs.dns.interface', net_interface)

    if dbcfgs['dcs_ha'] == 'Y':
        dcs_floating_ip = dbcfgs['dcs_floating_ip']
        dcs_backup_nodes = dbcfgs['dcs_backup_nodes']
        p.add_property('dcs.master.floating.ip', 'true')
        p.add_property('dcs.master.floating.ip.external.interface', net_interface)
        p.add_property('dcs.master.floating.ip.external.ip.address', dcs_floating_ip)
        p.rm_property('dcs.dns.interface')

        # set DCS_MASTER_FLOATING_IP ENV for trafci
        dcs_floating_ip_cfg = 'export DCS_MASTER_FLOATING_IP=%s' % dcs_floating_ip
        append_file(TRAF_CFG_FILE, dcs_floating_ip_cfg)

        # modify master with backup master host
        for dcs_backup_node in dcs_backup_nodes.split(','):
            append_file(dcs_master_file, dcs_backup_node)

    p.write_xml()

    ### rest setting ###
    p = ParseXML(rest_site_file)
    p.add_property('rest.zookeeper.property.clientPort', zk_port)
    p.add_property('rest.zookeeper.quorum', zk_hosts)
    p.write_xml()

    ### run sqcertgen ###
    run_cmd('sqcertgen')
示例#10
0
def run():
    """ create trafodion user, bashrc, setup passwordless SSH """
    dbcfgs = json.loads(dbcfgs_json)

    distro = dbcfgs['distro']
    if 'CDH' in distro:
        hadoop_type = 'cloudera'
    elif 'HDP' in distro:
        hadoop_type = 'hortonworks'
    elif 'APACHE' in distro:
        hadoop_type = 'apache'

    home_dir = get_default_home()
    # customize trafodion home dir
    if dbcfgs.has_key('home_dir') and dbcfgs['home_dir']:
        home_dir = dbcfgs['home_dir']

    traf_user = dbcfgs['traf_user']
    traf_user_dir = '%s/%s' % (home_dir, traf_user)
    traf_dirname = dbcfgs['traf_dirname']
    traf_home = '%s/%s' % (traf_user_dir, traf_dirname)

    hbase_xml_file = dbcfgs['hbase_xml_file']
    auth_key_file = '%s/.ssh/authorized_keys' % traf_user_dir
    ssh_cfg_file = '%s/.ssh/config' % traf_user_dir
    ulimits_file = '/etc/security/limits.d/%s.conf' % traf_user

    # create trafodion user and group
    if cmd_output('getent passwd %s' % traf_user):
        # trafodion user exists, set actual trafodion group
        traf_group = cmd_output('id -ng %s' % traf_user)
    else:
        # default trafodion group
        traf_group = traf_user
        if not cmd_output('getent group %s' % traf_group):
            run_cmd('groupadd %s > /dev/null 2>&1' % traf_group)
        traf_pwd = dbcfgs['traf_pwd']
        run_cmd('useradd --shell /bin/bash -m %s -g %s --home %s --password "$(openssl passwd %s)"' % (traf_user, traf_group, traf_user_dir, traf_pwd))
    # hbase group is generally either hbase or hadoop, depending on distro
    if cmd_output('getent group hbase'):
        cmd_output('/usr/sbin/usermod -a -G hbase %s' % traf_user)
    if cmd_output('getent group hadoop'):
        cmd_output('/usr/sbin/usermod -a -G hadoop %s' % traf_user)
    if cmd_output('getent group hive'):
        cmd_output('/usr/sbin/usermod -a -G hive %s' % traf_user)

    if not os.path.exists(traf_user_dir):
        run_cmd('mkdir -p %s' % traf_user_dir)
        run_cmd('chmod 700 %s' % traf_user_dir)

    # set ssh key
    run_cmd_as_user(traf_user, 'echo -e "y" | ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa')
    # the key is generated in copy_file script running on the installer node
    run_cmd('cp %s{,.pub} %s/.ssh/' % (SSHKEY_FILE, traf_user_dir))

    run_cmd_as_user(traf_user, 'cat ~/.ssh/id_rsa.pub > %s' % auth_key_file)
    run_cmd('chmod 644 %s' % auth_key_file)

    ssh_cfg = 'StrictHostKeyChecking=no\nNoHostAuthenticationForLocalhost=yes\n'
    with open(ssh_cfg_file, 'w') as f:
        f.write(ssh_cfg)
    run_cmd('chmod 600 %s' % ssh_cfg_file)

    run_cmd('chown -R %s:%s %s/.ssh/' % (traf_user, traf_group, traf_user_dir))

    hb = ParseXML(hbase_xml_file)
    zk_nodes = hb.get_property('hbase.zookeeper.quorum')
    zk_port = hb.get_property('hbase.zookeeper.property.clientPort')
    # set trafodion_config
    nodes = dbcfgs['node_list'].split(',')
    trafodion_config = """
export TRAF_HOME="%s"
export TRAF_VAR=$TRAF_HOME/tmp
export MY_SQROOT=$TRAF_HOME # for compatibility
export JAVA_HOME="%s"
export node_count="%s"
export HADOOP_TYPE="%s"
export ENABLE_HA="%s"
export ZOOKEEPER_NODES="%s"
export ZOOKEEPER_PORT="%s"
export SECURE_HADOOP="%s"
export CLUSTERNAME="%s"
""" % (traf_home, dbcfgs['java_home'], str(len(nodes)), hadoop_type, dbcfgs['enable_ha'],
       zk_nodes, zk_port, dbcfgs['secure_hadoop'], socket.gethostname())

    # save additonal configs for elastic
    trafodion_config += """
export hbase_xml_file="%s"
export hbase_lib_path="%s"
export traf_user="******"
export traf_version="%s"
export dcs_cnt_per_node="%s"
""" % (dbcfgs['hbase_xml_file'], dbcfgs['hbase_lib_path'], dbcfgs['traf_user'], dbcfgs['traf_version'], dbcfgs['dcs_cnt_per_node'])

    run_cmd('mkdir -p %s' % TRAF_CFG_DIR)
    write_file(TRAF_CFG_FILE, trafodion_config)

    if 'APACHE' in distro:
        extra_config = """
export HADOOP_PREFIX=%s
export HBASE_HOME=%s
export HIVE_HOME=%s
export PATH=$PATH:$HADOOP_PREFIX/bin:$HADOOP_PREFIX/sbin:$HBASE_HOME/bin
        """ % (dbcfgs['hadoop_home'], dbcfgs['hbase_home'], dbcfgs['hive_home'])
        append_file(TRAFODION_CFG_FILE, extra_config)

    # set permission
    run_cmd('chown -R %s:%s %s*' % (traf_user, traf_group, TRAF_CFG_DIR))

    # set ulimits for trafodion user
    ulimits_config = '''
# Trafodion settings
%s   soft   core unlimited
%s   hard   core unlimited
%s   soft   memlock unlimited
%s   hard   memlock unlimited
%s   soft   nofile 32768
%s   hard   nofile 65536
%s   soft   nproc 100000
%s   hard   nproc 100000
%s   soft nofile 8192
%s   hard nofile 65535
''' % ((traf_user,) * 10)

    write_file(ulimits_file, ulimits_config)

    # change permission for hsperfdata
    if os.path.exists(TRAF_HSPERFDATA_FILE):
        run_cmd('chown -R %s:%s %s' % (traf_user, traf_group, TRAF_HSPERFDATA_FILE))

    # clean up unused key file at the last step
    run_cmd('rm -rf %s{,.pub}' % SSHKEY_FILE)

    print 'Setup trafodion user successfully!'
示例#11
0
def run():
    dbcfgs = json.loads(dbcfgs_json)

    TRAF_HOME = os.environ['TRAF_HOME']
    TRAF_VER = dbcfgs['traf_version']
    HBASE_XML_FILE = dbcfgs['hbase_xml_file']

    DCS_CONF_DIR = '%s/dcs-%s/conf' % (TRAF_HOME, TRAF_VER)
    DCS_SRV_FILE = DCS_CONF_DIR + '/servers'
    DCS_MASTER_FILE = DCS_CONF_DIR + '/master'
    DCS_BKMASTER_FILE = DCS_CONF_DIR + '/backup-masters'
    DCS_SITE_FILE = DCS_CONF_DIR + '/dcs-site.xml'
    REST_SITE_FILE = '%s/rest-%s/conf/rest-site.xml' % (TRAF_HOME, TRAF_VER)

    ### dcs setting ###
    # servers
    nodes = dbcfgs['node_list'].split(',')
    dcs_cnt = dbcfgs['dcs_cnt_per_node']
    dcs_servers = ''
    for node in nodes:
        dcs_servers += '%s %s\n' % (node, dcs_cnt)

    write_file(DCS_SRV_FILE, dcs_servers)

    ### modify dcs config files ###
    # modify master
    dcs_master = nodes[0]
    append_file(DCS_MASTER_FILE, dcs_master)

    # modify dcs-site.xml
    net_interface = run_cmd('ip route |grep default|awk \'{print $5}\'')
    hb = ParseXML(HBASE_XML_FILE)
    zk_hosts = hb.get_property('hbase.zookeeper.quorum')
    zk_port = hb.get_property('hbase.zookeeper.property.clientPort')

    p = ParseXML(DCS_SITE_FILE)
    p.add_property('dcs.zookeeper.property.clientPort', zk_port)
    p.add_property('dcs.zookeeper.quorum', zk_hosts)
    p.add_property('dcs.dns.interface', net_interface)

    if dbcfgs['dcs_ha'] == 'Y':
        dcs_floating_ip = dbcfgs['dcs_floating_ip']
        dcs_backup_nodes = dbcfgs['dcs_backup_nodes']
        p.add_property('dcs.master.floating.ip', 'true')
        p.add_property('dcs.master.floating.ip.external.interface',
                       net_interface)
        p.add_property('dcs.master.floating.ip.external.ip.address',
                       dcs_floating_ip)
        p.rm_property('dcs.dns.interface')

        # modify backup_master
        write_file(DCS_BKMASTER_FILE, dcs_backup_nodes)

    p.write_xml()

    ### rest setting ###
    p = ParseXML(REST_SITE_FILE)
    p.add_property('rest.zookeeper.property.clientPort', zk_port)
    p.add_property('rest.zookeeper.quorum', zk_hosts)
    p.write_xml()

    ### run sqcertgen ###
    run_cmd('sqcertgen')
def run():
    dbcfgs = json.loads(dbcfgs_json)

    SQ_ROOT = os.environ['MY_SQROOT']
    TRAF_VER = dbcfgs['traf_version']
    HBASE_XML_FILE = dbcfgs['hbase_xml_file']

    DCS_INSTALL_ENV = 'export DCS_INSTALL_DIR=%s/dcs-%s' % (SQ_ROOT, TRAF_VER)
    REST_INSTALL_ENV = 'export REST_INSTALL_DIR=%s/rest-%s' % (SQ_ROOT,
                                                               TRAF_VER)

    DCS_CONF_DIR = '%s/dcs-%s/conf' % (SQ_ROOT, TRAF_VER)
    DCS_SRV_FILE = DCS_CONF_DIR + '/servers'
    DCS_MASTER_FILE = DCS_CONF_DIR + '/master'
    DCS_BKMASTER_FILE = DCS_CONF_DIR + '/backup-masters'
    DCS_ENV_FILE = DCS_CONF_DIR + '/dcs-env.sh'
    DCS_SITE_FILE = DCS_CONF_DIR + '/dcs-site.xml'
    REST_SITE_FILE = '%s/rest-%s/conf/rest-site.xml' % (SQ_ROOT, TRAF_VER)
    TRAFCI_FILE = SQ_ROOT + '/trafci/bin/trafci'
    SQENV_FILE = SQ_ROOT + '/sqenvcom.sh'

    ### dcs setting ###
    # servers
    nodes = dbcfgs['node_list'].split(',')
    dcs_cnt = dbcfgs['dcs_cnt_per_node']
    dcs_servers = ''
    for node in nodes:
        dcs_servers += '%s %s\n' % (node, dcs_cnt)

    write_file(DCS_SRV_FILE, dcs_servers)

    ### modify dcs config files ###
    # modify master
    dcs_master = nodes[0]
    append_file(DCS_MASTER_FILE, dcs_master)

    # modify sqenvcom.sh
    append_file(SQENV_FILE, DCS_INSTALL_ENV)
    append_file(SQENV_FILE, REST_INSTALL_ENV)

    # modify dcs-env.sh
    mod_file(DCS_ENV_FILE,
             {'.*DCS_MANAGES_ZK=.*': 'export DCS_MANAGES_ZK=false'})

    # modify trafci
    mod_file(TRAFCI_FILE, {'HNAME=.*': 'HNAME=%s:23400' % dcs_master})

    # modify dcs-site.xml
    net_interface = cmd_output(
        'netstat -rn | grep "^0.0.0.0" | awk \'{print $8}\'').strip()
    hb = ParseXML(HBASE_XML_FILE)
    zk_hosts = hb.get_property('hbase.zookeeper.quorum')
    zk_port = hb.get_property('hbase.zookeeper.property.clientPort')

    p = ParseXML(DCS_SITE_FILE)
    p.add_property('dcs.zookeeper.property.clientPort', zk_port)
    p.add_property('dcs.zookeeper.quorum', zk_hosts)
    p.add_property('dcs.dns.interface', net_interface)

    if dbcfgs['dcs_ha'] == 'Y':
        dcs_floating_ip = dbcfgs['dcs_floating_ip']
        dcs_backup_nodes = dbcfgs['dcs_backup_nodes']
        p.add_property('dcs.master.floating.ip', 'true')
        p.add_property('dcs.master.floating.ip.external.interface',
                       net_interface)
        p.add_property('dcs.master.floating.ip.external.ip.address',
                       dcs_floating_ip)
        p.rm_property('dcs.dns.interface')

        # modify backup_master
        write_file(DCS_BKMASTER_FILE, dcs_backup_nodes)

    p.write_xml()

    ### rest setting ###
    p = ParseXML(REST_SITE_FILE)
    p.add_property('rest.zookeeper.property.clientPort', zk_port)
    p.add_property('rest.zookeeper.quorum', zk_hosts)
    p.write_xml()

    ### run sqcertgen ###
    run_cmd('sqcertgen')
示例#13
0
def run():
    dbcfgs = json.loads(dbcfgs_json)

    SQ_ROOT = os.environ['MY_SQROOT']
    TRAF_VER = dbcfgs['traf_version']
    HBASE_XML_FILE = dbcfgs['hbase_xml_file']

    DCS_INSTALL_ENV = 'export DCS_INSTALL_DIR=%s/dcs-%s' % (SQ_ROOT, TRAF_VER)
    REST_INSTALL_ENV = 'export REST_INSTALL_DIR=%s/rest-%s' % (SQ_ROOT, TRAF_VER)

    DCS_CONF_DIR = '%s/dcs-%s/conf' % (SQ_ROOT, TRAF_VER)
    DCS_SRV_FILE = DCS_CONF_DIR + '/servers'
    DCS_MASTER_FILE = DCS_CONF_DIR + '/master'
    DCS_BKMASTER_FILE = DCS_CONF_DIR + '/backup-masters'
    DCS_ENV_FILE = DCS_CONF_DIR + '/dcs-env.sh'
    DCS_SITE_FILE = DCS_CONF_DIR + '/dcs-site.xml'
    REST_SITE_FILE = '%s/rest-%s/conf/rest-site.xml' % (SQ_ROOT, TRAF_VER)
    TRAFCI_FILE = SQ_ROOT + '/trafci/bin/trafci'
    SQENV_FILE = SQ_ROOT + '/sqenvcom.sh'

    ### dcs setting ###
    # servers
    nodes = dbcfgs['node_list'].split(',')
    dcs_cnt = dbcfgs['dcs_cnt_per_node']
    dcs_servers = ''
    for node in nodes:
        dcs_servers += '%s %s\n' % (node, dcs_cnt)

    write_file(DCS_SRV_FILE, dcs_servers)

    ### modify dcs config files ###
    # modify master
    dcs_master = nodes[0]
    append_file(DCS_MASTER_FILE, dcs_master)

    # modify sqenvcom.sh
    append_file(SQENV_FILE, DCS_INSTALL_ENV)
    append_file(SQENV_FILE, REST_INSTALL_ENV)

    # modify dcs-env.sh
    mod_file(DCS_ENV_FILE, {'.*DCS_MANAGES_ZK=.*':'export DCS_MANAGES_ZK=false'})

    # modify trafci
    mod_file(TRAFCI_FILE, {'HNAME=.*':'HNAME=%s:23400' % dcs_master})

    # modify dcs-site.xml
    net_interface = cmd_output('netstat -rn | grep "^0.0.0.0" | awk \'{print $8}\'').strip()
    hb = ParseXML(HBASE_XML_FILE)
    zk_hosts = hb.get_property('hbase.zookeeper.quorum')
    zk_port = hb.get_property('hbase.zookeeper.property.clientPort')

    p = ParseXML(DCS_SITE_FILE)
    p.add_property('dcs.zookeeper.property.clientPort', zk_port)
    p.add_property('dcs.zookeeper.quorum', zk_hosts)
    p.add_property('dcs.dns.interface', net_interface)

    if dbcfgs['dcs_ha'] == 'Y':
        dcs_floating_ip = dbcfgs['dcs_floating_ip']
        dcs_backup_nodes = dbcfgs['dcs_backup_nodes']
        p.add_property('dcs.master.floating.ip', 'true')
        p.add_property('dcs.master.floating.ip.external.interface', net_interface)
        p.add_property('dcs.master.floating.ip.external.ip.address', dcs_floating_ip)
        p.rm_property('dcs.dns.interface')

        # modify backup_master
        write_file(DCS_BKMASTER_FILE, dcs_backup_nodes)

    p.write_xml()

    ### rest setting ###
    p = ParseXML(REST_SITE_FILE)
    p.add_property('rest.zookeeper.property.clientPort', zk_port)
    p.add_property('rest.zookeeper.quorum', zk_hosts)
    p.write_xml()

    ### run sqcertgen ###
    run_cmd('sqcertgen')