def run(): hdfs_bin = "/usr/bin/hdfs" dbcfgs = json.loads(dbcfgs_json) DISTRO = dbcfgs["distro"] if "CDH" in DISTRO: parcel_lib = "/opt/cloudera/parcels/CDH/lib/hbase/lib" if os.path.exists(parcel_lib): hdfs_bin = "/opt/cloudera/parcels/CDH/bin/hdfs" elif "APACHE" in DISTRO: hdfs_bin = dbcfgs["hadoop_home"] + "/bin/hdfs" traf_loc = "/user/trafodion" traf_user = dbcfgs["traf_user"] hdfs_user = dbcfgs["hdfs_user"] hbase_user = dbcfgs["hbase_user"] run_cmd_as_user(hdfs_user, "%s dfsadmin -safemode wait" % hdfs_bin) run_cmd_as_user( hdfs_user, "%s dfs -mkdir -p %s/{trafodion_backups,bulkload,lobs} /hbase/archive /hbase-staging" % (hdfs_bin, traf_loc), ) run_cmd_as_user( hdfs_user, "%s dfs -chown -R %s:%s /hbase/archive /hbase-staging" % (hdfs_bin, hbase_user, hbase_user) ) run_cmd_as_user( hdfs_user, "%s dfs -chown -R %s:%s %s/{trafodion_backups,bulkload,lobs}" % (hdfs_bin, traf_user, traf_user, traf_loc), ) run_cmd_as_user(hdfs_user, "%s dfs -setfacl -R -m user:%s:rwx /hbase/archive" % (hdfs_bin, traf_user)) run_cmd_as_user(hdfs_user, "%s dfs -setfacl -R -m default:user:%s:rwx /hbase/archive" % (hdfs_bin, traf_user)) run_cmd_as_user(hdfs_user, "%s dfs -setfacl -R -m mask::rwx /hbase/archive" % hdfs_bin) # Grant all privileges to the Trafodion principal in HBase if dbcfgs["secure_hadoop"] == "Y": run_cmd('grant "%s", "RWXC" | sudo -u %s hbase shell > /tmp/hbase_shell.out' % (traf_user, hbase_user)) has_err = cmd_output("grep -c ERROR /tmp/hbase_shell.out") if int(has_err): err("Failed to grant HBase privileges to %s" % traf_user) run_cmd("rm /tmp/hbase_shell.out")
def run(): hdfs_bin = DEF_HDFS_BIN dbcfgs = json.loads(dbcfgs_json) distro = dbcfgs['distro'] if 'CDH' in distro: parcel_lib = PARCEL_HBASE_LIB if os.path.exists(parcel_lib): hdfs_bin = PARCEL_HDFS_BIN elif 'APACHE' in distro: hdfs_bin = dbcfgs['hadoop_home'] + '/bin/hdfs' traf_loc = '/user/trafodion' traf_user = dbcfgs['traf_user'] hdfs_user = dbcfgs['hdfs_user'] hbase_user = dbcfgs['hbase_user'] run_cmd_as_user(hdfs_user, '%s dfsadmin -safemode wait' % hdfs_bin) run_cmd_as_user( hdfs_user, '%s dfs -mkdir -p %s/{trafodion_backups,bulkload,lobs} /bulkload /lobs /hbase/archive /hbase-staging' % (hdfs_bin, traf_loc)) run_cmd_as_user( hdfs_user, '%s dfs -chown -R %s:%s /hbase/archive /hbase-staging' % (hdfs_bin, hbase_user, hbase_user)) run_cmd_as_user( hdfs_user, '%s dfs -chown -R %s:%s %s %s/{trafodion_backups,bulkload,lobs} /bulkload /lobs' % (hdfs_bin, traf_user, traf_user, traf_loc, traf_loc)) run_cmd_as_user( hdfs_user, '%s dfs -setfacl -R -m user:%s:rwx /hbase/archive' % (hdfs_bin, traf_user)) run_cmd_as_user( hdfs_user, '%s dfs -setfacl -R -m default:user:%s:rwx /hbase/archive' % (hdfs_bin, traf_user)) run_cmd_as_user( hdfs_user, '%s dfs -setfacl -R -m mask::rwx /hbase/archive' % hdfs_bin) # Grant all privileges to the Trafodion principal in HBase if dbcfgs['secure_hadoop'] == 'Y': run_cmd( 'echo "grant \'%s\', \'RWXC\'" | %s su - %s -s /bin/bash -c "hbase shell" > /tmp/hbase_shell.out' % (traf_user, get_sudo_prefix(), hbase_user)) has_err = cmd_output('grep -c ERROR /tmp/hbase_shell.out') if int(has_err): err('Failed to grant HBase privileges to %s' % traf_user) run_cmd('rm /tmp/hbase_shell.out')
def run(): hdfs_bin = '/usr/bin/hdfs' dbcfgs = json.loads(dbcfgs_json) DISTRO = dbcfgs['distro'] if 'CDH' in DISTRO: parcel_lib = '/opt/cloudera/parcels/CDH/lib/hbase/lib' if os.path.exists(parcel_lib): hdfs_bin = '/opt/cloudera/parcels/CDH/bin/hdfs' elif 'APACHE' in DISTRO: hdfs_bin = dbcfgs['hadoop_home'] + '/bin/hdfs' traf_loc = '/user/trafodion' traf_user = dbcfgs['traf_user'] hdfs_user = dbcfgs['hdfs_user'] hbase_user = dbcfgs['hbase_user'] run_cmd_as_user(hdfs_user, '%s dfsadmin -safemode wait' % hdfs_bin) run_cmd_as_user( hdfs_user, '%s dfs -mkdir -p %s/{trafodion_backups,bulkload,lobs} /bulkload /lobs /hbase/archive /hbase-staging' % (hdfs_bin, traf_loc)) run_cmd_as_user( hdfs_user, '%s dfs -chown -R %s:%s /hbase/archive /hbase-staging' % (hdfs_bin, hbase_user, hbase_user)) run_cmd_as_user( hdfs_user, '%s dfs -chown -R %s:%s %s/{trafodion_backups,bulkload,lobs} /bulkload /lobs' % (hdfs_bin, traf_user, traf_user, traf_loc)) run_cmd_as_user( hdfs_user, '%s dfs -setfacl -R -m user:%s:rwx /hbase/archive' % (hdfs_bin, traf_user)) run_cmd_as_user( hdfs_user, '%s dfs -setfacl -R -m default:user:%s:rwx /hbase/archive' % (hdfs_bin, traf_user)) run_cmd_as_user( hdfs_user, '%s dfs -setfacl -R -m mask::rwx /hbase/archive' % hdfs_bin) # Grant all privileges to the Trafodion principal in HBase if dbcfgs['secure_hadoop'] == 'Y': run_cmd( 'grant "%s", "RWXC" | sudo -u %s hbase shell > /tmp/hbase_shell.out' % (traf_user, hbase_user)) has_err = cmd_output('grep -c ERROR /tmp/hbase_shell.out') if int(has_err): err('Failed to grant HBase privileges to %s' % traf_user) run_cmd('rm /tmp/hbase_shell.out')
def run(): """ create trafodion user, bashrc, setup passwordless SSH """ dbcfgs = json.loads(dbcfgs_json) DISTRO = dbcfgs['distro'] if 'CDH' in DISTRO: hadoop_type = 'cloudera' elif 'HDP' in DISTRO: hadoop_type = 'hortonworks' elif 'APACHE' in DISTRO: hadoop_type = 'apache' TRAF_USER = dbcfgs['traf_user'] TRAF_PWD = dbcfgs['traf_pwd'] TRAF_GROUP = TRAF_USER TRAF_HOME = cmd_output( 'cat /etc/default/useradd |grep HOME |cut -d "=" -f 2').strip() TRAF_USER_DIR = '%s/%s' % (TRAF_HOME, TRAF_USER) SQ_ROOT = '%s/%s-%s' % (TRAF_USER_DIR, dbcfgs['traf_basename'], dbcfgs['traf_version']) KEY_FILE = '/tmp/id_rsa' AUTH_KEY_FILE = '%s/.ssh/authorized_keys' % TRAF_USER_DIR SSH_CFG_FILE = '%s/.ssh/config' % TRAF_USER_DIR BASHRC_TEMPLATE = '%s/bashrc.template' % TMP_DIR BASHRC_FILE = '%s/.bashrc' % TRAF_USER_DIR ULIMITS_FILE = '/etc/security/limits.d/%s.conf' % TRAF_USER HSPERFDATA_FILE = '/tmp/hsperfdata_trafodion' # create trafodion user and group if not cmd_output('getent group %s' % TRAF_GROUP): run_cmd('groupadd %s > /dev/null 2>&1' % TRAF_GROUP) if not cmd_output('getent passwd %s' % TRAF_USER): run_cmd( 'useradd --shell /bin/bash -m %s -g %s --password "$(openssl passwd %s)"' % (TRAF_USER, TRAF_GROUP, TRAF_PWD)) elif not os.path.exists(TRAF_USER_DIR): run_cmd('mkdir -p %s' % TRAF_USER_DIR) run_cmd('chmod 700 %s' % TRAF_USER_DIR) # set ssh key run_cmd_as_user(TRAF_USER, 'echo -e "y" | ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa') # the key is generated in copy_file script running on the installer node run_cmd('cp %s{,.pub} %s/.ssh/' % (KEY_FILE, TRAF_USER_DIR)) run_cmd_as_user(TRAF_USER, 'cat ~/.ssh/id_rsa.pub > %s' % AUTH_KEY_FILE) run_cmd('chmod 644 %s' % AUTH_KEY_FILE) ssh_cfg = 'StrictHostKeyChecking=no\nNoHostAuthenticationForLocalhost=yes\n' with open(SSH_CFG_FILE, 'w') as f: f.write(ssh_cfg) run_cmd('chmod 600 %s' % SSH_CFG_FILE) run_cmd('chown -R %s:%s %s/.ssh/' % (TRAF_USER, TRAF_GROUP, TRAF_USER_DIR)) # set bashrc nodes = dbcfgs['node_list'].split(',') change_items = { '{{ java_home }}': dbcfgs['java_home'], '{{ sq_home }}': SQ_ROOT, '{{ hadoop_type }}': hadoop_type, '{{ node_list }}': ' '.join(nodes), '{{ node_count }}': str(len(nodes)), '{{ enable_ha }}': dbcfgs['enable_ha'], '{{ my_nodes }}': ' -w ' + ' -w '.join(nodes) } mod_file(BASHRC_TEMPLATE, change_items) if 'APACHE' in DISTRO: bashrc_content = """ export HADOOP_PREFIX=%s export HBASE_HOME=%s export PATH=$PATH:$HADOOP_PREFIX/bin:$HADOOP_PREFIX/sbin:$HBASE_HOME/bin """ % (dbcfgs['hadoop_home'], dbcfgs['hbase_home']) append_file(BASHRC_TEMPLATE, bashrc_content, position='HADOOP_TYPE') # backup bashrc if exsits if os.path.exists(BASHRC_FILE): run_cmd('cp %s %s.bak' % ((BASHRC_FILE, ) * 2)) # copy bashrc to trafodion's home run_cmd('cp %s %s' % (BASHRC_TEMPLATE, BASHRC_FILE)) run_cmd('chown -R %s:%s %s*' % (TRAF_USER, TRAF_GROUP, BASHRC_FILE)) # set ulimits for trafodion user ulimits_config = ''' # Trafodion settings %s soft core unlimited %s hard core unlimited %s soft memlock unlimited %s hard memlock unlimited %s soft nofile 32768 %s hard nofile 65536 %s soft nproc 100000 %s hard nproc 100000 %s soft nofile 8192 %s hard nofile 65535 hbase soft nofile 8192 ''' % ((TRAF_USER, ) * 10) with open(ULIMITS_FILE, 'w') as f: f.write(ulimits_config) # change permission for hsperfdata if os.path.exists(HSPERFDATA_FILE): run_cmd('chown -R %s:%s %s' % (TRAF_USER, TRAF_GROUP, HSPERFDATA_FILE)) # clean up unused key file at the last step run_cmd('rm -rf %s{,.pub}' % KEY_FILE) print 'Setup trafodion user successfully!'
def run(): """ create trafodion user, bashrc, setup passwordless SSH """ dbcfgs = json.loads(dbcfgs_json) distro = dbcfgs['distro'] if 'CDH' in distro: hadoop_type = 'cloudera' elif 'HDP' in distro: hadoop_type = 'hortonworks' elif 'APACHE' in distro: hadoop_type = 'apache' home_dir = get_default_home() # customize trafodion home dir if dbcfgs.has_key('home_dir') and dbcfgs['home_dir']: home_dir = dbcfgs['home_dir'] traf_user = dbcfgs['traf_user'] traf_user_dir = '%s/%s' % (home_dir, traf_user) traf_dirname = dbcfgs['traf_dirname'] traf_home = '%s/%s' % (traf_user_dir, traf_dirname) traf_log = dbcfgs['traf_log'] traf_var = dbcfgs['traf_var'] hbase_xml_file = dbcfgs['hbase_xml_file'] auth_key_file = '%s/.ssh/authorized_keys' % traf_user_dir ssh_cfg_file = '%s/.ssh/config' % traf_user_dir ulimits_file = '/etc/security/limits.d/%s.conf' % traf_user # create trafodion user and group if cmd_output('getent passwd %s' % traf_user): # trafodion user exists, set actual trafodion group traf_group = cmd_output('id -ng %s' % traf_user) else: # default trafodion group traf_group = traf_user if not cmd_output('getent group %s' % traf_group): run_cmd('groupadd %s > /dev/null 2>&1' % traf_group) traf_pwd = dbcfgs['traf_pwd'] run_cmd('useradd --shell /bin/bash -m %s -g %s --home %s --password "$(openssl passwd %s)"' % (traf_user, traf_group, traf_user_dir, traf_pwd)) # hbase group is generally either hbase or hadoop, depending on distro if cmd_output('getent group hbase'): cmd_output('/usr/sbin/usermod -a -G hbase %s' % traf_user) if cmd_output('getent group hadoop'): cmd_output('/usr/sbin/usermod -a -G hadoop %s' % traf_user) if cmd_output('getent group hive'): cmd_output('/usr/sbin/usermod -a -G hive %s' % traf_user) if not os.path.exists(traf_user_dir): run_cmd('mkdir -p %s' % traf_user_dir) run_cmd('chmod 700 %s' % traf_user_dir) # set ssh key run_cmd_as_user(traf_user, 'echo -e "y" | ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa') # the key is generated in copy_file script running on the installer node run_cmd('cp %s{,.pub} %s/.ssh/' % (SSHKEY_FILE, traf_user_dir)) run_cmd_as_user(traf_user, 'cat ~/.ssh/id_rsa.pub > %s' % auth_key_file) run_cmd('chmod 644 %s' % auth_key_file) ssh_cfg = 'StrictHostKeyChecking=no\nNoHostAuthenticationForLocalhost=yes\n' with open(ssh_cfg_file, 'w') as f: f.write(ssh_cfg) run_cmd('chmod 600 %s' % ssh_cfg_file) run_cmd('chown -R %s:%s %s/.ssh/' % (traf_user, traf_group, traf_user_dir)) hb = ParseXML(hbase_xml_file) zk_nodes = hb.get_property('hbase.zookeeper.quorum') zk_port = hb.get_property('hbase.zookeeper.property.clientPort') # set trafodion_config nodes = dbcfgs['node_list'].split(',') trafodion_config = """ export TRAF_HOME="%s" export TRAF_VAR="%s" export TRAF_CONF="%s" export TRAF_LOG="%s" export JAVA_HOME="%s" export node_count="%s" export HADOOP_TYPE="%s" export ENABLE_HA="%s" export ZOOKEEPER_NODES="%s" export ZOOKEEPER_PORT="%s" export SECURE_HADOOP="%s" export CLUSTERNAME="%s" """ % (traf_home, traf_var, TRAF_CFG_DIR, traf_log, dbcfgs['java_home'], str(len(nodes)), hadoop_type, dbcfgs['enable_ha'], zk_nodes, zk_port, dbcfgs['secure_hadoop'], socket.gethostname()) # save additonal configs for elastic trafodion_config += """ export hbase_xml_file="%s" export hbase_lib_path="%s" export traf_user="******" export traf_version="%s" export dcs_cnt_per_node="%s" """ % (dbcfgs['hbase_xml_file'], dbcfgs['hbase_lib_path'], dbcfgs['traf_user'], dbcfgs['traf_version'], dbcfgs['dcs_cnt_per_node']) # save additonal configs for multi instance support trafodion_config += """ export TRAF_CLUSTER_NAME="%s" export TRAF_INSTANCE_NAME="%s" export TRAF_CLUSTER_ID="%s" export TRAF_INSTANCE_ID="%s" export TRAF_ROOT_ZNODE="/%s" """ % (dbcfgs['cluster_name'], dbcfgs['traf_instance_name'], dbcfgs['traf_cluster_id'], dbcfgs['traf_instance_id'], dbcfgs['traf_user']) run_cmd('mkdir -p %s' % TRAF_CFG_DIR) write_file(TRAF_CFG_FILE, trafodion_config) if 'APACHE' in distro: extra_config = """ export HADOOP_PREFIX=%s export HBASE_HOME=%s export HIVE_HOME=%s export PATH=$PATH:$HADOOP_PREFIX/bin:$HADOOP_PREFIX/sbin:$HBASE_HOME/bin """ % (dbcfgs['hadoop_home'], dbcfgs['hbase_home'], dbcfgs['hive_home']) append_file(TRAFODION_CFG_FILE, extra_config) # set permission run_cmd('chown -R %s:%s %s*' % (traf_user, traf_group, TRAF_CFG_DIR)) # set ulimits for trafodion user ulimits_config = ''' # Trafodion settings %s soft core unlimited %s hard core unlimited %s soft memlock unlimited %s hard memlock unlimited %s soft nofile 32768 %s hard nofile 65536 %s soft nproc 100000 %s hard nproc 100000 %s soft nofile 8192 %s hard nofile 65535 ''' % ((traf_user,) * 10) write_file(ulimits_file, ulimits_config) # change permission for hsperfdata if os.path.exists(TRAF_HSPERFDATA_FILE): run_cmd('chown -R %s:%s %s' % (traf_user, traf_group, TRAF_HSPERFDATA_FILE)) # clean up unused key file at the last step run_cmd('rm -rf %s{,.pub}' % SSHKEY_FILE) print 'Setup trafodion user successfully!'
def run(): """ create trafodion user, bashrc, setup passwordless SSH """ dbcfgs = json.loads(dbcfgs_json) DISTRO = dbcfgs['distro'] if 'CDH' in DISTRO: hadoop_type = 'cloudera' elif 'HDP' in DISTRO: hadoop_type = 'hortonworks' elif 'APACHE' in DISTRO: hadoop_type = 'apache' TRAF_USER = dbcfgs['traf_user'] TRAF_GROUP = TRAF_USER HOME_DIR = cmd_output('cat /etc/default/useradd |grep HOME |cut -d "=" -f 2').strip() # customize trafodion home dir if dbcfgs.has_key('home_dir') and dbcfgs['home_dir']: HOME_DIR = dbcfgs['home_dir'] TRAF_USER_DIR = '%s/%s' % (HOME_DIR, TRAF_USER) TRAF_DIRNAME = dbcfgs['traf_dirname'] TRAF_HOME = '%s/%s' % (TRAF_USER_DIR, TRAF_DIRNAME) TRAFODION_CFG_DIR = '/etc/trafodion/' TRAFODION_CFG_FILE = '/etc/trafodion/trafodion_config' HBASE_XML_FILE = dbcfgs['hbase_xml_file'] KEY_FILE = '/tmp/id_rsa' AUTH_KEY_FILE = '%s/.ssh/authorized_keys' % TRAF_USER_DIR SSH_CFG_FILE = '%s/.ssh/config' % TRAF_USER_DIR ULIMITS_FILE = '/etc/security/limits.d/%s.conf' % TRAF_USER HSPERFDATA_FILE = '/tmp/hsperfdata_trafodion' # create trafodion user and group if not cmd_output('getent group %s' % TRAF_GROUP): run_cmd('groupadd %s > /dev/null 2>&1' % TRAF_GROUP) if not cmd_output('getent passwd %s' % TRAF_USER): TRAF_PWD = dbcfgs['traf_pwd'] run_cmd('useradd --shell /bin/bash -m %s -g %s --home %s --password "$(openssl passwd %s)"' % (TRAF_USER, TRAF_GROUP, TRAF_USER_DIR, TRAF_PWD)) elif not os.path.exists(TRAF_USER_DIR): run_cmd('mkdir -p %s' % TRAF_USER_DIR) run_cmd('chmod 700 %s' % TRAF_USER_DIR) # set ssh key run_cmd_as_user(TRAF_USER, 'echo -e "y" | ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa') # the key is generated in copy_file script running on the installer node run_cmd('cp %s{,.pub} %s/.ssh/' % (KEY_FILE, TRAF_USER_DIR)) run_cmd_as_user(TRAF_USER, 'cat ~/.ssh/id_rsa.pub > %s' % AUTH_KEY_FILE) run_cmd('chmod 644 %s' % AUTH_KEY_FILE) ssh_cfg = 'StrictHostKeyChecking=no\nNoHostAuthenticationForLocalhost=yes\n' with open(SSH_CFG_FILE, 'w') as f: f.write(ssh_cfg) run_cmd('chmod 600 %s' % SSH_CFG_FILE) run_cmd('chown -R %s:%s %s/.ssh/' % (TRAF_USER, TRAF_GROUP, TRAF_USER_DIR)) hb = ParseXML(HBASE_XML_FILE) zk_nodes = hb.get_property('hbase.zookeeper.quorum') zk_port = hb.get_property('hbase.zookeeper.property.clientPort') # set trafodion_config nodes = dbcfgs['node_list'].split(',') trafodion_config = """ export TRAF_HOME="%s" export MY_SQROOT=$TRAF_HOME # for compatibility export JAVA_HOME="%s" export NODE_LIST="%s" export MY_NODES="%s" export node_count="%s" export HADOOP_TYPE="%s" export ENABLE_HA="%s" export ZOOKEEPER_NODES="%s" export ZOOKEEPER_PORT="%s" export SECURE_HADOOP="%s" """ % (TRAF_HOME, dbcfgs['java_home'], ' '.join(nodes), ' -w ' + ' -w '.join(nodes), str(len(nodes)), hadoop_type, dbcfgs['enable_ha'], zk_nodes, zk_port, dbcfgs['secure_hadoop']) run_cmd('mkdir -p %s' % TRAFODION_CFG_DIR) write_file(TRAFODION_CFG_FILE, trafodion_config) if 'APACHE' in DISTRO: extra_config = """ export HADOOP_PREFIX=%s export HBASE_HOME=%s export PATH=$PATH:$HADOOP_PREFIX/bin:$HADOOP_PREFIX/sbin:$HBASE_HOME/bin """ % (dbcfgs['hadoop_home'], dbcfgs['hbase_home']) append_file(TRAFODION_CFG_FILE, extra_config) # set permission run_cmd('chown -R %s:%s %s*' % (TRAF_USER, TRAF_GROUP, TRAFODION_CFG_DIR)) # set ulimits for trafodion user ulimits_config = ''' # Trafodion settings %s soft core unlimited %s hard core unlimited %s soft memlock unlimited %s hard memlock unlimited %s soft nofile 32768 %s hard nofile 65536 %s soft nproc 100000 %s hard nproc 100000 %s soft nofile 8192 %s hard nofile 65535 ''' % ((TRAF_USER,) * 10) write_file(ULIMITS_FILE, ulimits_config) # change permission for hsperfdata if os.path.exists(HSPERFDATA_FILE): run_cmd('chown -R %s:%s %s' % (TRAF_USER, TRAF_GROUP, HSPERFDATA_FILE)) # clean up unused key file at the last step run_cmd('rm -rf %s{,.pub}' % KEY_FILE) print 'Setup trafodion user successfully!'
def run(): """ create trafodion user, bashrc, setup passwordless SSH """ dbcfgs = json.loads(dbcfgs_json) DISTRO = dbcfgs['distro'] if 'CDH' in DISTRO: hadoop_type = 'cloudera' elif 'HDP' in DISTRO: hadoop_type = 'hortonworks' elif 'APACHE' in DISTRO: hadoop_type = 'apache' TRAF_USER = dbcfgs['traf_user'] TRAF_PWD = dbcfgs['traf_pwd'] TRAF_GROUP = TRAF_USER TRAF_HOME = cmd_output('cat /etc/default/useradd |grep HOME |cut -d "=" -f 2').strip() TRAF_USER_DIR = '%s/%s' % (TRAF_HOME, TRAF_USER) SQ_ROOT = '%s/%s-%s' % (TRAF_USER_DIR, dbcfgs['traf_basename'], dbcfgs['traf_version']) KEY_FILE = '/tmp/id_rsa' AUTH_KEY_FILE = '%s/.ssh/authorized_keys' % TRAF_USER_DIR SSH_CFG_FILE = '%s/.ssh/config' % TRAF_USER_DIR BASHRC_TEMPLATE = '%s/bashrc.template' % TMP_DIR BASHRC_FILE = '%s/.bashrc' % TRAF_USER_DIR ULIMITS_FILE = '/etc/security/limits.d/%s.conf' % TRAF_USER HSPERFDATA_FILE = '/tmp/hsperfdata_trafodion' # create trafodion user and group if not cmd_output('getent group %s' % TRAF_GROUP): run_cmd('groupadd %s > /dev/null 2>&1' % TRAF_GROUP) if not cmd_output('getent passwd %s' % TRAF_USER): run_cmd('useradd --shell /bin/bash -m %s -g %s --password "$(openssl passwd %s)"' % (TRAF_USER, TRAF_GROUP, TRAF_PWD)) elif not os.path.exists(TRAF_USER_DIR): run_cmd('mkdir -p %s' % TRAF_USER_DIR) run_cmd('chmod 700 %s' % TRAF_USER_DIR) # set ssh key run_cmd_as_user(TRAF_USER, 'echo -e "y" | ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa') # the key is generated in copy_file script running on the installer node run_cmd('cp %s{,.pub} %s/.ssh/' % (KEY_FILE, TRAF_USER_DIR)) run_cmd_as_user(TRAF_USER, 'cat ~/.ssh/id_rsa.pub > %s' % AUTH_KEY_FILE) run_cmd('chmod 644 %s' % AUTH_KEY_FILE) ssh_cfg = 'StrictHostKeyChecking=no\nNoHostAuthenticationForLocalhost=yes\n' with open(SSH_CFG_FILE, 'w') as f: f.write(ssh_cfg) run_cmd('chmod 600 %s' % SSH_CFG_FILE) run_cmd('chown -R %s:%s %s/.ssh/' % (TRAF_USER, TRAF_GROUP, TRAF_USER_DIR)) # set bashrc nodes = dbcfgs['node_list'].split(',') change_items = { '{{ java_home }}': dbcfgs['java_home'], '{{ sq_home }}': SQ_ROOT, '{{ hadoop_type }}': hadoop_type, '{{ node_list }}': ' '.join(nodes), '{{ node_count }}': str(len(nodes)), '{{ enable_ha }}': dbcfgs['enable_ha'], '{{ my_nodes }}': ' -w ' + ' -w '.join(nodes) } mod_file(BASHRC_TEMPLATE, change_items) if 'APACHE' in DISTRO: bashrc_content = """ export HADOOP_PREFIX=%s export HBASE_HOME=%s export PATH=$PATH:$HADOOP_PREFIX/bin:$HADOOP_PREFIX/sbin:$HBASE_HOME/bin """ % (dbcfgs['hadoop_home'], dbcfgs['hbase_home']) append_file(BASHRC_TEMPLATE, bashrc_content, position='HADOOP_TYPE') # backup bashrc if exsits if os.path.exists(BASHRC_FILE): run_cmd('cp %s %s.bak' % ((BASHRC_FILE,) *2)) # copy bashrc to trafodion's home run_cmd('cp %s %s' % (BASHRC_TEMPLATE, BASHRC_FILE)) run_cmd('chown -R %s:%s %s*' % (TRAF_USER, TRAF_GROUP, BASHRC_FILE)) # set ulimits for trafodion user ulimits_config = ''' # Trafodion settings %s soft core unlimited %s hard core unlimited %s soft memlock unlimited %s hard memlock unlimited %s soft nofile 32768 %s hard nofile 65536 %s soft nproc 100000 %s hard nproc 100000 %s soft nofile 8192 %s hard nofile 65535 hbase soft nofile 8192 ''' % ((TRAF_USER,) * 10) with open(ULIMITS_FILE, 'w') as f: f.write(ulimits_config) # change permission for hsperfdata if os.path.exists(HSPERFDATA_FILE): run_cmd('chown -R %s:%s %s' % (TRAF_USER, TRAF_GROUP, HSPERFDATA_FILE)) # clean up unused key file at the last step run_cmd('rm -rf %s{,.pub}' % KEY_FILE) print 'Setup trafodion user successfully!'
def run(): hdfs_bin = DEF_HDFS_BIN dbcfgs = json.loads(dbcfgs_json) distro = dbcfgs['distro'] if 'CDH' in distro: parcel_lib = PARCEL_HBASE_LIB if os.path.exists(parcel_lib): hdfs_bin = PARCEL_HDFS_BIN elif 'APACHE' in distro: hdfs_bin = dbcfgs['hadoop_home'] + '/bin/hdfs' traf_loc = '/user/trafodion' traf_user = dbcfgs['traf_user'] hdfs_user = dbcfgs['hdfs_user'] hbase_user = dbcfgs['hbase_user'] hbase_group = cmd_output('%s groups %s | cut -d" " -f3' % (hdfs_bin, hbase_user)) run_cmd_as_user(hdfs_user, '%s dfsadmin -safemode wait' % hdfs_bin) run_cmd_as_user(hdfs_user, '%s dfs -mkdir -p %s/{trafodion_backups,bulkload,lobs} /hbase/archive' % (hdfs_bin, traf_loc)) run_cmd_as_user(hdfs_user, '%s dfs -chown -R %s:%s /hbase/archive' % (hdfs_bin, hbase_user, hbase_user)) run_cmd_as_user(hdfs_user, '%s dfs -chown -R %s:%s %s %s/{trafodion_backups,bulkload,lobs}' % (hdfs_bin, traf_user, traf_user, traf_loc, traf_loc)) run_cmd_as_user(hdfs_user, '%s dfs -chmod 0755 %s' % (hdfs_bin, traf_loc)) run_cmd_as_user(hdfs_user, '%s dfs -chmod 0750 %s/{trafodion_backups,bulkload,lobs}' % (hdfs_bin, traf_loc)) run_cmd_as_user(hdfs_user, '%s dfs -chgrp %s %s/bulkload' % (hdfs_bin, hbase_group, traf_loc)) run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m user:%s:rwx /hbase/archive' % (hdfs_bin, traf_user)) run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m default:user:%s:rwx /hbase/archive' % (hdfs_bin, traf_user)) run_cmd_as_user(hdfs_user, '%s dfs -setfacl -R -m mask::rwx /hbase/archive' % hdfs_bin) # Grant all privileges to the Trafodion principal in HBase if dbcfgs['secure_hadoop'] == 'Y': run_cmd('echo "grant \'%s\', \'RWXC\'" | %s su - %s -s /bin/bash -c "hbase shell" > /tmp/hbase_shell.out' % (traf_user, get_sudo_prefix(), hbase_user)) has_err = cmd_output('grep -c ERROR /tmp/hbase_shell.out') if int(has_err): err('Failed to grant HBase privileges to %s' % traf_user) run_cmd('rm /tmp/hbase_shell.out')
def run(): """ create trafodion user, bashrc, setup passwordless SSH """ dbcfgs = json.loads(dbcfgs_json) distro = dbcfgs['distro'] if 'CDH' in distro: hadoop_type = 'cloudera' elif 'HDP' in distro: hadoop_type = 'hortonworks' elif 'APACHE' in distro: hadoop_type = 'apache' home_dir = get_default_home() # customize trafodion home dir if dbcfgs.has_key('home_dir') and dbcfgs['home_dir']: home_dir = dbcfgs['home_dir'] traf_user = dbcfgs['traf_user'] traf_user_dir = '%s/%s' % (home_dir, traf_user) traf_dirname = dbcfgs['traf_dirname'] traf_home = '%s/%s' % (traf_user_dir, traf_dirname) hbase_xml_file = dbcfgs['hbase_xml_file'] auth_key_file = '%s/.ssh/authorized_keys' % traf_user_dir ssh_cfg_file = '%s/.ssh/config' % traf_user_dir ulimits_file = '/etc/security/limits.d/%s.conf' % traf_user # create trafodion user and group if cmd_output('getent passwd %s' % traf_user): # trafodion user exists, set actual trafodion group traf_group = cmd_output('id -ng %s' % traf_user) else: # default trafodion group traf_group = traf_user if not cmd_output('getent group %s' % traf_group): run_cmd('groupadd %s > /dev/null 2>&1' % traf_group) traf_pwd = dbcfgs['traf_pwd'] run_cmd('useradd --shell /bin/bash -m %s -g %s --home %s --password "$(openssl passwd %s)"' % (traf_user, traf_group, traf_user_dir, traf_pwd)) # hbase group is generally either hbase or hadoop, depending on distro if cmd_output('getent group hbase'): cmd_output('/usr/sbin/usermod -a -G hbase %s' % traf_user) if cmd_output('getent group hadoop'): cmd_output('/usr/sbin/usermod -a -G hadoop %s' % traf_user) if cmd_output('getent group hive'): cmd_output('/usr/sbin/usermod -a -G hive %s' % traf_user) if not os.path.exists(traf_user_dir): run_cmd('mkdir -p %s' % traf_user_dir) run_cmd('chmod 700 %s' % traf_user_dir) # set ssh key run_cmd_as_user(traf_user, 'echo -e "y" | ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa') # the key is generated in copy_file script running on the installer node run_cmd('cp %s{,.pub} %s/.ssh/' % (SSHKEY_FILE, traf_user_dir)) run_cmd_as_user(traf_user, 'cat ~/.ssh/id_rsa.pub > %s' % auth_key_file) run_cmd('chmod 644 %s' % auth_key_file) ssh_cfg = 'StrictHostKeyChecking=no\nNoHostAuthenticationForLocalhost=yes\n' with open(ssh_cfg_file, 'w') as f: f.write(ssh_cfg) run_cmd('chmod 600 %s' % ssh_cfg_file) run_cmd('chown -R %s:%s %s/.ssh/' % (traf_user, traf_group, traf_user_dir)) hb = ParseXML(hbase_xml_file) zk_nodes = hb.get_property('hbase.zookeeper.quorum') zk_port = hb.get_property('hbase.zookeeper.property.clientPort') # set trafodion_config nodes = dbcfgs['node_list'].split(',') trafodion_config = """ export TRAF_HOME="%s" export TRAF_VAR=$TRAF_HOME/tmp export MY_SQROOT=$TRAF_HOME # for compatibility export JAVA_HOME="%s" export node_count="%s" export HADOOP_TYPE="%s" export ENABLE_HA="%s" export ZOOKEEPER_NODES="%s" export ZOOKEEPER_PORT="%s" export SECURE_HADOOP="%s" export CLUSTERNAME="%s" """ % (traf_home, dbcfgs['java_home'], str(len(nodes)), hadoop_type, dbcfgs['enable_ha'], zk_nodes, zk_port, dbcfgs['secure_hadoop'], socket.gethostname()) # save additonal configs for elastic trafodion_config += """ export hbase_xml_file="%s" export hbase_lib_path="%s" export traf_user="******" export traf_version="%s" export dcs_cnt_per_node="%s" """ % (dbcfgs['hbase_xml_file'], dbcfgs['hbase_lib_path'], dbcfgs['traf_user'], dbcfgs['traf_version'], dbcfgs['dcs_cnt_per_node']) run_cmd('mkdir -p %s' % TRAF_CFG_DIR) write_file(TRAF_CFG_FILE, trafodion_config) if 'APACHE' in distro: extra_config = """ export HADOOP_PREFIX=%s export HBASE_HOME=%s export HIVE_HOME=%s export PATH=$PATH:$HADOOP_PREFIX/bin:$HADOOP_PREFIX/sbin:$HBASE_HOME/bin """ % (dbcfgs['hadoop_home'], dbcfgs['hbase_home'], dbcfgs['hive_home']) append_file(TRAFODION_CFG_FILE, extra_config) # set permission run_cmd('chown -R %s:%s %s*' % (traf_user, traf_group, TRAF_CFG_DIR)) # set ulimits for trafodion user ulimits_config = ''' # Trafodion settings %s soft core unlimited %s hard core unlimited %s soft memlock unlimited %s hard memlock unlimited %s soft nofile 32768 %s hard nofile 65536 %s soft nproc 100000 %s hard nproc 100000 %s soft nofile 8192 %s hard nofile 65535 ''' % ((traf_user,) * 10) write_file(ulimits_file, ulimits_config) # change permission for hsperfdata if os.path.exists(TRAF_HSPERFDATA_FILE): run_cmd('chown -R %s:%s %s' % (traf_user, traf_group, TRAF_HSPERFDATA_FILE)) # clean up unused key file at the last step run_cmd('rm -rf %s{,.pub}' % SSHKEY_FILE) print 'Setup trafodion user successfully!'