def getConfigValue(cls, propertyValue, defaultValue=None): _localHiveConf = os.path.join(Config.getEnv('ARTIFACTS_DIR'), cls._localConfDir) if not os.path.exists(_localHiveConf): admin_user = Machine.getAdminUser() Machine.copyToLocal(admin_user, cls.getHiveHost(), cls._configDir, _localHiveConf) Machine.chmod('777', _localHiveConf, recursive=True, user=admin_user) if cls._modifiedConfigs.has_key(propertyValue): return cls._modifiedConfigs[propertyValue] value = util.getPropertyValueFromConfigXMLFile( os.path.join(_localHiveConf, 'hiveserver2-site.xml'), propertyValue ) if value is None or value == '': value = util.getPropertyValueFromConfigXMLFile( os.path.join(_localHiveConf, 'hive-site.xml'), propertyValue, defaultValue=defaultValue ) return value
def HDFS_getConfigValue(cls, propertyValue, defaultValue=None): try: hdfs_conf_path = os.path.join(Config.get('hadoop', 'HADOOP_CONF'), "hdfs-site.xml") logger.info("hdfs_conf_path is %s", hdfs_conf_path) return util.getPropertyValueFromConfigXMLFile( hdfs_conf_path, propertyValue, defaultValue=defaultValue) except Exception: logger.error("Exception occured during HDFS_getConfigValue() call") logger.error(traceback.format_exc()) return None
def insertCSVDataViaPSQL(cls, csvFile, tableName, env=None): """ If tableName==None, the table will be created with the name of the file. """ global ZK_ZPARENT if Slider.isSlider(): ZK_ZPARENT = util.getPropertyValueFromConfigXMLFile( os.path.join(Config.get('hbase', 'HBASE_CONF_DIR'), 'hbase-site.xml'), "zookeeper.znode.parent") if not env: env = {} env['HBASE_CONF_PATH'] = Config.get('hbase', 'HBASE_CONF_DIR') # We save the current directory currentDir = os.getcwd() # We go to phoenix / bin folder os.chdir(os.path.join(PHOENIX_HOME, 'bin')) # We run the command if ZK_ZPARENT is not None and ZK_ZPARENT != '': command = "%s -t %s %s:%s:%s %s" % (PSQL_SCRIPT, tableName.upper(), ZK_HOST, ZK_PORT, ZK_ZPARENT, csvFile) else: command = "%s -t %s %s:%s %s" % (PSQL_SCRIPT, tableName.upper(), ZK_HOST, ZK_PORT, csvFile) result = Machine.run( command, env=dict(env.items() + ENVIRONMENT.items() if env is not None else ENVIRONMENT)) # We restore the execution directory os.chdir(currentDir) # we return the results return result
def getConfigValue(cls, propertyValue, defaultValue=None): return util.getPropertyValueFromConfigXMLFile(os.path.join(Config.get('hadoop', 'HADOOP_CONF'), "core-site.xml"), propertyValue, defaultValue=defaultValue)
def getConfigValue(cls, propertyValue, defaultValue=None): return util.getPropertyValueFromConfigXMLFile( os.path.join(OOZIE_CONF_DIR, "oozie-site.xml"), propertyValue, defaultValue=defaultValue)
def getLocalDirInfo(host): return util.getPropertyValueFromConfigXMLFile(os.path.join(Config.get('hadoop', 'HADOOP_CONF'), "mapred-site.xml"), "mapred.local.dir")
def insertCSVDataViaCSVBuildLoad(cls, csvFile, tableName, putIntoHDFS=True, deleteAfterExec=True, runInBackground=False, user=None, config=None, optionAndParameter="", env=None, delimiter=",", arrayDelimiter=None, schema=None): """ By default, the files will be allocated under /tmp/ folder in HDFS. """ global ZK_ZPARENT if Slider.isSlider(): ZK_ZPARENT = util.getPropertyValueFromConfigXMLFile( os.path.join(Config.get('hbase', 'HBASE_CONF_DIR'), 'hbase-site.xml'), "zookeeper.znode.parent") if Machine.isLinux(): clientjar = Machine.find(user=Machine.getAdminUser(), host="localhost", filepath=PHOENIX_HOME, searchstr="phoenix-*[0-9]-client.jar", passwd=Machine.getAdminPasswd()) else: clientjar = Machine.find(user=Machine.getAdminUser(), host="localhost", filepath=PHOENIX_HOME, searchstr="phoenix-*-client.jar", passwd=Machine.getAdminPasswd()) if Machine.isWindows(): clientjar = (clientjar[0].strip("\\localhost")).replace("$", ":") fileName = csvFile.split('\\')[-1] else: clientjar = clientjar[0] fileName = csvFile.split('/')[-1] # If we need to, we insert it into HDFS, since the library will take it from there. executingUser = (HADOOPQA_USER) if user is None else user if putIntoHDFS: if not HDFS.fileExists('/tmp/'): HDFS.mkdir('/tmp/') HDFS.copyFromLocal(csvFile, '/tmp/', executingUser, config, optionAndParameter) hbaseConfDir = HBASE_CONF_DIR if Slider.isSlider(): hbaseConfDir = Config.get('hbase', 'HBASE_CONF_DIR') classpath = hbaseConfDir finalCommand = "%s jar %s org.apache.phoenix.mapreduce.CsvBulkLoadTool --table %s --input %s" \ % (HADOOP_CMD, clientjar, tableName, '/tmp/%s' % fileName) if schema is not None: finalCommand = '%s -schema %s' % (finalCommand, schema) if Machine.isWindows(): os.environ['HADOOP_USER_CLASSPATH_FIRST'] = 'true' os.environ['HADOOP_CLASSPATH'] = classpath if delimiter != "," or arrayDelimiter != None: finalCommand = "%s -d `\\\"`%s`\\\" -a `\\\"`%s`\\\"" \ % (finalCommand, delimiter, arrayDelimiter.strip("'")) finalCommand = "%s --zookeeper %s" % (finalCommand, ZK_HOST) if runInBackground: exit_code = 0 stdout = '' Machine.runinbackground( finalCommand, env=dict(env.items() + ENVIRONMENT.items() if env is not None else ENVIRONMENT)) else: exit_code, stdout = Machine.run( finalCommand, env=dict(env.items() + ENVIRONMENT.items() if env is not None else ENVIRONMENT)) else: # delimiter options if delimiter != "," or arrayDelimiter != None: finalCommand = "%s --delimiter %s --array-delimiter %s" % ( finalCommand, delimiter, arrayDelimiter) # ZKHosts options finalCommand = "%s --zookeeper %s" % (finalCommand, cls.getZKConnectString()) if runInBackground: exit_code = 0 stdout = '' Machine.runinbackground( "HADOOP_CLASSPATH=%s %s" % (classpath, finalCommand), env=dict(env.items() + ENVIRONMENT.items() if env is not None else ENVIRONMENT)) else: exit_code, stdout = Machine.run( "HADOOP_CLASSPATH=%s %s" % (classpath, finalCommand), env=dict(env.items() + ENVIRONMENT.items() if env is not None else ENVIRONMENT)) # If selected, after insertion into HBase we will delete the csvFile from HDFS if deleteAfterExec and not runInBackground: # Does not work for "run in background" option HDFS.deleteFile('/tmp/%s' % fileName, executingUser) # return 0,"" return exit_code, stdout
def runSQLLineCmdsWithoutEnvVars(cls, cmd, env=None, logoutput=True, outputFormat=None, threadNumber=None, user=None): global ZK_ZPARENT if Slider.isSlider(): ZK_ZPARENT = util.getPropertyValueFromConfigXMLFile( os.path.join(Config.get('hbase', 'HBASE_CONF_DIR'), 'hbase-site.xml'), "zookeeper.znode.parent") if not env: env = {} if Hadoop.isSecure(): if user is None: user = Config.getEnv('USER') kerbTicket = Machine.getKerberosTicket(user) env['KRB5CCNAME'] = kerbTicket user = None folder = os.path.join(ARTIFACTS_DIR, 'phoenixSQLQuery') if threadNumber is None: sqlConsultFile = os.path.join(folder, INPUT_FILE + '.txt') else: sqlConsultFile = os.path.join( folder, INPUT_FILE + "_thread_" + str(threadNumber) + ".txt") # We remove the file if exists Machine.rm(user, HOST, sqlConsultFile) # We create aux file Machine.makedirs(user, HOST, folder) # We write the sql command to the file. If we are in ASV, we get the output in csv format if outputFormat is not None: cmd = '!outputformat %s\n%s' % (outputFormat, cmd) if HDFS.isASV() and Machine.isWindows(): cmd = '!outputformat csv\n%s' % cmd # We write the sql command to the file fh = open(sqlConsultFile, 'w') fh.write(cmd) fh.close() logger.info("RUNNING Command in input file: %s", cmd) # We save the current directory currentDirectory = os.getcwd() # We change the path to execute sqlline.py from /bin folder os.chdir(os.path.join(PHOENIX_HOME, 'bin')) exportCommand = "" # We run the command to execute SQL instruction if RUN_WITH_SQLLINE_THIN: queryServer = None if QUERYSERVER_URL is None: # Since phoenix-QueryServer will be running on all regionservers, we re-direct the query # to any currently active regionserver phoenixqueryservers = cls.getPhoenixQueryServers() assert phoenixqueryservers, "Query failed. No active RS found with cls.getPhoenixQueryServers()" queryServer = phoenixqueryservers[0] else: queryServer = QUERYSERVER_URL assert queryServer is not None, "Should have a URL to a PQS instance" completeCommand = '%s %s %s %s' % (exportCommand, SQLLINE_THIN_SCRIPT, queryServer, sqlConsultFile) else: if ZK_ZPARENT is not None and ZK_ZPARENT != '': completeCommand = '%s %s %s:%s:%s %s' % ( exportCommand, SQLLINE_SCRIPT, ZK_HOST, ZK_PORT, ZK_ZPARENT, sqlConsultFile) else: completeCommand = '%s %s %s:%s %s' % (exportCommand, SQLLINE_SCRIPT, ZK_HOST, ZK_PORT, sqlConsultFile) result = Machine.runas( user, completeCommand, env=dict(env.items() + ENVIRONMENT.items() if env is not None else ENVIRONMENT), logoutput=logoutput) # we restore the previous directory os.chdir(currentDirectory) # We remove the file Machine.rm(user, HOST, sqlConsultFile) return result
HBASE_LIB_DIR = os.path.join(Config.get('hbase', 'HBASE_HOME'), 'lib') HBASE_CONF_DIR = HBase.getModifiedConfigPath() if HBASE_CONF_DIR is None or HBASE_CONF_DIR == '' or not os.path.exists( HBASE_CONF_DIR): HBASE_CONF_DIR = Config.get('hbase', 'HBASE_CONF_DIR') HADOOP_CMD = Config.get('hadoop', 'HADOOP_CMD') HBASE_USER = None if Hadoop.isSecure(): HBASE_USER = Config.get('hbase', 'HBASE_USER') HOST = Machine.getfqdn() ZK_HOST = util.getPropertyValueFromConfigXMLFile( os.path.join(HBASE_CONF_DIR, 'hbase-site.xml'), "hbase.zookeeper.quorum").split(',')[0] ZK_PORT = util.getPropertyValueFromConfigXMLFile( os.path.join(HBASE_CONF_DIR, 'hbase-site.xml'), "hbase.zookeeper.property.clientPort") if os.path.exists(os.path.join(HBASE_CONF_DIR, 'hbase-site.xml')): ZK_ZPARENT = util.getPropertyValueFromConfigXMLFile( os.path.join(HBASE_CONF_DIR, 'hbase-site.xml'), "zookeeper.znode.parent") else: ZK_ZPARENT = util.getPropertyValueFromConfigXMLFile( os.path.join(Config.get('hbase', 'HBASE_CONF_DIR'), 'hbase-site.xml'), "zookeeper.znode.parent") ENVIRONMENT = {}