def getConfigValue(cls, propertyValue, defaultValue=None):
_localHiveConf = os.path.join(Config.getEnv('ARTIFACTS_DIR'), cls._localConfDir)
if not os.path.exists(_localHiveConf):
admin_user = Machine.getAdminUser()
Machine.copyToLocal(admin_user, cls.getHiveHost(), cls._configDir, _localHiveConf)
Machine.chmod('777', _localHiveConf, recursive=True, user=admin_user)
if cls._modifiedConfigs.has_key(propertyValue):
return cls._modifiedConfigs[propertyValue]
value = util.getPropertyValueFromConfigXMLFile(
os.path.join(_localHiveConf, 'hiveserver2-site.xml'), propertyValue
)
if value is None or value == '':
value = util.getPropertyValueFromConfigXMLFile(
os.path.join(_localHiveConf, 'hive-site.xml'), propertyValue, defaultValue=defaultValue
)
return value
def HDFS_getConfigValue(cls, propertyValue, defaultValue=None):
try:
hdfs_conf_path = os.path.join(Config.get('hadoop', 'HADOOP_CONF'),
"hdfs-site.xml")
logger.info("hdfs_conf_path is %s", hdfs_conf_path)
return util.getPropertyValueFromConfigXMLFile(
hdfs_conf_path, propertyValue, defaultValue=defaultValue)
except Exception:
logger.error("Exception occured during HDFS_getConfigValue() call")
logger.error(traceback.format_exc())
return None
def insertCSVDataViaPSQL(cls, csvFile, tableName, env=None):
"""
If tableName==None, the table will be created with the name of the file.
"""
global ZK_ZPARENT
if Slider.isSlider():
ZK_ZPARENT = util.getPropertyValueFromConfigXMLFile(
os.path.join(Config.get('hbase', 'HBASE_CONF_DIR'),
'hbase-site.xml'), "zookeeper.znode.parent")
if not env:
env = {}
env['HBASE_CONF_PATH'] = Config.get('hbase', 'HBASE_CONF_DIR')
# We save the current directory
currentDir = os.getcwd()
# We go to phoenix / bin folder
os.chdir(os.path.join(PHOENIX_HOME, 'bin'))
# We run the command
if ZK_ZPARENT is not None and ZK_ZPARENT != '':
command = "%s -t %s %s:%s:%s %s" % (PSQL_SCRIPT, tableName.upper(),
ZK_HOST, ZK_PORT, ZK_ZPARENT,
csvFile)
else:
command = "%s -t %s %s:%s %s" % (PSQL_SCRIPT, tableName.upper(),
ZK_HOST, ZK_PORT, csvFile)
result = Machine.run(
command,
env=dict(env.items() +
ENVIRONMENT.items() if env is not None else ENVIRONMENT))
# We restore the execution directory
os.chdir(currentDir)
# we return the results
return result
def getConfigValue(cls, propertyValue, defaultValue=None):
return util.getPropertyValueFromConfigXMLFile(os.path.join(Config.get('hadoop', 'HADOOP_CONF'), "core-site.xml"), propertyValue, defaultValue=defaultValue)
def getConfigValue(cls, propertyValue, defaultValue=None):
return util.getPropertyValueFromConfigXMLFile(
os.path.join(OOZIE_CONF_DIR, "oozie-site.xml"),
propertyValue,
defaultValue=defaultValue)
def getLocalDirInfo(host):
return util.getPropertyValueFromConfigXMLFile(os.path.join(Config.get('hadoop', 'HADOOP_CONF'), "mapred-site.xml"), "mapred.local.dir")
def insertCSVDataViaCSVBuildLoad(cls,
csvFile,
tableName,
putIntoHDFS=True,
deleteAfterExec=True,
runInBackground=False,
user=None,
config=None,
optionAndParameter="",
env=None,
delimiter=",",
arrayDelimiter=None,
schema=None):
"""
By default, the files will be allocated under /tmp/ folder in HDFS.
"""
global ZK_ZPARENT
if Slider.isSlider():
ZK_ZPARENT = util.getPropertyValueFromConfigXMLFile(
os.path.join(Config.get('hbase', 'HBASE_CONF_DIR'),
'hbase-site.xml'), "zookeeper.znode.parent")
if Machine.isLinux():
clientjar = Machine.find(user=Machine.getAdminUser(),
host="localhost",
filepath=PHOENIX_HOME,
searchstr="phoenix-*[0-9]-client.jar",
passwd=Machine.getAdminPasswd())
else:
clientjar = Machine.find(user=Machine.getAdminUser(),
host="localhost",
filepath=PHOENIX_HOME,
searchstr="phoenix-*-client.jar",
passwd=Machine.getAdminPasswd())
if Machine.isWindows():
clientjar = (clientjar[0].strip("\\localhost")).replace("$", ":")
fileName = csvFile.split('\\')[-1]
else:
clientjar = clientjar[0]
fileName = csvFile.split('/')[-1]
# If we need to, we insert it into HDFS, since the library will take it from there.
executingUser = (HADOOPQA_USER) if user is None else user
if putIntoHDFS:
if not HDFS.fileExists('/tmp/'):
HDFS.mkdir('/tmp/')
HDFS.copyFromLocal(csvFile, '/tmp/', executingUser, config,
optionAndParameter)
hbaseConfDir = HBASE_CONF_DIR
if Slider.isSlider():
hbaseConfDir = Config.get('hbase', 'HBASE_CONF_DIR')
classpath = hbaseConfDir
finalCommand = "%s jar %s org.apache.phoenix.mapreduce.CsvBulkLoadTool --table %s --input %s" \
% (HADOOP_CMD, clientjar, tableName, '/tmp/%s' % fileName)
if schema is not None:
finalCommand = '%s -schema %s' % (finalCommand, schema)
if Machine.isWindows():
os.environ['HADOOP_USER_CLASSPATH_FIRST'] = 'true'
os.environ['HADOOP_CLASSPATH'] = classpath
if delimiter != "," or arrayDelimiter != None:
finalCommand = "%s -d `\\\"`%s`\\\" -a `\\\"`%s`\\\"" \
% (finalCommand, delimiter, arrayDelimiter.strip("'"))
finalCommand = "%s --zookeeper %s" % (finalCommand, ZK_HOST)
if runInBackground:
exit_code = 0
stdout = ''
Machine.runinbackground(
finalCommand,
env=dict(env.items() + ENVIRONMENT.items()
if env is not None else ENVIRONMENT))
else:
exit_code, stdout = Machine.run(
finalCommand,
env=dict(env.items() + ENVIRONMENT.items()
if env is not None else ENVIRONMENT))
else:
# delimiter options
if delimiter != "," or arrayDelimiter != None:
finalCommand = "%s --delimiter %s --array-delimiter %s" % (
finalCommand, delimiter, arrayDelimiter)
# ZKHosts options
finalCommand = "%s --zookeeper %s" % (finalCommand,
cls.getZKConnectString())
if runInBackground:
exit_code = 0
stdout = ''
Machine.runinbackground(
"HADOOP_CLASSPATH=%s %s" % (classpath, finalCommand),
env=dict(env.items() + ENVIRONMENT.items()
if env is not None else ENVIRONMENT))
else:
exit_code, stdout = Machine.run(
"HADOOP_CLASSPATH=%s %s" % (classpath, finalCommand),
env=dict(env.items() + ENVIRONMENT.items()
if env is not None else ENVIRONMENT))
# If selected, after insertion into HBase we will delete the csvFile from HDFS
if deleteAfterExec and not runInBackground:
# Does not work for "run in background" option
HDFS.deleteFile('/tmp/%s' % fileName, executingUser)
# return 0,""
return exit_code, stdout
def runSQLLineCmdsWithoutEnvVars(cls,
cmd,
env=None,
logoutput=True,
outputFormat=None,
threadNumber=None,
user=None):
global ZK_ZPARENT
if Slider.isSlider():
ZK_ZPARENT = util.getPropertyValueFromConfigXMLFile(
os.path.join(Config.get('hbase', 'HBASE_CONF_DIR'),
'hbase-site.xml'), "zookeeper.znode.parent")
if not env:
env = {}
if Hadoop.isSecure():
if user is None:
user = Config.getEnv('USER')
kerbTicket = Machine.getKerberosTicket(user)
env['KRB5CCNAME'] = kerbTicket
user = None
folder = os.path.join(ARTIFACTS_DIR, 'phoenixSQLQuery')
if threadNumber is None:
sqlConsultFile = os.path.join(folder, INPUT_FILE + '.txt')
else:
sqlConsultFile = os.path.join(
folder, INPUT_FILE + "_thread_" + str(threadNumber) + ".txt")
# We remove the file if exists
Machine.rm(user, HOST, sqlConsultFile)
# We create aux file
Machine.makedirs(user, HOST, folder)
# We write the sql command to the file. If we are in ASV, we get the output in csv format
if outputFormat is not None:
cmd = '!outputformat %s\n%s' % (outputFormat, cmd)
if HDFS.isASV() and Machine.isWindows():
cmd = '!outputformat csv\n%s' % cmd
# We write the sql command to the file
fh = open(sqlConsultFile, 'w')
fh.write(cmd)
fh.close()
logger.info("RUNNING Command in input file: %s", cmd)
# We save the current directory
currentDirectory = os.getcwd()
# We change the path to execute sqlline.py from /bin folder
os.chdir(os.path.join(PHOENIX_HOME, 'bin'))
exportCommand = ""
# We run the command to execute SQL instruction
if RUN_WITH_SQLLINE_THIN:
queryServer = None
if QUERYSERVER_URL is None:
# Since phoenix-QueryServer will be running on all regionservers, we re-direct the query
# to any currently active regionserver
phoenixqueryservers = cls.getPhoenixQueryServers()
assert phoenixqueryservers, "Query failed. No active RS found with cls.getPhoenixQueryServers()"
queryServer = phoenixqueryservers[0]
else:
queryServer = QUERYSERVER_URL
assert queryServer is not None, "Should have a URL to a PQS instance"
completeCommand = '%s %s %s %s' % (exportCommand,
SQLLINE_THIN_SCRIPT,
queryServer, sqlConsultFile)
else:
if ZK_ZPARENT is not None and ZK_ZPARENT != '':
completeCommand = '%s %s %s:%s:%s %s' % (
exportCommand, SQLLINE_SCRIPT, ZK_HOST, ZK_PORT,
ZK_ZPARENT, sqlConsultFile)
else:
completeCommand = '%s %s %s:%s %s' % (exportCommand,
SQLLINE_SCRIPT, ZK_HOST,
ZK_PORT, sqlConsultFile)
result = Machine.runas(
user,
completeCommand,
env=dict(env.items() +
ENVIRONMENT.items() if env is not None else ENVIRONMENT),
logoutput=logoutput)
# we restore the previous directory
os.chdir(currentDirectory)
# We remove the file
Machine.rm(user, HOST, sqlConsultFile)
return result
HBASE_LIB_DIR = os.path.join(Config.get('hbase', 'HBASE_HOME'), 'lib')
HBASE_CONF_DIR = HBase.getModifiedConfigPath()
if HBASE_CONF_DIR is None or HBASE_CONF_DIR == '' or not os.path.exists(
HBASE_CONF_DIR):
HBASE_CONF_DIR = Config.get('hbase', 'HBASE_CONF_DIR')
HADOOP_CMD = Config.get('hadoop', 'HADOOP_CMD')
HBASE_USER = None
if Hadoop.isSecure():
HBASE_USER = Config.get('hbase', 'HBASE_USER')
HOST = Machine.getfqdn()
ZK_HOST = util.getPropertyValueFromConfigXMLFile(
os.path.join(HBASE_CONF_DIR, 'hbase-site.xml'),
"hbase.zookeeper.quorum").split(',')[0]
ZK_PORT = util.getPropertyValueFromConfigXMLFile(
os.path.join(HBASE_CONF_DIR, 'hbase-site.xml'),
"hbase.zookeeper.property.clientPort")
if os.path.exists(os.path.join(HBASE_CONF_DIR, 'hbase-site.xml')):
ZK_ZPARENT = util.getPropertyValueFromConfigXMLFile(
os.path.join(HBASE_CONF_DIR, 'hbase-site.xml'),
"zookeeper.znode.parent")
else:
ZK_ZPARENT = util.getPropertyValueFromConfigXMLFile(
os.path.join(Config.get('hbase', 'HBASE_CONF_DIR'), 'hbase-site.xml'),
"zookeeper.znode.parent")
ENVIRONMENT = {}
