def verifyLogMessageInServiceLog(text, service, timestamp=0, dateTimeFormat=None):
'''
Returns true when given log message appears in service log
'''
hiveLog = Hive.getServiceLog(service)
if not hiveLog or not text:
return None
hiveHost = Hive.getHiveHost(service)
destlog = os.path.join(Config.getEnv('ARTIFACTS_DIR'), 'tmp-%d.log' % int(999999 * random.random()))
Machine.copyToLocal(None, hiveHost, hiveLog, destlog)
return util.findMatchingPatternInFileAfterTimestamp(destlog, text, timestamp, dateTimeFormat=dateTimeFormat)
def background_job_teardown(cls):
'''
Cleanup for long running Hive jobs
'''
from beaver.component.hive import Hive
logger.info(
"Make sure to switch the HiveServer2 to use the default port")
adminUser = Machine.getAdminUser()
hiveHost = Hive.getHiveHost()
for port in cls._hs2_live_ports:
pid = Machine.getPIDByPort(port, host=hiveHost, user=adminUser)
if pid:
Machine.killProcessRemote(pid, host=hiveHost, user=adminUser)
time.sleep(2)
if len(cls._hs2_live_ports) > 0:
Hive.startService(services=["hiveserver2"])
def switch_master_version(cls, action, version, config=None):
'''
Switches Hive master services' version
:param action: Whether to "upgrade" or "downgrade"
:param version: Version to be switched to
:param config: Configuration location
'''
from beaver.component.rollingupgrade.ruCommon import hdpSelect
from beaver.component.hive import Hive
currentHiveVersion = Hive.getVersion()
if action == 'upgrade':
# Backup the database used by the Hive Metastore
logger.info(
"Performing backup of the Hive Metastore DB before starting the upgrade"
)
Hive.backupMetastoreDB(cls._metastore_backup_file)
node = Hive.getHiveHost()
# Stop the old Hive Metastore
logger.info("Stopping the Hive Metastore")
Hive.stopService(services=["metastore"])
# Upgrade Hive Metastore servers to new version
hdpSelect.changeVersion("hive-metastore", version, node)
if action == 'upgrade':
logger.info("Upgrading the Hive metastore schema")
Hive.upgradeSchema()
# Restart Hive Metastore servers one at a time
logger.info("Restarting the Hive Metastore")
Hive.startService(services=["metastore"])
# Start new Hive Server 2 instance
confHS2Port = Hive.getHiveserver2ThriftPort()
hs2port = util.getNextAvailablePort(node, confHS2Port)
hdpSelect.changeVersion("hive-server2", version, node)
Hive.modifyConfig(config,
services=['hiveserver2'],
restartService=False)
logger.info(
"Starting a new HiveServer2 at port '%d' for assisting rolling-upgrade"
% hs2port)
if hs2port != confHS2Port:
changes = {'hive-site.xml': {'hive.server2.thrift.port': hs2port}}
Hive.modifyConfig(changes,
services=["hiveserver2"],
restartService=False)
Hive.startService(services=["hiveserver2"])
cls._hs2_live_ports = [Hive.getHiveserver2ThriftPort(), hs2port]
# Deregister the old Hive Server 2 instances
logger.info("Deregistering the HiveServer2 on version '%s'" %
currentHiveVersion)
Hive.deregisterHiveServer2(version=currentHiveVersion)
from beaver.component.hcatalog import Hcatalog
# Stop the old WebHCat server
logger.info("Stopping the WebHCat server")
node = Config.get('templeton',
'TEMPLETON_HOST',
default=Machine.getfqdn())
webhcatPort = Config.get('templeton',
'TEMPLETON_PORT',
default="50111")
# Stop the old WebHCat server
logger.info("Stop the WebHCat server")
Hcatalog.stop(node)
# Upgrade WebHCat to the new version
hdpSelect.changeVersion("hive-webhcat", version, node)
# Start the WebHCat server
logger.info("Restarting the WebHCat server")
newConfDir = os.path.join(Config.getEnv('ARTIFACTS_DIR'),
'localWebhcatConf')
if os.path.exists(newConfDir):
Hcatalog.start(node, hcat_confdir=newConfDir)
else:
Hcatalog.start(node)
def setup_storm_hive_topology(cls, useStandaloneCmd):
from beaver.component.hive import Hive
storm_version = Storm.getVersion(useStandaloneCmd=True)
hive_version = Hive.getVersion()
HIVE_METASTORE_URI = Hive.getConfigValue(
"hive.metastore.uris", defaultValue="thrift://localhost:9083")
global HIVE_METASTORE_URI
global HIVE_HOST
global HIVE_PORT
global HIVE_WAREHOUSE_DIR
HIVE_WAREHOUSE_DIR = Hive.getConfigValue(
"hive.metastore.warehouse.dir",
defaultValue="/apps/hive/warehouse")
HIVE_HOST = Hive.getHiveHost()
HIVE_PORT = Hive.getMetastoreThriftPort()
if Storm.isDalorBeyond():
JAVA_HIVE_SRC_DIR = os.path.join(Config.getEnv('WORKSPACE'),
'tests', 'rolling_upgrade',
'Storm', '2_3', 'storm-hive',
'java')
else:
JAVA_HIVE_SRC_DIR = os.path.join(Config.getEnv('WORKSPACE'),
'tests', 'rolling_upgrade',
'Storm', '2_2', 'storm-hive',
'java')
# hive.txn.manager and hive.support.concurrency are set through ambari as per bug-40500
#logger.info("Restart Hive")
#changes = {'hive-site.xml': {'hive.txn.manager': 'org.apache.hadoop.hive.ql.lockmgr.DbTxnManager',
# 'hive.support.concurrency': 'true'}}
#Hive.modifyConfig(changes, services=['metastore'], restartService=True)
logger.info("Create test database in Hive")
exit_code, stdout = Hive.runQuery(
cls.get_set_queue_cmd(useStandaloneCmd) +
" drop database if exists stormdb cascade; \
create database stormdb;")
ruAssert("Storm", exit_code == 0,
"[StormHiveSetup] Failed to create test database" + stdout)
HDFS.chmod(runasUser=HDFS.getHDFSUser(),
perm=777,
directory=HIVE_WAREHOUSE_DIR + "/" + DATABASE_NAME + ".db")
#copy tests/storm/storm-hive/java to artifacts/storm-hive-tests
logger.info("JAVA_SRC_DIR " + JAVA_HIVE_SRC_DIR)
logger.info("LOCAL_WORK_DIR " + LOCAL_HIVE_WORK_DIR)
Machine.copy(JAVA_HIVE_SRC_DIR,
LOCAL_HIVE_WORK_DIR,
user=None,
passwd=None)
#mvn package
if Machine.isWindows():
(_, _) = Maven.run(
'package -D%s=%s -D%s=%s -D%s=%s -D%s=%s' %
(HADOOP_VERSION_MAVEN_PARAMETER, HADOOP_VERSION,
STORM_VERSION_MAVEN_PARAMETER, storm_version,
HIVE_VERSION_MAVEN_PARAMETER,
hive_version, PUBLIC_REPO_MAVEN_PARAMETER,
Maven.getPublicRepoUrl(), CORE_FILE_MAVEN_PARAMETER,
CORE_FILE, HADOOP_CORE_MAVEN_PARAMETER, HADOOP_CONF,
HIVE_CORE_MAVEN_PARAMETER, HIVE_CORE_DIR,
HIVE_FILE_MAVEN_PARAMETER, HIVE_FILE),
cwd=LOCAL_HIVE_WORK_DIR)
else:
(_, _) = Maven.run('package',
cwd=LOCAL_HIVE_WORK_DIR,
env={
HADOOP_VERSION_MAVEN_PARAMETER:
HADOOP_VERSION,
STORM_VERSION_MAVEN_PARAMETER:
storm_version,
HIVE_VERSION_MAVEN_PARAMETER:
hive_version,
PUBLIC_REPO_MAVEN_PARAMETER:
Maven.getPublicRepoUrl(),
CORE_FILE_MAVEN_PARAMETER:
CORE_FILE,
HADOOP_CONF_MAVEN_PARAMETER:
HADOOP_CONF,
HDFS_FILE_MAVEN_PARAMETER:
HDFS_FILE,
HADOOP_CORE_MAVEN_PARAMETER:
HADOOP_CONF,
HIVE_CORE_MAVEN_PARAMETER:
HIVE_CORE_DIR,
HIVE_FILE_MAVEN_PARAMETER:
HIVE_FILE
})
create_table_q = "use %s; \
drop table if exists %s; \
create table %s (id int, name string, phone string, street string) \
partitioned by (city string, state string) \
clustered by (id) into %s buckets \
stored as orc \
tblproperties ('transactional'='true');" % (
DATABASE_NAME, HIVE_TABLE_NAME, HIVE_TABLE_NAME, "5")
exit_code, stdout = Hive.runQuery(
cls.get_set_queue_cmd(useStandaloneCmd) + create_table_q)
ruAssert(
"Storm", exit_code == 0,
"[StormHiveSetup] Failed to create test table userdata_partitioned"
)
HDFS.chmod(runasUser=HDFS.getHDFSUser(),
perm=777,
directory=HIVE_WAREHOUSE_DIR + "/" + DATABASE_NAME +
".db/" + HIVE_TABLE_NAME)