def init_test_database(cls, dbflavor):
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
if dbflavor == "mysql":
from beaver.dbutil import MySQL
MySQL.recreateDatabase(cls._test_db)
MySQL.createUserWithAllPriveleges(
cls._test_db_user,
cls._test_db_passwd,
hosts=['%', cls._test_db_host, 'localhost'],
database=cls._test_db)
elif dbflavor == "oracle":
from beaver.dbutil import Oracle
Oracle.dropUser(cls._test_db_user, database=cls._test_db_oracle)
Oracle.createUser(cls._test_db_user,
cls._test_db_passwd,
database=cls._test_db_oracle)
elif dbflavor.startswith("postgres"):
from beaver.dbutil import Postgres
Postgres.recreateDatabase(cls._test_db)
Postgres.createUser(cls._test_db_user, cls._test_db_passwd)
Postgres.grantAllPrivileges(cls._test_db_user, cls._test_db)
else:
UpgradePerNode.reportProgress(
"[FAILED][SQOOP][INIT] Invalid database flavor '%s' " %
dbflavor)
def submit_storm_jdbc_topology(cls, tcId, className, args, topologyName,
useStandaloneCmd):
"""
Name:
Storm-JDBC Topology
Description:
Testing storm-jdbc topology in distributed mode
1. UserPersistanceTopology
"""
from beaver.dbutil import MySQL
MySQL.runAsRoot("CREATE DATABASE test")
MySQL.runAsRoot("show databases")
exit_code, stdout = Storm.runStormJdbcTopology(
TARGET_JDBC_STORM_JAR,
className,
args,
None,
logoutput=True,
inBackground=False,
useStandaloneCmd=useStandaloneCmd)
ruAssert("Storm", exit_code == 0,
"[StormJDBCSubmit] %s Failed" % (tcId))
def backupMetastoreDB(cls, backupfile):
dbflavor = cls.getDatabaseFlavor()
jdbcUrl = cls.getConfigValue("javax.jdo.option.ConnectionURL")
if dbflavor == 'mysql':
m = re.search("jdbc:mysql://.*/([^?]*)", jdbcUrl)
if m:
dbname = m.group(1)
from beaver.dbutil import MySQL
logger.info("Backing up the Hive metastore database [%s]" % dbname)
MySQL.backupDatabase(dbname, backupfile)
else:
logger.info("Failed to extract the database name from the JDBC connection url")
elif dbflavor.startswith('postgres'):
m = re.search("jdbc:postgresql://.*/([^?]*)", jdbcUrl)
if m:
dbname = m.group(1)
from beaver.dbutil import Postgres
logger.info("Backing up the Hive metastore database [%s]" % dbname)
Postgres.backupDatabase(dbname, backupfile)
else:
logger.info("Failed to extract the database name from the JDBC connection url")
elif dbflavor == 'oracle':
from beaver.dbutil import Oracle
hiveuser = cls.getConfigValue("javax.jdo.option.ConnectionUserName")
hivepasswd = cls.getConfigValue("javax.jdo.option.ConnectionPassword", defaultValue="hive")
Oracle.backupSchema(backupfile, hiveuser, hivepasswd)
else:
logger.info("Not implemented for DB flavor: %s" % dbflavor)
def tear_down_jdbc_topology(cls, topologyName, useStandaloneCmd):
"""
kills Jbbc topology.
"""
from beaver.dbutil import MySQL
Storm.killTopology(topologyName,
logoutput=True,
useStandaloneCmd=useStandaloneCmd)
MySQL.runCmd("drop table user",
database="test",
host=mysqlnode,
user=Config.get('machine', 'MYSQL_ROOT_USER'))
MySQL.runAsRoot("DROP DATABASE IF EXISTS test")
def run_database_query(cls, query, dbflavor):
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
if dbflavor == "mysql":
from beaver.dbutil import MySQL
return MySQL.runCmd(query,
host=cls._test_db_host,
database=cls._test_db,
user=cls._test_db_user,
passwd=cls._test_db_passwd)
elif dbflavor == "oracle":
from beaver.dbutil import Oracle
return Oracle.runCmd(query,
database=cls._test_db_oracle,
user=cls._test_db_user,
passwd=cls._test_db_passwd)
elif dbflavor.startswith("postgres"):
from beaver.dbutil import Postgres
return Postgres.runCmd(query,
database=cls._test_db,
user=cls._test_db_user,
passwd=cls._test_db_passwd)
else:
UpgradePerNode.reportProgress(
"[FAILED][SQOOP][INIT] Invalid database flavor '%s' " %
dbflavor)
def setup_storm_jdbc_topology(cls, useStandaloneCmd):
from beaver.dbutil import MySQL
MySQL.createUserWithAllPriveleges(STORM_JDBC_TEST_USER,
STORM_JDBC_TEST_PASSWD)
storm_version = Storm.getVersion(useStandaloneCmd=True)
try:
file_obj = open(HDFS_CONFIG_FILE, 'w')
if Hadoop.isSecure():
file_obj.write('hdfs.keytab.file: ' +
Machine.getHeadlessUserKeytab(
user=HADOOPQA_USER) + '\n')
file_obj.write('hdfs.kerberos.principal: ' +
Machine.get_user_principal(user=HADOOPQA_USER) +
'\n')
finally:
file_obj.close()
Machine.copy(JAVA_JDBC_SRC_DIR,
LOCAL_JDBC_WORK_DIR,
user=None,
passwd=None)
post_fenton_opt = " -DpostFenton=true" if Storm.isAfterFenton() else ""
package_cmd = 'package ' + post_fenton_opt
(exit_code, _) = Maven.run(package_cmd,
cwd=LOCAL_JDBC_WORK_DIR,
env={
HADOOP_VERSION_MAVEN_PARAMETER:
HADOOP_VERSION,
STORM_VERSION_MAVEN_PARAMETER:
storm_version,
HADOOP_CONF_MAVEN_PARAMETER:
HADOOP_CONF,
HDFS_FILE_MAVEN_PARAMETER:
HDFS_FILE,
CORE_FILE_MAVEN_PARAMETER:
CORE_FILE,
PUBLIC_REPO_MAVEN_PARAMETER:
Maven.getPublicRepoUrl()
})
ruAssert("Storm", exit_code == 0,
"[StormJDBCSetup] maven package command failed")
def getRowCountFromMySQL(cls, tablename, database):
'''
finds out row count from a table
:param tablename:
:param database:
:return:
'''
from beaver.dbutil import MySQL
exit_code, stdout = MySQL.runCmd("SELECT COUNT(*) FROM %s" % tablename,
host=mysqlnode,
database="%s" % database,
user=Config.get(
'machine', 'MYSQL_ROOT_USER'))
if exit_code != 0:
return 0
else:
return int(stdout.split("\n")[1])
def restoreMetastoreDB(cls, backupfile):
dbflavor = cls.getDatabaseFlavor()
jdbcUrl = cls.getConfigValue("javax.jdo.option.ConnectionURL")
hiveuser = cls.getConfigValue("javax.jdo.option.ConnectionUserName")
if dbflavor == 'mysql':
m = re.search("jdbc:mysql://.*/([^?]*)", jdbcUrl)
if m:
dbname = m.group(1)
from beaver.dbutil import MySQL
logger.info("Recreating the database [%s]" % dbname)
MySQL.recreateDatabase(dbname)
logger.info("Grant all privileges to user [%s] on database [%s]" % (hiveuser, dbname))
MySQL.grantAllPrivileges(hiveuser, host='%', database=dbname)
MySQL.flushPriveleges()
logger.info("Restoring the backup from file '%s'" % backupfile)
MySQL.runAsRoot("source " + backupfile, database=dbname)
else:
logger.info("Failed to extract the database name from the JDBC connection url")
elif dbflavor.startswith('postgres'):
m = re.search("jdbc:postgresql://.*/([^?]*)", jdbcUrl)
if m:
dbname = m.group(1)
from beaver.dbutil import Postgres
logger.info("Recreating the database [%s]" % dbname)
Postgres.recreateDatabase(dbname)
logger.info("Grant all privileges to user [%s] on database [%s]" % (hiveuser, dbname))
Postgres.grantAllPrivileges(hiveuser, dbname)
logger.info("Restoring the backup from file '%s'" % backupfile)
Postgres.importDmp(backupfile, dbname)
else:
logger.info("Failed to extract the database name from the JDBC connection url")
elif dbflavor == "oracle":
from beaver.dbutil import Oracle
hiveuser = cls.getConfigValue("javax.jdo.option.ConnectionUserName")
hivepasswd = cls.getConfigValue("javax.jdo.option.ConnectionPassword", defaultValue="hive")
logger.info("Restoring the backup from file '%s'" % backupfile)
Oracle.importDmp(backupfile, fromuser=hiveuser, touser=hiveuser, touserpasswd=hivepasswd)
else:
logger.info("Not implemented for DB flavor: %s" % dbflavor)