def create_table(self, table):
Logger.info("Creating table {0}".format(table['name']))
sql_cmd = "CREATE {0} TABLE {1} {2}".format(table['create_type'],
table['name'],
table['description'])
exec_psql_cmd(sql_cmd, self.active_master_host,
self.active_master_port)
def start_component(component_name, port, data_dir):
"""
If data directory exists start the component, else initialize the component.
Initialization starts the component
"""
import params
__check_dfs_truncate_enforced()
if component_name == hawq_constants.MASTER:
# Check the owner for hawq_data directory
data_dir_owner = hawq_constants.hawq_user_secured if params.security_enabled else hawq_constants.hawq_user
# Change owner recursively (if needed)
if __get_hdfs_dir_owner() != data_dir_owner:
params.HdfsResource(params.hawq_hdfs_data_dir,
type="directory",
action="create_on_execute",
owner=data_dir_owner,
group=hawq_constants.hawq_group,
recursive_chown=True,
mode=0755)
params.HdfsResource(None, action="execute")
options_str = "{0} -a -v".format(component_name)
if os.path.exists(os.path.join(data_dir, hawq_constants.postmaster_opts_filename)):
return utils.exec_hawq_operation(hawq_constants.START, options_str,
not_if=utils.generate_hawq_process_status_cmd(component_name, port))
# Initialize HAWQ
if component_name == hawq_constants.MASTER:
utils.exec_hawq_operation(hawq_constants.INIT, options_str + " --ignore-bad-hosts")
utils.exec_psql_cmd('create database {0};'.format(hawq_constants.hawq_user),
params.hawqmaster_host, params.hawq_master_address_port, ignore_error=True)
else:
utils.exec_hawq_operation(hawq_constants.INIT, options_str)
def hawq_clear_cache(self, env):
import params
from utils import exec_psql_cmd
cmd = "SELECT gp_metadata_cache_clear()"
Logger.info("Clearing HAWQ's HDFS Metadata cache ...")
exec_psql_cmd(cmd, params.hawqmaster_host,
params.hawq_master_address_port)
def check_data_correctness(self):
expected_data = "55"
Logger.info("Validating data inserted, finding sum of all the inserted entries. Expected output: {0}".format(expected_data))
sql_cmd = "select sum(col1) from {0}".format(hawq_constants.smoke_check_table_name)
_, stdout, _ = exec_psql_cmd(sql_cmd, self.active_master_host, tuples_only=False)
if expected_data != stdout.strip():
Logger.error("Incorrect data returned. Expected Data: {0} Actual Data: {1}".format(expected_data, stdout))
raise Fail("Incorrect data returned.")
def validate_data(self, table):
expected_data = "55"
Logger.info("Validating data inserted, finding sum of all the inserted entries. Expected output: {0}".format(expected_data))
sql_cmd = "SELECT sum(col1) FROM {0}".format(table['name'])
_, stdout, _ = exec_psql_cmd(sql_cmd, self.active_master_host, self.active_master_port, tuples_only=False)
if expected_data != stdout.strip():
Logger.error("Incorrect data returned. Expected Data: {0} Actual Data: {1}".format(expected_data, stdout))
raise Fail("Incorrect data returned.")
def start_component(component_name, port, data_dir):
"""
If data directory exists start the component, else initialize the component.
Initialization starts the component
"""
import params
__check_dfs_truncate_enforced()
if component_name == hawq_constants.MASTER:
# Check the owner for hawq_data directory
kinit_cmd = "{0} -kt {1} {2};".format(params.kinit_path_local, params.hdfs_user_keytab, params.hdfs_principal_name) if params.security_enabled else ""
data_dir_owner = hawq_constants.hawq_user_secured if params.security_enabled else hawq_constants.hawq_user
cmd = kinit_cmd + "hdfs dfs -ls {0} | sed '1d;s/ */ /g' | cut -d\\ -f3".format(params.hawq_hdfs_data_dir)
returncode, stdout = call(cmd, user=params.hdfs_superuser, timeout=300)
if returncode:
raise
# Change owner recursively (if needed)
if stdout.strip() != data_dir_owner:
params.HdfsResource(params.hawq_hdfs_data_dir,
type="directory",
action="create_on_execute",
owner=data_dir_owner,
group=hawq_constants.hawq_group,
recursive_chown=True,
mode=0755)
params.HdfsResource(None, action="execute")
options_str = "{0} -a -v".format(component_name)
if os.path.exists(os.path.join(data_dir, hawq_constants.postmaster_opts_filename)):
return utils.exec_hawq_operation(hawq_constants.START, options_str,
not_if=utils.generate_hawq_process_status_cmd(component_name, port))
# Initialize HAWQ
if component_name == hawq_constants.MASTER:
utils.exec_hawq_operation(hawq_constants.INIT, options_str + " --ignore-bad-hosts")
utils.exec_psql_cmd('create database {0};'.format(hawq_constants.hawq_user),
params.hawqmaster_host, params.hawq_master_address_port, ignore_error=True)
else:
utils.exec_hawq_operation(hawq_constants.INIT, options_str)
def check_data_correctness(self):
expected_data = "55"
Logger.info(
"Validating data inserted, finding sum of all the inserted entries. Expected output: {0}"
.format(expected_data))
sql_cmd = "select sum(col1) from {0}".format(
hawqconstants.smoke_check_table_name)
_, stdout, _ = exec_psql_cmd(sql_cmd,
self.active_master_host,
tuples_only=False)
if expected_data != stdout.strip():
Logger.error(
"Incorrect data returned. Expected Data: {0} Actual Data: {1}".
format(expected_data, stdout))
raise Fail("Incorrect data returned.")
def query_data(self):
Logger.info("Querying data from table {0}".format(hawq_constants.smoke_check_table_name))
sql_cmd = "select * from {0}".format(hawq_constants.smoke_check_table_name)
exec_psql_cmd(sql_cmd, self.active_master_host)
def insert_data(self):
Logger.info("Inserting data to table {0}".format(hawq_constants.smoke_check_table_name))
sql_cmd = "insert into {0} select * from generate_series(1,10)".format(hawq_constants.smoke_check_table_name)
exec_psql_cmd(sql_cmd, self.active_master_host)
def create_table(self):
Logger.info("Creating table {0}".format(hawq_constants.smoke_check_table_name))
sql_cmd = "create table {0} (col1 int) distributed randomly".format(hawq_constants.smoke_check_table_name)
exec_psql_cmd(sql_cmd, self.active_master_host)
def drop_table(self):
Logger.info("Dropping {0} table if exists".format(hawq_constants.smoke_check_table_name))
sql_cmd = "drop table if exists {0}".format(hawq_constants.smoke_check_table_name)
exec_psql_cmd(sql_cmd, self.active_master_host)
def query_data(self, table):
Logger.info("Querying data from table {0}".format(table['name']))
sql_cmd = "SELECT * FROM {0}".format(table['name'])
exec_psql_cmd(sql_cmd, self.active_master_host,
self.active_master_port)
def insert_data(self, table):
Logger.info("Inserting data to table {0}".format(table['name']))
sql_cmd = "INSERT INTO {0} SELECT * FROM generate_series(1,10)".format(
table['name'])
exec_psql_cmd(sql_cmd, self.active_master_host,
self.active_master_port)
def drop_table(self):
Logger.info("Dropping {0} table if exists".format(
hawqconstants.smoke_check_table_name))
sql_cmd = "drop table if exists {0}".format(
hawqconstants.smoke_check_table_name)
exec_psql_cmd(sql_cmd, self.active_master_host)
def drop_table(self, table):
Logger.info("Dropping {0} table if exists".format(table['name']))
sql_cmd = "DROP {0} TABLE IF EXISTS {1}".format(
table['drop_type'], table['name'])
exec_psql_cmd(sql_cmd, self.active_master_host,
self.active_master_port)
def query_data(self):
Logger.info("Querying data from table {0}".format(
hawqconstants.smoke_check_table_name))
sql_cmd = "select * from {0}".format(
hawqconstants.smoke_check_table_name)
exec_psql_cmd(sql_cmd, self.active_master_host)
def insert_data(self):
Logger.info("Inserting data to table {0}".format(
hawqconstants.smoke_check_table_name))
sql_cmd = "insert into {0} select * from generate_series(1,10)".format(
hawqconstants.smoke_check_table_name)
exec_psql_cmd(sql_cmd, self.active_master_host)
def create_table(self):
Logger.info("Creating table {0}".format(
hawqconstants.smoke_check_table_name))
sql_cmd = "create table {0} (col1 int) distributed randomly".format(
hawqconstants.smoke_check_table_name)
exec_psql_cmd(sql_cmd, self.active_master_host)
