def create_table(self, table): Logger.info("Creating table {0}".format(table['name'])) sql_cmd = "CREATE {0} TABLE {1} {2}".format(table['create_type'], table['name'], table['description']) exec_psql_cmd(sql_cmd, self.active_master_host, self.active_master_port)
def start_component(component_name, port, data_dir): """ If data directory exists start the component, else initialize the component. Initialization starts the component """ import params __check_dfs_truncate_enforced() if component_name == hawq_constants.MASTER: # Check the owner for hawq_data directory data_dir_owner = hawq_constants.hawq_user_secured if params.security_enabled else hawq_constants.hawq_user # Change owner recursively (if needed) if __get_hdfs_dir_owner() != data_dir_owner: params.HdfsResource(params.hawq_hdfs_data_dir, type="directory", action="create_on_execute", owner=data_dir_owner, group=hawq_constants.hawq_group, recursive_chown=True, mode=0755) params.HdfsResource(None, action="execute") options_str = "{0} -a -v".format(component_name) if os.path.exists(os.path.join(data_dir, hawq_constants.postmaster_opts_filename)): return utils.exec_hawq_operation(hawq_constants.START, options_str, not_if=utils.generate_hawq_process_status_cmd(component_name, port)) # Initialize HAWQ if component_name == hawq_constants.MASTER: utils.exec_hawq_operation(hawq_constants.INIT, options_str + " --ignore-bad-hosts") utils.exec_psql_cmd('create database {0};'.format(hawq_constants.hawq_user), params.hawqmaster_host, params.hawq_master_address_port, ignore_error=True) else: utils.exec_hawq_operation(hawq_constants.INIT, options_str)
def hawq_clear_cache(self, env): import params from utils import exec_psql_cmd cmd = "SELECT gp_metadata_cache_clear()" Logger.info("Clearing HAWQ's HDFS Metadata cache ...") exec_psql_cmd(cmd, params.hawqmaster_host, params.hawq_master_address_port)
def check_data_correctness(self): expected_data = "55" Logger.info("Validating data inserted, finding sum of all the inserted entries. Expected output: {0}".format(expected_data)) sql_cmd = "select sum(col1) from {0}".format(hawq_constants.smoke_check_table_name) _, stdout, _ = exec_psql_cmd(sql_cmd, self.active_master_host, tuples_only=False) if expected_data != stdout.strip(): Logger.error("Incorrect data returned. Expected Data: {0} Actual Data: {1}".format(expected_data, stdout)) raise Fail("Incorrect data returned.")
def validate_data(self, table): expected_data = "55" Logger.info("Validating data inserted, finding sum of all the inserted entries. Expected output: {0}".format(expected_data)) sql_cmd = "SELECT sum(col1) FROM {0}".format(table['name']) _, stdout, _ = exec_psql_cmd(sql_cmd, self.active_master_host, self.active_master_port, tuples_only=False) if expected_data != stdout.strip(): Logger.error("Incorrect data returned. Expected Data: {0} Actual Data: {1}".format(expected_data, stdout)) raise Fail("Incorrect data returned.")
def start_component(component_name, port, data_dir): """ If data directory exists start the component, else initialize the component. Initialization starts the component """ import params __check_dfs_truncate_enforced() if component_name == hawq_constants.MASTER: # Check the owner for hawq_data directory kinit_cmd = "{0} -kt {1} {2};".format(params.kinit_path_local, params.hdfs_user_keytab, params.hdfs_principal_name) if params.security_enabled else "" data_dir_owner = hawq_constants.hawq_user_secured if params.security_enabled else hawq_constants.hawq_user cmd = kinit_cmd + "hdfs dfs -ls {0} | sed '1d;s/ */ /g' | cut -d\\ -f3".format(params.hawq_hdfs_data_dir) returncode, stdout = call(cmd, user=params.hdfs_superuser, timeout=300) if returncode: raise # Change owner recursively (if needed) if stdout.strip() != data_dir_owner: params.HdfsResource(params.hawq_hdfs_data_dir, type="directory", action="create_on_execute", owner=data_dir_owner, group=hawq_constants.hawq_group, recursive_chown=True, mode=0755) params.HdfsResource(None, action="execute") options_str = "{0} -a -v".format(component_name) if os.path.exists(os.path.join(data_dir, hawq_constants.postmaster_opts_filename)): return utils.exec_hawq_operation(hawq_constants.START, options_str, not_if=utils.generate_hawq_process_status_cmd(component_name, port)) # Initialize HAWQ if component_name == hawq_constants.MASTER: utils.exec_hawq_operation(hawq_constants.INIT, options_str + " --ignore-bad-hosts") utils.exec_psql_cmd('create database {0};'.format(hawq_constants.hawq_user), params.hawqmaster_host, params.hawq_master_address_port, ignore_error=True) else: utils.exec_hawq_operation(hawq_constants.INIT, options_str)
def check_data_correctness(self): expected_data = "55" Logger.info( "Validating data inserted, finding sum of all the inserted entries. Expected output: {0}" .format(expected_data)) sql_cmd = "select sum(col1) from {0}".format( hawqconstants.smoke_check_table_name) _, stdout, _ = exec_psql_cmd(sql_cmd, self.active_master_host, tuples_only=False) if expected_data != stdout.strip(): Logger.error( "Incorrect data returned. Expected Data: {0} Actual Data: {1}". format(expected_data, stdout)) raise Fail("Incorrect data returned.")
def query_data(self): Logger.info("Querying data from table {0}".format(hawq_constants.smoke_check_table_name)) sql_cmd = "select * from {0}".format(hawq_constants.smoke_check_table_name) exec_psql_cmd(sql_cmd, self.active_master_host)
def insert_data(self): Logger.info("Inserting data to table {0}".format(hawq_constants.smoke_check_table_name)) sql_cmd = "insert into {0} select * from generate_series(1,10)".format(hawq_constants.smoke_check_table_name) exec_psql_cmd(sql_cmd, self.active_master_host)
def create_table(self): Logger.info("Creating table {0}".format(hawq_constants.smoke_check_table_name)) sql_cmd = "create table {0} (col1 int) distributed randomly".format(hawq_constants.smoke_check_table_name) exec_psql_cmd(sql_cmd, self.active_master_host)
def drop_table(self): Logger.info("Dropping {0} table if exists".format(hawq_constants.smoke_check_table_name)) sql_cmd = "drop table if exists {0}".format(hawq_constants.smoke_check_table_name) exec_psql_cmd(sql_cmd, self.active_master_host)
def query_data(self, table): Logger.info("Querying data from table {0}".format(table['name'])) sql_cmd = "SELECT * FROM {0}".format(table['name']) exec_psql_cmd(sql_cmd, self.active_master_host, self.active_master_port)
def insert_data(self, table): Logger.info("Inserting data to table {0}".format(table['name'])) sql_cmd = "INSERT INTO {0} SELECT * FROM generate_series(1,10)".format( table['name']) exec_psql_cmd(sql_cmd, self.active_master_host, self.active_master_port)
def drop_table(self): Logger.info("Dropping {0} table if exists".format( hawqconstants.smoke_check_table_name)) sql_cmd = "drop table if exists {0}".format( hawqconstants.smoke_check_table_name) exec_psql_cmd(sql_cmd, self.active_master_host)
def drop_table(self, table): Logger.info("Dropping {0} table if exists".format(table['name'])) sql_cmd = "DROP {0} TABLE IF EXISTS {1}".format( table['drop_type'], table['name']) exec_psql_cmd(sql_cmd, self.active_master_host, self.active_master_port)
def query_data(self): Logger.info("Querying data from table {0}".format( hawqconstants.smoke_check_table_name)) sql_cmd = "select * from {0}".format( hawqconstants.smoke_check_table_name) exec_psql_cmd(sql_cmd, self.active_master_host)
def insert_data(self): Logger.info("Inserting data to table {0}".format( hawqconstants.smoke_check_table_name)) sql_cmd = "insert into {0} select * from generate_series(1,10)".format( hawqconstants.smoke_check_table_name) exec_psql_cmd(sql_cmd, self.active_master_host)
def create_table(self): Logger.info("Creating table {0}".format( hawqconstants.smoke_check_table_name)) sql_cmd = "create table {0} (col1 int) distributed randomly".format( hawqconstants.smoke_check_table_name) exec_psql_cmd(sql_cmd, self.active_master_host)