def after_scenario(context, scenario):
if 'gpexpand' in context.feature.tags or 'gpaddmirrors' in context.feature.tags:
return
if 'analyzedb' not in context.feature.tags:
start_database_if_not_started(context)
home_dir = os.path.expanduser('~')
if not check_user_permissions(home_dir, 'write') and hasattr(context, 'orig_write_permission')\
and context.orig_write_permission:
run_command(context, 'sudo chmod u+w %s' % home_dir)
if os.path.isdir('%s/gpAdminLogs.bk' % home_dir):
shutil.move('%s/gpAdminLogs.bk' % home_dir, '%s/gpAdminLogs' % home_dir)
if 'gpssh' in context.feature.tags:
run_command(context, 'sudo tc qdisc del dev lo root netem')
# for cleaning up after @given('"{path}" has its permissions set to "{perm}"')
if (hasattr(context, 'path_for_which_to_restore_the_permissions') and
hasattr(context, 'permissions_to_restore_path_to')):
os.chmod(context.path_for_which_to_restore_the_permissions, context.permissions_to_restore_path_to)
elif hasattr(context, 'path_for_which_to_restore_the_permissions'):
raise Exception('Missing permissions_to_restore_path_to for %s' %
context.path_for_which_to_restore_the_permissions)
elif hasattr(context, 'permissions_to_restore_path_to'):
raise Exception('Missing path_for_which_to_restore_the_permissions despite the specified permission %o' %
context.permissions_to_restore_path_to)
def after_scenario(context, scenario):
tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpstate', 'gpinitstandby']
if set(context.feature.tags).intersection(tags_to_skip):
return
if 'gpmovemirrors' in context.feature.tags:
if 'temp_base_dir' in context:
shutil.rmtree(context.temp_base_dir)
if 'analyzedb' not in context.feature.tags:
start_database_if_not_started(context)
home_dir = os.path.expanduser('~')
if not check_user_permissions(home_dir, 'write') and hasattr(context, 'orig_write_permission')\
and context.orig_write_permission:
run_command(context, 'sudo chmod u+w %s' % home_dir)
if os.path.isdir('%s/gpAdminLogs.bk' % home_dir):
shutil.move('%s/gpAdminLogs.bk' % home_dir, '%s/gpAdminLogs' % home_dir)
if 'gpssh' in context.feature.tags:
run_command(context, 'sudo tc qdisc del dev lo root netem')
# for cleaning up after @given('"{path}" has its permissions set to "{perm}"')
if (hasattr(context, 'path_for_which_to_restore_the_permissions') and
hasattr(context, 'permissions_to_restore_path_to')):
os.chmod(context.path_for_which_to_restore_the_permissions, context.permissions_to_restore_path_to)
elif hasattr(context, 'path_for_which_to_restore_the_permissions'):
raise Exception('Missing permissions_to_restore_path_to for %s' %
context.path_for_which_to_restore_the_permissions)
elif hasattr(context, 'permissions_to_restore_path_to'):
raise Exception('Missing path_for_which_to_restore_the_permissions despite the specified permission %o' %
context.permissions_to_restore_path_to)
def after_scenario(context, scenario):
if 'analyzedb' not in context.feature.tags:
master_data_dir = os.environ.get('MASTER_DATA_DIRECTORY')
fn = os.path.join(master_data_dir, 'dirty_hack.txt')
if os.path.exists(fn):
#os.remove(fn)
pass
start_database_if_not_started(context)
home_dir = os.path.expanduser('~')
if not check_user_permissions(home_dir, 'write') and hasattr(context, 'orig_write_permission')\
and context.orig_write_permission:
run_command(context, 'sudo chmod u+w %s' % home_dir)
if os.path.isdir('%s/gpAdminLogs.bk' % home_dir):
shutil.move('%s/gpAdminLogs.bk' % home_dir, '%s/gpAdminLogs' % home_dir)
if 'gpssh' in context.feature.tags:
run_command(context, 'sudo tc qdisc del dev lo root netem')
# for cleaning up after @given('"{path}" has its permissions set to "{perm}"')
if (hasattr(context, 'path_for_which_to_restore_the_permissions') and
hasattr(context, 'permissions_to_restore_path_to')):
os.chmod(context.path_for_which_to_restore_the_permissions, context.permissions_to_restore_path_to)
elif hasattr(context, 'path_for_which_to_restore_the_permissions'):
raise Exception('Missing permissions_to_restore_path_to for %s' %
context.path_for_which_to_restore_the_permissions)
elif hasattr(context, 'permissions_to_restore_path_to'):
raise Exception('Missing path_for_which_to_restore_the_permissions despite the specified permission %o' %
context.permissions_to_restore_path_to)
def after_scenario(context, scenario):
#TODO: you'd think that the scenario.skip() in before_scenario() would
# cause this to not be needed
if "skip" in scenario.effective_tags:
return
if 'tablespaces' in context:
for tablespace in list(context.tablespaces.values()):
tablespace.cleanup()
if 'gpstop' in scenario.effective_tags:
context.execute_steps('''
# restart the cluster so that subsequent tests re-use the existing demo cluster
Then the user runs "gpstart -a"
And gpstart should return a return code of 0
''')
# NOTE: gpconfig after_scenario cleanup is in the step `the gpconfig context is setup`
tags_to_skip = [
'gpexpand', 'gpaddmirrors', 'gpstate', 'gpinitstandby', 'gpconfig',
'gpstop', 'gpinitsystem', 'cross_subnet'
]
if set(context.feature.tags).intersection(tags_to_skip):
return
tags_to_cleanup = ['gpmovemirrors', 'gpssh-exkeys']
if set(context.feature.tags).intersection(tags_to_cleanup):
if 'temp_base_dir' in context:
shutil.rmtree(context.temp_base_dir)
tags_to_not_restart_db = ['analyzedb', 'gpssh-exkeys']
if not set(context.feature.tags).intersection(tags_to_not_restart_db):
start_database_if_not_started(context)
home_dir = os.path.expanduser('~')
if not check_user_permissions(home_dir, 'write') and hasattr(context, 'orig_write_permission')\
and context.orig_write_permission:
run_command(context, 'sudo chmod u+w %s' % home_dir)
if os.path.isdir('%s/gpAdminLogs.bk' % home_dir):
shutil.move('%s/gpAdminLogs.bk' % home_dir,
'%s/gpAdminLogs' % home_dir)
if 'gpssh' in context.feature.tags:
run_command(context, 'sudo tc qdisc del dev lo root netem')
# for cleaning up after @given('"{path}" has its permissions set to "{perm}"')
if (hasattr(context, 'path_for_which_to_restore_the_permissions')
and hasattr(context, 'permissions_to_restore_path_to')):
os.chmod(context.path_for_which_to_restore_the_permissions,
context.permissions_to_restore_path_to)
elif hasattr(context, 'path_for_which_to_restore_the_permissions'):
raise Exception('Missing permissions_to_restore_path_to for %s' %
context.path_for_which_to_restore_the_permissions)
elif hasattr(context, 'permissions_to_restore_path_to'):
raise Exception(
'Missing path_for_which_to_restore_the_permissions despite the specified permission %o'
% context.permissions_to_restore_path_to)
def before_feature(context, feature):
# we should be able to run gpexpand without having a cluster initialized
tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpstate', 'gpmovemirrors',
'gpconfig', 'gpssh-exkeys', 'gpstop', 'gpinitsystem', 'cross_subnet']
if set(context.feature.tags).intersection(tags_to_skip):
return
drop_database_if_exists(context, 'testdb')
drop_database_if_exists(context, 'bkdb')
drop_database_if_exists(context, 'fullbkdb')
drop_database_if_exists(context, 'schematestdb')
if 'analyzedb' in feature.tags:
start_database_if_not_started(context)
drop_database_if_exists(context, 'incr_analyze')
create_database(context, 'incr_analyze')
drop_database_if_exists(context, 'incr_analyze_2')
create_database(context, 'incr_analyze_2')
context.conn = dbconn.connect(dbconn.DbURL(dbname='incr_analyze'), unsetSearchPath=False)
context.dbname = 'incr_analyze'
# setting up the tables that will be used
context.execute_steps("""
Given there is a regular "ao" table "t1_ao" with column name list "x,y,z" and column type list "int,text,real" in schema "public"
And there is a regular "heap" table "t2_heap" with column name list "x,y,z" and column type list "int,text,real" in schema "public"
And there is a regular "ao" table "t3_ao" with column name list "a,b,c" and column type list "int,text,real" in schema "public"
And there is a hard coded partition table "sales" with 4 child partitions in schema "public"
""")
if 'gpreload' in feature.tags:
start_database_if_not_started(context)
drop_database_if_exists(context, 'gpreload_db')
create_database(context, 'gpreload_db')
context.conn = dbconn.connect(dbconn.DbURL(dbname='gpreload_db'), unsetSearchPath=False)
context.dbname = 'gpreload_db'
if 'minirepro' in feature.tags:
start_database_if_not_started(context)
minirepro_db = 'minireprodb'
drop_database_if_exists(context, minirepro_db)
create_database(context, minirepro_db)
context.conn = dbconn.connect(dbconn.DbURL(dbname=minirepro_db), unsetSearchPath=False)
context.dbname = minirepro_db
dbconn.execSQL(context.conn, 'create table t1(a integer, b integer)')
dbconn.execSQL(context.conn, 'create table t2(c integer, d integer)')
dbconn.execSQL(context.conn, 'create table t3(e integer, f integer)')
dbconn.execSQL(context.conn, 'create view v1 as select a, b from t1, t3 where t1.a=t3.e')
dbconn.execSQL(context.conn, 'create view v2 as select c, d from t2, t3 where t2.c=t3.f')
dbconn.execSQL(context.conn, 'create view v3 as select a, d from v1, v2 where v1.a=v2.c')
dbconn.execSQL(context.conn, 'insert into t1 values(1, 2)')
dbconn.execSQL(context.conn, 'insert into t2 values(1, 3)')
dbconn.execSQL(context.conn, 'insert into t3 values(1, 4)')
context.conn.commit()
if 'gppkg' in feature.tags:
run_command(context, 'bash demo/gppkg/generate_sample_gppkg.sh buildGppkg')
run_command(context, 'cp -f /tmp/sample-gppkg/sample.gppkg test/behave/mgmt_utils/steps/data/')
def impl(context, cmd):
if hasattr(context, 'netbackup_service_host'):
netbackup_service_host = context.netbackup_service_host
if hasattr(context, 'netbackup_policy'):
netbackup_policy = context.netbackup_policy
if hasattr(context, 'netbackup_schedule'):
netbackup_schedule = context.netbackup_schedule
command_str = cmd + " --netbackup-service-host " + netbackup_service_host + " --netbackup-policy " + netbackup_policy + " --netbackup-schedule " + netbackup_schedule
run_command(context, command_str)
def expand(context):
ensure_temp_directory_is_empty(context, "behave_test_expansion_primary")
ensure_temp_directory_is_empty(context, "behave_test_expansion_mirror")
expansion_command = """gpexpand --input <(echo '
localhost:localhost:25438:/tmp/behave_test_expansion_primary:8:3:p
localhost:localhost:25439:/tmp/behave_test_expansion_mirror:9:3:m
')
"""
# Initialize
run_command(context, expansion_command)
assert_successful_command(context)
# Redistribute tables
run_command(context, expansion_command)
assert_successful_command(context)
def expand(context):
ensure_temp_directory_is_empty(context, "behave_test_expansion_primary")
ensure_temp_directory_is_empty(context, "behave_test_expansion_mirror")
expansion_command = """gpexpand --input <(echo '
localhost|localhost|25438|/tmp/behave_test_expansion_primary|8|3|p
localhost|localhost|25439|/tmp/behave_test_expansion_mirror|9|3|m
')
"""
# Initialize
run_command(context, expansion_command)
assert_successful_command(context)
# Redistribute tables
run_command(context, expansion_command)
assert_successful_command(context)
def after_scenario(context, scenario):
if 'analyzedb' not in context.feature.tags:
master_data_dir = os.environ.get('MASTER_DATA_DIRECTORY')
fn = os.path.join(master_data_dir, 'dirty_hack.txt')
if os.path.exists(fn):
#os.remove(fn)
pass
start_database_if_not_started(context)
home_dir = os.path.expanduser('~')
if not check_user_permissions(home_dir, 'write') and hasattr(context, 'orig_write_permission')\
and context.orig_write_permission:
run_command(context, 'sudo chmod u+w %s' % home_dir)
if os.path.isdir('%s/gpAdminLogs.bk' % home_dir):
shutil.move('%s/gpAdminLogs.bk' % home_dir, '%s/gpAdminLogs' % home_dir)
def after_scenario(context, scenario):
if 'analyzedb' not in context.feature.tags:
master_data_dir = os.environ.get('MASTER_DATA_DIRECTORY')
fn = os.path.join(master_data_dir, 'dirty_hack.txt')
if os.path.exists(fn):
#os.remove(fn)
pass
start_database_if_not_started(context)
home_dir = os.path.expanduser('~')
if not check_user_permissions(home_dir, 'write') and hasattr(context, 'orig_write_permission')\
and context.orig_write_permission:
run_command(context, 'sudo chmod u+w %s' % home_dir)
if os.path.isdir('%s/gpAdminLogs.bk' % home_dir):
shutil.move('%s/gpAdminLogs.bk' % home_dir, '%s/gpAdminLogs' % home_dir)
def step_impl(context):
host, datadir = query_sql("postgres",
"SELECT hostname, datadir FROM gp_segment_configuration WHERE role='m' AND content=0"
).fetchone()
# NOTE that these commands are manually escaped; beware when adding dollar
# signs or double-quotes!
cmd = "ps aux | grep '[p]ostgres .* %s' | awk '{print \$2}' | xargs kill -9" % datadir
cmd = 'ssh %s "%s"' % (host, cmd)
run_command(context, cmd)
# If the kill succeeds, recover the mirror when this test is done.
def recover():
run_command(context, "gprecoverseg -a")
context.add_cleanup(recover)
wait_for_unblocked_transactions(context)
def create_cluster(context):
cmd = """
cd ../gpAux/gpdemo; \
export MASTER_DEMO_PORT={master_port} && \
export DEMO_PORT_BASE={port_base} && \
export NUM_PRIMARY_MIRROR_PAIRS={num_primary_mirror_pairs} && \
export WITH_MIRRORS={with_mirrors} && \A
./demo_cluster.sh -d && ./demo_cluster.sh -c && \
./demo_cluster.sh
""".format(master_port=os.getenv('MASTER_PORT', 15432),
port_base=os.getenv('PORT_BASE', 25432),
num_primary_mirror_pairs=os.getenv('NUM_PRIMARY_MIRROR_PAIRS',
3),
with_mirrors='true')
run_command(context, cmd)
if context.ret_code != 0:
raise Exception('%s' % context.error_message)
def after_scenario(context, scenario):
if 'tablespaces' in context:
for tablespace in context.tablespaces.values():
tablespace.cleanup()
# NOTE: gpconfig after_scenario cleanup is in the step `the gpconfig context is setup`
tags_to_skip = [
'gpexpand', 'gpaddmirrors', 'gpstate', 'gpinitstandby', 'gpconfig'
]
if set(context.feature.tags).intersection(tags_to_skip):
return
if 'gpmovemirrors' in context.feature.tags:
if 'temp_base_dir' in context:
shutil.rmtree(context.temp_base_dir)
if 'analyzedb' not in context.feature.tags:
start_database_if_not_started(context)
home_dir = os.path.expanduser('~')
if not check_user_permissions(home_dir, 'write') and hasattr(context, 'orig_write_permission')\
and context.orig_write_permission:
run_command(context, 'sudo chmod u+w %s' % home_dir)
if os.path.isdir('%s/gpAdminLogs.bk' % home_dir):
shutil.move('%s/gpAdminLogs.bk' % home_dir,
'%s/gpAdminLogs' % home_dir)
if 'gpssh' in context.feature.tags:
run_command(context, 'sudo tc qdisc del dev lo root netem')
# for cleaning up after @given('"{path}" has its permissions set to "{perm}"')
if (hasattr(context, 'path_for_which_to_restore_the_permissions')
and hasattr(context, 'permissions_to_restore_path_to')):
os.chmod(context.path_for_which_to_restore_the_permissions,
context.permissions_to_restore_path_to)
elif hasattr(context, 'path_for_which_to_restore_the_permissions'):
raise Exception('Missing permissions_to_restore_path_to for %s' %
context.path_for_which_to_restore_the_permissions)
elif hasattr(context, 'permissions_to_restore_path_to'):
raise Exception(
'Missing path_for_which_to_restore_the_permissions despite the specified permission %o'
% context.permissions_to_restore_path_to)
def create_cluster(context, with_mirrors=True):
context.initial_cluster_size = 3
context.current_cluster_size = context.initial_cluster_size
cmd = """
cd ../gpAux/gpdemo; \
export MASTER_DEMO_PORT={master_port} && \
export DEMO_PORT_BASE={port_base} && \
export NUM_PRIMARY_MIRROR_PAIRS={num_primary_mirror_pairs} && \
export WITH_MIRRORS={with_mirrors} && \A
./demo_cluster.sh -d && ./demo_cluster.sh -c && \
./demo_cluster.sh
""".format(master_port=os.getenv('MASTER_PORT', 15432),
port_base=os.getenv('PORT_BASE', 25432),
num_primary_mirror_pairs=os.getenv(
'NUM_PRIMARY_MIRROR_PAIRS', context.initial_cluster_size),
with_mirrors=('true' if with_mirrors else 'false'))
run_command(context, cmd)
assert_successful_command(context)
def create_cluster(context, with_mirrors=True):
context.initial_cluster_size = 3
context.current_cluster_size = context.initial_cluster_size
cmd = """
cd ../gpAux/gpdemo; \
export MASTER_DEMO_PORT={master_port} && \
export DEMO_PORT_BASE={port_base} && \
export NUM_PRIMARY_MIRROR_PAIRS={num_primary_mirror_pairs} && \
export WITH_MIRRORS={with_mirrors} && \A
./demo_cluster.sh -d && ./demo_cluster.sh -c && \
./demo_cluster.sh
""".format(master_port=os.getenv('MASTER_PORT', 15432),
port_base=os.getenv('PORT_BASE', 25432),
num_primary_mirror_pairs=os.getenv(
'NUM_PRIMARY_MIRROR_PAIRS', context.initial_cluster_size),
with_mirrors=('true' if with_mirrors else 'false'))
run_command(context, cmd)
assert_successful_command(context)
def expand(context):
ensure_temp_directory_is_empty(context, "behave_test_expansion_primary")
ensure_temp_directory_is_empty(context, "behave_test_expansion_mirror")
run_command(context, "createdb expansion_database")
expansion_command = """gpexpand -D expansion_database --input <(echo '
localhost:localhost:55432:/tmp/behave_test_expansion_primary:8:3:p
localhost:localhost:55433:/tmp/behave_test_expansion_mirror:9:3:m
')
"""
# Initialize
run_command(context, expansion_command)
assert_successful_command(context)
# Redistribute tables
run_command(context, expansion_command)
assert_successful_command(context)
def step_impl(context):
run_command(
context,
"ps aux | grep dbfast_mirror1 | awk '{print $2}' | xargs kill -9")
wait_for_unblocked_transactions(context)
def run_recovery_for_segments(context):
run_command(context, "gprecoverseg -aFv")
assert_successful_command(context)
def ensure_temp_directory_is_empty(context, temp_directory):
run_command(
context,
"rm -rf /tmp/{temp_directory}".format(temp_directory=temp_directory))
def ensure_temp_directory_is_empty(context, temp_directory):
run_command(context, "rm -rf /tmp/{temp_directory}".format(
temp_directory=temp_directory))
def step_impl(context):
run_command(context, "ps aux | grep dbfast_mirror1 | awk '{print $2}' | xargs kill -9")
wait_for_unblocked_transactions(context)
def impl(context, cmd):
if hasattr(context, 'netbackup_service_host'):
netbackup_service_host = context.netbackup_service_host
command_str = cmd + " --netbackup-service-host " + netbackup_service_host
run_command(context, command_str)
def recover():
run_command(context, "gprecoverseg -a")
def step_impl(context):
run_command(context, "ps aux | grep dbfast_mirror1 | awk '{print $2}' | xargs kill -9")
def run_recovery_for_segments(context):
run_command(context, "gprecoverseg -aFv")
assert_successful_command(context)