def test_marathon_zk_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
common.block_iptable_rules_for_seconds(original_leader, 2181, sleep_seconds=30)
common.assert_marathon_leadership_changed(original_leader)
def test_marathon_zk_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
common.block_iptable_rules_for_seconds(original_leader, 2181, sleep_seconds=30)
common.assert_marathon_leadership_changed(original_leader)
def test_marathon_master_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
# blocking outbound connection to mesos master
# Marathon has a Mesos heartbeat interval of 15 seconds. If 5 are missed it
# disconnects. Thus we should wait more than 75 seconds.
common.block_iptable_rules_for_seconds(original_leader, 5050, sleep_seconds=100,
block_input=False, block_output=True)
common.assert_marathon_leadership_changed(original_leader)
def test_marathon_master_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
# blocking outbound connection to mesos master
# Marathon has a Mesos heartbeat interval of 15 seconds. If 5 are missed it
# disconnects. Thus we should wait more than 75 seconds.
common.block_iptable_rules_for_seconds(original_leader, 5050, sleep_seconds=100,
block_input=False, block_output=True)
common.assert_marathon_leadership_changed(original_leader)
def test_marathon_zk_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
common.block_iptable_rules_for_seconds(original_leader,
2181,
sleep_seconds=30)
common.marathon_leadership_changed(original_leader)
# Make sure marathon is available
shakedown.wait_for_service_endpoint(marathon_service_name,
timedelta(minutes=5).total_seconds())
def test_marathon_master_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
# blocking outbound connection to mesos master
with common.iptable_rules(original_leader):
common.block_port(original_leader, 5050, direction='OUTPUT')
# Wait for a leader change before restoring iptables rules
common.marathon_leadership_changed(original_leader)
# Make sure marathon is available
shakedown.wait_for_service_endpoint(
marathon_service_name,
timedelta(minutes=5).total_seconds())
def test_marathon_master_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
# blocking outbound connection to mesos master
with shakedown.iptable_rules(original_leader):
block_port(original_leader, 5050, direction='OUTPUT')
# time of the master block
time.sleep(timedelta(minutes=1.5).total_seconds())
common.wait_for_marathon_up()
current_leader = shakedown.marathon_leader_ip()
assert original_leader != current_leader
def test_marathon_master_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
# blocking outbound connection to mesos master
with shakedown.iptable_rules(original_leader):
block_port(original_leader, 5050, direction='OUTPUT')
# time of the master block
time.sleep(timedelta(minutes=1.5).total_seconds())
common.wait_for_marathon_up()
current_leader = shakedown.marathon_leader_ip()
assert original_leader != current_leader
def test_marathon_zk_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
# blocking zk on marathon leader (not master leader)
with shakedown.iptable_rules(original_leader):
block_port(original_leader, 2181, direction='INPUT')
block_port(original_leader, 2181, direction='OUTPUT')
# time of the zk block
time.sleep(5)
common.wait_for_marathon_up()
current_leader = shakedown.marathon_leader_ip()
assert original_leader != current_leader
def test_marathon_zk_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
# blocking zk on marathon leader (not master leader)
with shakedown.iptable_rules(original_leader):
common.block_port(original_leader, 2181, direction='INPUT')
common.block_port(original_leader, 2181, direction='OUTPUT')
# time of the zk block, the duration must be greater than all the default ZK timeout values in Marathon
time.sleep(20)
shakedown.wait_for_service_endpoint(marathon_service_name, timedelta(minutes=5).total_seconds())
current_leader = shakedown.marathon_leader_ip()
assert original_leader != current_leader, "A new Marathon leader has not been elected"
def test_marathon_zk_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
# blocking zk on marathon leader (not master leader)
with shakedown.iptable_rules(original_leader):
block_port(original_leader, 2181, direction='INPUT')
block_port(original_leader, 2181, direction='OUTPUT')
# time of the zk block
time.sleep(5)
common.wait_for_marathon_up()
current_leader = shakedown.marathon_leader_ip()
assert original_leader != current_leader
def test_marathon_master_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
# blocking outbound connection to mesos master
common.block_iptable_rules_for_seconds(original_leader,
5050,
sleep_seconds=60,
block_input=False,
block_output=True)
common.marathon_leadership_changed(original_leader)
# Make sure marathon is available
shakedown.wait_for_service_endpoint(marathon_service_name,
timedelta(minutes=5).total_seconds())
def test_marathon_zk_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
# blocking zk on marathon leader (not master leader)
with shakedown.iptable_rules(original_leader):
common.block_port(original_leader, 2181, direction='INPUT')
common.block_port(original_leader, 2181, direction='OUTPUT')
# time of the zk block, the duration must be greater than all the default ZK timeout values in Marathon
time.sleep(20)
shakedown.wait_for_service_endpoint(marathon_service_name, timedelta(minutes=5).total_seconds())
current_leader = shakedown.marathon_leader_ip()
assert original_leader != current_leader, "A new Marathon leader has not been elected"
