def do():
logging.info(
"Going to drop all networking (save ssh with %s) for %d seconds..."
% (bastion_host, seconds))
# TODO check connectivity, or atleast DNS resolution, for bastion_host
chains = iptables.create_gremlin_network_failure(bastion_host)
logging.info("Created iptables chains: %s" % repr(chains))
iptables.add_user_chain_to_input_chain(chains[0])
iptables.add_user_chain_to_output_chain(chains[1])
logging.info("Gremlin chains %s installed, sleeping %d seconds" %
(repr(chains), seconds))
time.sleep(seconds)
if use_flush:
logging.info("Using flush to remove gremlin chains")
iptables.flush()
else:
logging.info("Removing gremlin chains %s" % repr(chains))
iptables.remove_user_chain_from_input_chain(chains[0])
iptables.remove_user_chain_from_output_chain(chains[1])
iptables.delete_user_chain(chains[0])
iptables.delete_user_chain(chains[1])
logging.info("Removed gremlin chains %s" % repr(chains))
if restart_daemons:
logging.info("Restarting daemons: %s", repr(restart_daemons))
for daemon in restart_daemons:
procutils.start_daemon(daemon)
def do():
logging.info("Going to drop all networking (save ssh with %s) for %d seconds..." %
(bastion_host, seconds))
# TODO check connectivity, or atleast DNS resolution, for bastion_host
chains = iptables.create_gremlin_network_failure(bastion_host)
logging.info("Created iptables chains: %s" % repr(chains))
iptables.add_user_chain_to_input_chain(chains[0])
iptables.add_user_chain_to_output_chain(chains[1])
logging.info("Gremlin chains %s installed, sleeping %d seconds" % (repr(chains), seconds))
time.sleep(seconds)
if use_flush:
logging.info("Using flush to remove gremlin chains")
iptables.flush()
else:
logging.info("Removing gremlin chains %s" % repr(chains))
iptables.remove_user_chain_from_input_chain(chains[0])
iptables.remove_user_chain_from_output_chain(chains[1])
iptables.delete_user_chain(chains[0])
iptables.delete_user_chain(chains[1])
logging.info("Removed gremlin chains %s" % repr(chains))
if restart_daemons:
logging.info("Restarting daemons: %s", repr(restart_daemons))
for daemon in restart_daemons:
procutils.start_daemon(daemon)
def do():
# First kill
for daemon in daemons:
pid = procutils.find_jvm(daemon)
if pid:
logging.info("Killing %s pid %d with signal %d" % (daemon, pid, signal))
os.kill(pid, signal)
else:
logging.info("There was no %s running!" % daemon)
logging.info("Sleeping for %d seconds" % restart_after)
time.sleep(restart_after)
for daemon in daemons:
logging.info("Restarting %s" % daemon);
procutils.start_daemon(daemon)
def do():
# First kill
for daemon in daemons:
pid = procutils.find_jvm(daemon)
if pid:
logging.info("Killing %s pid %d with signal %d" %
(daemon, pid, signal))
os.kill(pid, signal)
else:
logging.info("There was no %s running!" % daemon)
logging.info("Sleeping for %d seconds" % restart_after)
time.sleep(restart_after)
for daemon in daemons:
logging.info("Restarting %s" % daemon)
procutils.start_daemon(daemon)
import sys
import random
from gremlins import faults, metafaults, triggers, procutils
# TODO use an environment variable for paths.. no more absolutes
# TODO make shutdown proper.. right now, bg processes are continuing ot run
procutils.START_COMMANDS = {
'QuorumPeerMain': ["/Users/criccomi/Downloads/apache-zookeeper-ef198fc/bin/zkServer.sh", "start"],
'LoadBalancerEchoServer': ["/Users/criccomi/svn/pegasus/trunk/d2/scripts/lb-echo-server.sh", "2345", "prpc", "cluster-1", "service-1", "service-2"],
'LoadBalancerEchoClient': ["/Users/criccomi/svn/pegasus/trunk/d2/scripts/lb-echo-client.sh", "service-1", "service-2"],
}
# start zoo keeper
procutils.start_daemon('QuorumPeerMain');
# put cluster and service properties
procutils.run(["/Users/criccomi/svn/pegasus/trunk/d2/scripts/lb-tool.sh", "--put_cluster", "cluster-1", "--schemes", "prpc,http", "--banned", "http://www.google.com", "--store", "zk://localhost:2181/echo/lb/clusters"])
procutils.run(["/Users/criccomi/svn/pegasus/trunk/d2/scripts/lb-tool.sh", "--put_service", "service-1", "--cluster", "cluster-1", "--path", "/service-1", "--balancer", "degrader", "--store", "zk://localhost:2181/echo/lb/services"])
procutils.run(["/Users/criccomi/svn/pegasus/trunk/d2/scripts/lb-tool.sh", "--put_service", "service-2", "--cluster", "cluster-1", "--path", "/service-2", "--balancer", "degrader", "--store", "zk://localhost:2181/echo/lb/services"])
# start server and client
procutils.start_daemon('LoadBalancerEchoServer');
procutils.start_daemon('LoadBalancerEchoClient');
# declare faults
kill_short_zk = faults.kill_daemons(["QuorumPeerMain"], signal.SIGKILL, 5)
kill_short_server = faults.kill_daemons(["LoadBalancerEchoServer"], signal.SIGKILL, 5)
kill_short_client = faults.kill_daemons(["LoadBalancerEchoClient"], signal.SIGKILL, 5)
