def do(): logging.info( "Going to drop all networking (save ssh with %s) for %d seconds..." % (bastion_host, seconds)) # TODO check connectivity, or atleast DNS resolution, for bastion_host chains = iptables.create_gremlin_network_failure(bastion_host) logging.info("Created iptables chains: %s" % repr(chains)) iptables.add_user_chain_to_input_chain(chains[0]) iptables.add_user_chain_to_output_chain(chains[1]) logging.info("Gremlin chains %s installed, sleeping %d seconds" % (repr(chains), seconds)) time.sleep(seconds) if use_flush: logging.info("Using flush to remove gremlin chains") iptables.flush() else: logging.info("Removing gremlin chains %s" % repr(chains)) iptables.remove_user_chain_from_input_chain(chains[0]) iptables.remove_user_chain_from_output_chain(chains[1]) iptables.delete_user_chain(chains[0]) iptables.delete_user_chain(chains[1]) logging.info("Removed gremlin chains %s" % repr(chains)) if restart_daemons: logging.info("Restarting daemons: %s", repr(restart_daemons)) for daemon in restart_daemons: procutils.start_daemon(daemon)
def do(): logging.info("Going to drop all networking (save ssh with %s) for %d seconds..." % (bastion_host, seconds)) # TODO check connectivity, or atleast DNS resolution, for bastion_host chains = iptables.create_gremlin_network_failure(bastion_host) logging.info("Created iptables chains: %s" % repr(chains)) iptables.add_user_chain_to_input_chain(chains[0]) iptables.add_user_chain_to_output_chain(chains[1]) logging.info("Gremlin chains %s installed, sleeping %d seconds" % (repr(chains), seconds)) time.sleep(seconds) if use_flush: logging.info("Using flush to remove gremlin chains") iptables.flush() else: logging.info("Removing gremlin chains %s" % repr(chains)) iptables.remove_user_chain_from_input_chain(chains[0]) iptables.remove_user_chain_from_output_chain(chains[1]) iptables.delete_user_chain(chains[0]) iptables.delete_user_chain(chains[1]) logging.info("Removed gremlin chains %s" % repr(chains)) if restart_daemons: logging.info("Restarting daemons: %s", repr(restart_daemons)) for daemon in restart_daemons: procutils.start_daemon(daemon)
def do(): # First kill for daemon in daemons: pid = procutils.find_jvm(daemon) if pid: logging.info("Killing %s pid %d with signal %d" % (daemon, pid, signal)) os.kill(pid, signal) else: logging.info("There was no %s running!" % daemon) logging.info("Sleeping for %d seconds" % restart_after) time.sleep(restart_after) for daemon in daemons: logging.info("Restarting %s" % daemon); procutils.start_daemon(daemon)
def do(): # First kill for daemon in daemons: pid = procutils.find_jvm(daemon) if pid: logging.info("Killing %s pid %d with signal %d" % (daemon, pid, signal)) os.kill(pid, signal) else: logging.info("There was no %s running!" % daemon) logging.info("Sleeping for %d seconds" % restart_after) time.sleep(restart_after) for daemon in daemons: logging.info("Restarting %s" % daemon) procutils.start_daemon(daemon)
import sys import random from gremlins import faults, metafaults, triggers, procutils # TODO use an environment variable for paths.. no more absolutes # TODO make shutdown proper.. right now, bg processes are continuing ot run procutils.START_COMMANDS = { 'QuorumPeerMain': ["/Users/criccomi/Downloads/apache-zookeeper-ef198fc/bin/zkServer.sh", "start"], 'LoadBalancerEchoServer': ["/Users/criccomi/svn/pegasus/trunk/d2/scripts/lb-echo-server.sh", "2345", "prpc", "cluster-1", "service-1", "service-2"], 'LoadBalancerEchoClient': ["/Users/criccomi/svn/pegasus/trunk/d2/scripts/lb-echo-client.sh", "service-1", "service-2"], } # start zoo keeper procutils.start_daemon('QuorumPeerMain'); # put cluster and service properties procutils.run(["/Users/criccomi/svn/pegasus/trunk/d2/scripts/lb-tool.sh", "--put_cluster", "cluster-1", "--schemes", "prpc,http", "--banned", "http://www.google.com", "--store", "zk://localhost:2181/echo/lb/clusters"]) procutils.run(["/Users/criccomi/svn/pegasus/trunk/d2/scripts/lb-tool.sh", "--put_service", "service-1", "--cluster", "cluster-1", "--path", "/service-1", "--balancer", "degrader", "--store", "zk://localhost:2181/echo/lb/services"]) procutils.run(["/Users/criccomi/svn/pegasus/trunk/d2/scripts/lb-tool.sh", "--put_service", "service-2", "--cluster", "cluster-1", "--path", "/service-2", "--balancer", "degrader", "--store", "zk://localhost:2181/echo/lb/services"]) # start server and client procutils.start_daemon('LoadBalancerEchoServer'); procutils.start_daemon('LoadBalancerEchoClient'); # declare faults kill_short_zk = faults.kill_daemons(["QuorumPeerMain"], signal.SIGKILL, 5) kill_short_server = faults.kill_daemons(["LoadBalancerEchoServer"], signal.SIGKILL, 5) kill_short_client = faults.kill_daemons(["LoadBalancerEchoClient"], signal.SIGKILL, 5)