def setUp(self):
logging.debug("setUp")
for node in self._nodes:
home = '%s/%s' % (self._root, node)
X.removeDirTree(home)
X.createDir(home)
self._server.start()
cfg = self._config()
cfg_s = X.cfg2str(cfg)
logging.debug("cfg_s='%s'", cfg_s)
self._etcdClient.set(self._cluster_id, cfg_s)
def teardown_ram_fs(removeDirs):
logging.info("Tearing down")
Common.stop_all()
# Copy over log files
if not removeDirs:
destination = X.tmpDir + Common.data_base_dir[1:]
if os.path.isdir(destination):
X.removeDirTree(destination)
X.createDir(destination)
X.copyDirTree(system_tests_common.data_base_dir, destination)
for i in range(len(Common.node_names)):
Common.destroy_ram_fs(i)
cluster = Common._getCluster(CONFIG.cluster_id)
cluster.tearDown()
def start(self):
logging.debug("ETCD:start on %s", self._home)
X.removeDirTree(self._home)
X.createDir(self._home)
address = "http://%s:%i" % (self._host, self._port)
real_cmd = [
'nohup', 'etcd',
'-advertise-client-urls=%s' % address,
'-listen-client-urls=%s' % address, '-data-dir', self._home,
'>> %s/stdout' % self._home, '2>&1 &'
]
real_cmd_s = ' '.join(real_cmd)
fn = '%s/start.sh' % self._home
with open(fn, 'w') as f:
print >> f, real_cmd_s
os.chmod(fn, 0755)
os.system(fn)
time.sleep(5)
def mount_ram_fs ( node_index ) :
(mount_target,log_dir,tlf_dir,head_dir) = Common.build_node_dir_names( Common.node_names[node_index] )
if X.fileExists( mount_target ) :
Common.stopOne( Common.node_names[node_index] )
cmd = ["umount", mount_target]
X.subprocess.check_call ( cmd )
X.removeDirTree( mount_target )
X.createDir ( mount_target )
if not os.path.isdir( mount_target ) :
raise Exception( "%s is not valid mount target as it is not a directory")
cmd = ["sudo", "mount", "-t", "tmpfs","-o","size=20m","tmpfs", mount_target]
(rc,out,err) = X.run(cmd)
if rc:
logging.info("out=%s", out)
logging.info("err = %s", err)
raise Exception("Mounting failed (rc=%s)" % rc)
def setup_3_nodes_ram_fs(home_dir):
cluster = Common._getCluster(Common.cluster_id)
cluster.remove()
cluster = Common._getCluster(Common.cluster_id)
logging.info("Creating data base dir %s" % home_dir)
X.createDir(home_dir)
try:
for i in range(len(Common.node_names)):
mount_ram_fs(i)
nodeName = Common.node_names[i]
(db_dir, log_dir, tlf_dir,
head_dir) = Common.build_node_dir_names(Common.node_names[i])
cluster.addNode(nodeName,
Common.node_ips[i],
clientPort=Common.node_client_base_port + i,
messagingPort=Common.node_msg_base_port + i,
logDir=log_dir,
tlfDir=tlf_dir,
home=db_dir,
headDir=head_dir)
cluster.addLocalNode(nodeName)
cluster.createDirs(nodeName)
except Exception as ex:
teardown_ram_fs(True)
(a, b, c) = sys.exc_info()
raise a, b, c
logging.info("Changing log level to debug for all nodes")
cluster.setLogLevel("debug")
cluster.setMasterLease(int(CONFIG.lease_duration))
logging.info("Creating client config")
Common.regenerateClientConfig(Common.cluster_id)
Common.start_all()
def test_start_stop_wrapper():
cluster = C._getCluster()
fn = "%s/my_wrapper.sh" % X.tmpDir
X.createDir(X.tmpDir)
try:
with open(fn, 'w') as f:
f.write('#!/bin/bash -xue\n')
f.write("logger wrapper called with '$@'\n")
f.write('$@\n')
subprocess.call(['chmod', '+x', fn])
nn = "wrapper"
cluster.addNode(nn, "127.0.0.1", 8000, wrapper=fn)
cluster.addLocalNode(nn)
cluster.createDirs(nn)
C.assert_running_nodes(0)
cluster.start()
time.sleep(1)
C.assert_running_nodes(1)
cluster.stop()
C.assert_running_nodes(0)
finally:
cluster.remove()
def setup_n_nodes_base(c_id, node_names, force_master,
base_dir, base_msg_port, base_client_port,
extra = None, witness_nodes = False, useIPV6=False,
slowCollapser = False):
X.subprocess.check_call("sudo /sbin/iptables -F".split(' ') )
cluster = _getCluster( c_id )
cluster.tearDown()
cluster = _getCluster(c_id)
logging.info( "Creating data base dir %s" % base_dir )
X.createDir ( base_dir )
n = len(node_names)
ip = "127.0.0.1"
if useIPV6:
ip = "::1"
for i in range (n) :
is_witness = witness_nodes & (i % 2 != 0)
nodeName = node_names[ i ]
(db_dir,log_dir,tlf_dir,head_dir) = build_node_dir_names( nodeName )
if slowCollapser and (i % 2 == 1):
collapseSlowdown = 3
else:
collapseSlowdown = None
cluster.addNode(name=nodeName,
ip = ip,
clientPort = base_client_port+i,
messagingPort = base_msg_port+i,
logDir = log_dir,
home = db_dir,
tlfDir = tlf_dir,
headDir = head_dir,
isWitness = is_witness,
collapseSlowdown = collapseSlowdown)
cluster.addLocalNode(nodeName)
cluster.createDirs(nodeName)
cluster.disableFsync()
if force_master:
logging.info( "Forcing master to %s", node_names[0] )
cluster.forceMaster(node_names[0] )
else :
logging.info( "Using master election" )
cluster.forceMaster(None )
config = cluster._getConfigFile()
for i in range (n):
nodeName = node_names[ i ]
config.set(nodeName, '__tainted_fsync_tlog_dir', 'false')
#
#
#
if extra :
logging.info("EXTRA!")
for k,v in extra.items():
logging.info("%s -> %s", k, v)
config.set("global", k, v)
fn = cluster._getConfigFileName()
X.writeConfig(config, fn)
logging.info( "Creating client config" )
regenerateClientConfig( c_id )
logging.info( "Changing log level to debug for all nodes" )
cluster.setLogLevel("debug")
lease = int(lease_duration)
logging.info( "Setting lease expiration to %d" % lease)
cluster.setMasterLease( lease )
def wipe_dir(d):
X.removeDirTree(d)
X.createDir(d)
def make_monkey_run():
global monkey_dies
C.data_base_dir = '/'.join([X.tmpDir, "/arakoon_monkey/"])
t = threading.Thread(target=memory_monitor)
t.start()
C.stop_all()
cluster = C._getCluster(C.cluster_id)
cluster.tearDown()
#setup_3_nodes_forced_master()
C.setup_3_nodes(C.data_base_dir)
monkey_dir = get_monkey_work_dir()
if X.fileExists(monkey_dir):
X.removeDirTree(monkey_dir)
X.createDir(monkey_dir)
iteration = 0
C.start_all()
time.sleep(1.0)
while (True):
iteration += 1
logging.info("Preparing iteration %d" % iteration)
thr_list = list()
try:
(disruption, f_list) = generate_work_list(iteration)
logging.info("Starting iteration %d" % iteration)
thr_list = C.create_and_start_thread_list(f_list)
disruption()
for thr in thr_list:
thr.join(60.0 * 60.0)
if thr.isAlive():
logging.fatal(
"Thread did not complete in a timely fashion.")
monkey_dies = True
if not monkey_dies:
wait_for_it()
if not monkey_dies:
health_check()
except SystemExit, ex:
if str(ex) == "0":
sys.exit(0)
else:
logging.fatal("Caught SystemExit => %s: %s" %
(ex.__class__.__name__, ex))
tb = traceback.format_exc()
logging.fatal(tb)
for thr in thr_list:
thr.join()
monkey_dies = True
except Exception, ex:
logging.fatal("Caught fatal exception => %s: %s" %
(ex.__class__.__name__, ex))
tb = traceback.format_exc()
logging.fatal(tb)
for thr in thr_list:
thr.join()
monkey_dies = True