def inspect_cluster(cluster):
# cleanup inspect dumps
cluster_id = cluster._getClusterId()
node_names = cluster.listLocalNodes()
for node_name in node_names:
dn = '%s/%s/' % (
cluster_id,
node_name,
)
logging.info("deleting %s", dn)
X.removeDirTree(dn)
cfg_fn = cluster._getConfigFileName()
r = call_arakoon("--inspect-cluster", "-config", "%s.cfg" % cfg_fn)
dump_paths = []
for node_name in node_names:
dump_path = './%s/%s/store.db.dump' % (cluster_id, node_name)
dump_paths.append(dump_path)
size = len(node_names)
for i in xrange(size - 1):
for j in xrange(i + 1, size):
fn0 = dump_paths[i]
fn1 = dump_paths[j]
compare_files(fn0, fn1)
logging.info("%s == %s", fn0, fn1)
logging.info("inspect_cluster(%s): ok", cluster_id)
def setUp(self):
logging.debug("setUp")
for node in self._nodes:
home = '%s/%s' % (self._root, node)
X.removeDirTree(home)
X.createDir(home)
self._server.start()
cfg = self._config()
cfg_s = X.cfg2str(cfg)
logging.debug("cfg_s='%s'", cfg_s)
self._etcdClient.set(self._cluster_id, cfg_s)
def setup(c_id, tlog_max_entries):
node_0 = "%s_0" % c_id
c_nodes = [node_0]
c_home = "/".join([Common.data_base_dir, c_id])
node_dir = "/".join([Common.data_base_dir, node_0])
X.removeDirTree(node_dir)
c = Common.setup_n_nodes_base(
c_id,
c_nodes,
False,
c_home,
Common.node_msg_base_port,
Common.node_client_base_port,
extra={'tlog_max_entries': str(tlog_max_entries)})
return c
def teardown_ram_fs(removeDirs):
logging.info("Tearing down")
Common.stop_all()
# Copy over log files
if not removeDirs:
destination = X.tmpDir + Common.data_base_dir[1:]
if os.path.isdir(destination):
X.removeDirTree(destination)
X.createDir(destination)
X.copyDirTree(system_tests_common.data_base_dir, destination)
for i in range(len(Common.node_names)):
Common.destroy_ram_fs(i)
cluster = Common._getCluster(CONFIG.cluster_id)
cluster.tearDown()
def start(self):
logging.debug("ETCD:start on %s", self._home)
X.removeDirTree(self._home)
X.createDir(self._home)
address = "http://%s:%i" % (self._host, self._port)
real_cmd = [
'nohup', 'etcd',
'-advertise-client-urls=%s' % address,
'-listen-client-urls=%s' % address, '-data-dir', self._home,
'>> %s/stdout' % self._home, '2>&1 &'
]
real_cmd_s = ' '.join(real_cmd)
fn = '%s/start.sh' % self._home
with open(fn, 'w') as f:
print >> f, real_cmd_s
os.chmod(fn, 0755)
os.system(fn)
time.sleep(5)
def mount_ram_fs ( node_index ) :
(mount_target,log_dir,tlf_dir,head_dir) = Common.build_node_dir_names( Common.node_names[node_index] )
if X.fileExists( mount_target ) :
Common.stopOne( Common.node_names[node_index] )
cmd = ["umount", mount_target]
X.subprocess.check_call ( cmd )
X.removeDirTree( mount_target )
X.createDir ( mount_target )
if not os.path.isdir( mount_target ) :
raise Exception( "%s is not valid mount target as it is not a directory")
cmd = ["sudo", "mount", "-t", "tmpfs","-o","size=20m","tmpfs", mount_target]
(rc,out,err) = X.run(cmd)
if rc:
logging.info("out=%s", out)
logging.info("err = %s", err)
raise Exception("Mounting failed (rc=%s)" % rc)
def remove_dirs():
X.removeDirTree( data_base_dir )
def wipe_dir(d):
X.removeDirTree(d)
X.createDir(d)
def make_monkey_run():
global monkey_dies
C.data_base_dir = '/'.join([X.tmpDir, "/arakoon_monkey/"])
t = threading.Thread(target=memory_monitor)
t.start()
C.stop_all()
cluster = C._getCluster(C.cluster_id)
cluster.tearDown()
#setup_3_nodes_forced_master()
C.setup_3_nodes(C.data_base_dir)
monkey_dir = get_monkey_work_dir()
if X.fileExists(monkey_dir):
X.removeDirTree(monkey_dir)
X.createDir(monkey_dir)
iteration = 0
C.start_all()
time.sleep(1.0)
while (True):
iteration += 1
logging.info("Preparing iteration %d" % iteration)
thr_list = list()
try:
(disruption, f_list) = generate_work_list(iteration)
logging.info("Starting iteration %d" % iteration)
thr_list = C.create_and_start_thread_list(f_list)
disruption()
for thr in thr_list:
thr.join(60.0 * 60.0)
if thr.isAlive():
logging.fatal(
"Thread did not complete in a timely fashion.")
monkey_dies = True
if not monkey_dies:
wait_for_it()
if not monkey_dies:
health_check()
except SystemExit, ex:
if str(ex) == "0":
sys.exit(0)
else:
logging.fatal("Caught SystemExit => %s: %s" %
(ex.__class__.__name__, ex))
tb = traceback.format_exc()
logging.fatal(tb)
for thr in thr_list:
thr.join()
monkey_dies = True
except Exception, ex:
logging.fatal("Caught fatal exception => %s: %s" %
(ex.__class__.__name__, ex))
tb = traceback.format_exc()
logging.fatal(tb)
for thr in thr_list:
thr.join()
monkey_dies = True
