def run_clients(cluster, clients, export, mtype, start_vip, end_vip, locking_type):
# Will explicitly pass public IP of the controller to clients since we won't rely on DNS existence
controller = socket.gethostbyname(socket.gethostname())
dynamo_cmd_line = "{} --controller {} --server {} --export {} --mtype {} --start_vip {} --end_vip {} " \
"--locking {}".format(config.DYNAMO_BIN_PATH, controller, cluster, export, mtype, start_vip,
end_vip, locking_type)
for client in clients:
ShellUtils.run_shell_remote_command_background(client, dynamo_cmd_line)
wait_clients_to_start(clients)
def wait_clients_to_start(clients):
cmd_line = "ps aux | grep dynamo | grep -v grep | wc -l"
while True:
total_processes = 0
for client in clients:
outp = ShellUtils.run_shell_remote_command(client, cmd_line)
logger.info(f"SSH command response with {int(outp)} processes on client {client}")
num_processes_per_client = int(outp)
total_processes += num_processes_per_client
if total_processes >= config.MAX_WORKERS_PER_CLIENT * len(clients):
break
time.sleep(1)
logger.info(f"All {len(clients)} clients started. {total_processes // len(clients)} processes per client")
def cleanup(clients=None):
logger.info("Cleaning up on exit....")
if clients:
for client in clients:
logger.info(f"{client}: Killing workers")
ShellUtils.run_shell_remote_command_no_exception(client, 'pkill -9 -f dynamo')
logger.info(f"{client}: Unmounting")
ShellUtils.run_shell_remote_command_no_exception(client, 'sudo umount -fl /mnt/{}'.format('VFS*'))
logger.info(f"{client}: Removing mountpoint folder/s")
ShellUtils.run_shell_remote_command_no_exception(client, 'sudo rm -fr /mnt/{}'.format('VFS*'))
def deploy_clients(clients, access):
"""
Args:
access: dict
clients: list
Returns: None
"""
with open(os.path.expanduser(os.path.join('~', '.ssh', 'id_rsa.pub')), 'r') as f:
rsa_pub_key = f.read()
for client in clients:
logger.info(f"Setting SSH connection to {client}")
ssh_utils.set_key_policy(rsa_pub_key, client, logger, access['user'],
access['password'])
logger.info(f"Deploying to {client}")
ShellUtils.run_shell_remote_command_no_exception(client, 'mkdir -p {}'.format(config.DYNAMO_PATH))
ShellUtils.run_shell_command('rsync',
'-avz {} {}:{}'.format('client', client, config.DYNAMO_PATH))
ShellUtils.run_shell_command('rsync',
'-avz {} {}:{}'.format('config', client, config.DYNAMO_PATH))
ShellUtils.run_shell_command('rsync',
'-avz {} {}:{}'.format('logger', client, config.DYNAMO_PATH))
ShellUtils.run_shell_command('rsync',
'-avz {} {}:{}'.format('utils', client, config.DYNAMO_PATH))
ShellUtils.run_shell_remote_command_no_exception(client, 'chmod +x {}'.format(config.DYNAMO_BIN_PATH))
def deploy_rpyc_server(cluster_node):
ShellUtils.run_shell_remote_command(cluster_node,
"yes | easy_install rpyc")
def stop_rpyc_server(cluster_node):
ShellUtils.run_shell_remote_command(cluster_node,
"pkill -f rpyc_classic.py")
def start_rpyc_server(cluster_node):
ShellUtils.run_shell_remote_command_background(cluster_node,
"rpyc_classic.py &")
def mount(server, export, mount_point, mtype):
try:
ShellUtils.run_shell_command("mount", "-o nfsvers={0} {1}:/{2} {3}".format(mtype, server, export, mount_point))
except OSError:
return False
return True
def umount(mount_point):
try:
ShellUtils.run_shell_command("umount", "-fl {0}".format(mount_point))
except OSError:
return False
return True