#!/usr/bin/env python3
"""This program checks if the Hillview service is running on all machines
specified in the configuration file."""
# pylint: disable=invalid-name
from argparse import ArgumentParser
from hillviewCommon import ClusterConfiguration, RemoteHost, get_config, get_logger
logger = get_logger("status")
def check_webserver(config):
"""Checks if the Hillview web server is running"""
assert isinstance(config, ClusterConfiguration)
rh = config.get_webserver()
message = "Checking hillview status on " + str(rh)
logger.info(message)
rh.run_remote_shell_command(config.service_folder +
"/hillview-webserver-manager.sh status")
def check_worker(config, rh):
"""Checks if the Hillview service is running on a remote machine"""
assert isinstance(config, ClusterConfiguration)
assert isinstance(rh, RemoteHost)
message = "Checking hillview status on " + str(rh.host)
logger.info(message)
rh.run_remote_shell_command(config.service_folder +
"/hillview-worker-manager.sh status")
#!/usr/bin/env python
# We attempted to make this program work with both python2 and python3
"""This script takes a cluster configuration and a file pattern.
It downloads the files that match from all machines in the cluster."""
# pylint: disable=invalid-name
from argparse import ArgumentParser
import os.path
import os
import errno
from hillviewCommon import execute_command, ClusterConfiguration, get_config, get_logger
logger = get_logger("download-data")
def copy_from_remote_host(rh, pattern):
"""Copy files matching the pattern from the remote machine"""
try:
os.mkdir(rh.host)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
u = rh.user
if u is None:
u = ""
else:
u = rh.user + "@"
command = "scp -r " + u + rh.host + ":" + pattern + " " + rh.host + " || true"
message = "Copying the files from the remote machine " + str(rh.host)
logger.info(message)
execute_command(command)
#!/usr/bin/env python3
"""This python program deploys the files needed by the Hillview service
on the machines specified in the configuration file."""
# pylint: disable=invalid-name
from argparse import ArgumentParser
import tempfile
import os.path
from hillviewCommon import ClusterConfiguration, get_config, get_logger
logger = get_logger("deploy")
def generate_script(config, rh, template):
"""Generates a shell script based on a template inserting configuration variables"""
logger.info("Generating script for host " + rh.host + " from " + template)
variables = ""
variables += "SERVICE_DIRECTORY=" + config.service_folder + "\n"
variables += "HEAPSIZE=\"" + rh.heapsize + "\"\n"
variables += "USER="******"\n"
variables += "WORKER_PORT=" + str(config.worker_port) + "\n"
variables += "AGGREGATOR_PORT=" + str(config.aggregator_port) + "\n"
variables += "CLEANUP=" + str(
1 if config.cleanup_on_install() else 0) + "\n"
variables += "TOMCAT=" + config.tomcat + "\n"
lines = list(open(template))
filename = template.replace("-template", "")
lines = [variables if "REPLACE_WITH_VARIABLES" in x else x for x in lines]
with open(filename, "w") as f:
f.write("# Automatically generated from " + template)
for l in lines:
#!/usr/bin/env python3
"""This python starts the Hillview service on the machines
specified in the configuration file."""
# pylint: disable=invalid-name
from argparse import ArgumentParser
from hillviewCommon import RemoteHost, RemoteAggregator, ClusterConfiguration, get_config, get_logger
logger = get_logger("start")
def start_webserver(config):
"""Starts the Hillview web server"""
assert isinstance(config, ClusterConfiguration)
rh = config.get_webserver()
message = "Starting web server " + str(rh)
logger.info(message)
rh.run_remote_shell_command(config.service_folder +
"/hillview-webserver-manager.sh start")
def start_worker(config, rh):
"""Starts the Hillview worker on a remote machine"""
assert isinstance(rh, RemoteHost)
assert isinstance(config, ClusterConfiguration)
message = "Starting worker " + str(rh)
logger.info(message)
rh.run_remote_shell_command(config.service_folder +
"/hillview-worker-manager.sh start")
#!/usr/bin/env python3
"""This script deletes a specific folder on all the machines in a Hillview cluster."""
# pylint: disable=invalid-name
import os.path
from argparse import ArgumentParser
from hillviewCommon import ClusterConfiguration, get_config, get_logger
logger = get_logger("delete-data")
def delete_remote_folder(rh, folder):
"""Deletes folder on the remote host"""
rh.run_remote_shell_command("if [ -d " + folder + " ]; then " + "rm -rf " +
folder + "; echo Deleted " +
"; else echo \"Directory " + str(folder) +
" doesn't exist.\"; fi")
def delete_folder(config, folder):
"""Delete a folder on all remote hosts"""
assert isinstance(config, ClusterConfiguration)
message = "Deleting " + folder + " from all hosts"
logger.info(message)
config.run_on_all_workers(lambda rh: delete_remote_folder(rh, folder))
def main():
"""Main function"""
parser = ArgumentParser()
parser.add_argument("config", help="json cluster configuration file")
#!/usr/bin/env python
# We attempted to make this program work with both python2 and python3
# -*-python-*-
"""This script runs a command on all worker hosts of a Hillview cluster."""
# pylint: disable=invalid-name
from argparse import ArgumentParser, REMAINDER
from hillviewCommon import ClusterConfiguration, get_config, get_logger
logger = get_logger("run-on-all")
def execute_command_on_all(config, command):
"""Executes command on all workers"""
assert isinstance(config, ClusterConfiguration)
message = "Executing `" + str(command) + "' on " + str(
len(config.get_workers())) + " hosts"
logger.info(message)
lam = lambda rh: rh.run_remote_shell_command(command)
config.run_on_all_workers(lam)
def main():
"""Main function"""
parser = ArgumentParser()
parser.add_argument("config", help="json cluster configuration file")
parser.add_argument("command", help="command to run", nargs=REMAINDER)
args = parser.parse_args()
config = get_config(parser, args)
command = " ".join(args.command)
execute_command_on_all(config, command)
#!/usr/bin/env python
# We attempted to make this program work with both python2 and python3
"""This script takes a set of files and a cluster configuration describing a set of machines.
It uploads the files to the given machines in round-robin fashion.
The script can also be given an optional schema file.
This file will be uploaded to all machines.
The list of machines is provided in a Hillview configuration file."""
# pylint: disable=invalid-name
from argparse import ArgumentParser, REMAINDER
import os.path
from hillviewCommon import ClusterConfiguration, get_config, get_logger
logger = get_logger("upload-data")
created_folders = set()
def create_remote_folder(remoteHost, folder):
"""Creates a folder on a remote machine"""
shortcut = "" + remoteHost.host + ":" + folder
if shortcut in created_folders:
return
remoteHost.create_remote_folder(folder)
created_folders.add(shortcut)
def copy_file_to_remote_host(rh, source, folder, copyOption):
"""Copy files in the specified folder to the remote machine"""
create_remote_folder(rh, folder)
rh.copy_file_to_remote(source, folder, copyOption)
def copy_everywhere(config, file, folder, copyOption):
#!/usr/bin/env python3
"""This Python program stops the Hillview service on the machines specified in the
configuration file."""
# pylint: disable=invalid-name
from argparse import ArgumentParser
from hillviewCommon import ClusterConfiguration, get_config, get_logger
logger = get_logger("stop")
def stop_webserver(config):
"""Stops the Hillview web server"""
assert isinstance(config, ClusterConfiguration)
rh = config.get_webserver()
message = "Stopping web server on " + str(rh)
logger.info(message)
rh.run_remote_shell_command(config.service_folder +
"/hillview-webserver-manager.sh stop")
def stop_worker(config, rh):
"""Stops a Hillview worker service on a remote machine"""
rh.run_remote_shell_command(config.service_folder +
"/hillview-worker-manager.sh stop")
def stop_aggregator(config, rh):
"""Stops a Hillview aggregator service on a remote machine"""
rh.run_remote_shell_command(config.service_folder +
"/hillview-aggregator-manager.sh stop")
