def main():
# connect to master
with open("conf/master-port", "r") as f:
master = Master(int(f.read()))
# connect to slaves
with open("conf/slave-ports", "r") as f:
ports: List[int] = [int(line) for line in f.readlines()]
slaves: List[Server] = []
for port in ports:
print("")
slave = Server(port)
slaves.append(slave)
master.stop_monitor()
for slave in slaves:
slave.stop_monitor()
def main(job: str):
# connect to master
with open("conf/master-port", "r") as f:
master = Master(int(f.read()))
# connect to slaves
with open("conf/slave-ports", "r") as f:
ports: List[int] = [int(line) for line in f.readlines()]
slaves: List[Server] = []
for port in ports:
print("")
slave = Server(port)
slaves.append(slave)
# start monitor on each slave
for slave in slaves:
slave.start_monitor(interval=0.01)
# send command to master to start the job
master.get_connection().run("source /etc/profile && cd $SPARK_HOME && " + job)
# job is done, stop monitors, the slaves will write data to their own disks
for slave in slaves:
slave.stop_monitor()
# collect data to "./monitor_data/"
current_time = str(datetime.now())[:-7]
folder_name = input("Please input the folder name (default: {current_time}):\n".format(current_time=current_time))
folder_name = folder_name if folder_name else current_time
folder_path = "./monitor_data/" + folder_name
mkdir(folder_path)
i: int = 1
for slave in slaves:
slave.get_connection().get(slave.get_log_path(), "{folder_path}/slave{i}.csv".format(folder_path=folder_path,
i=i))
i += 1
def main(job: str):
# connect to master
with open("conf/master-port", "r") as f:
master = Master(int(f.read()))
# connect to slaves
with open("conf/slave-ports", "r") as f:
ports: List[int] = [int(line) for line in f.readlines()]
slaves: List[Server] = []
for port in ports:
print("")
slave = Server(port)
slaves.append(slave)
master.start_monitor(interval=0.01)
# start monitor on each slave
for slave in slaves:
slave.start_monitor(interval=0.01)
# send command to master to start the job
try:
master.get_connection().run(
"source /etc/profile && cd $SPARK_HOME && " + job)
except UnexpectedExit as exp:
pass
# job is done, stop monitors, the slaves will write data to their own disks
master.stop_monitor()
for slave in slaves:
slave.stop_monitor()
# collect data to "./monitor_data/"
current_time = str(datetime.now())[:-7]
folder_name = input(
"Please input the folder name (default: {current_time}):\n".format(
current_time=current_time))
folder_name = folder_name if folder_name else current_time
folder_path = "./monitor_data/" + folder_name
mkdir(folder_path)
i: int = 1
for slave in slaves:
file_path = "{folder_path}/slave{i}.csv".format(
folder_path=folder_path, i=i)
slave.get_connection().get(slave.get_log_path(), file_path)
array = np.genfromtxt(file_path, delimiter=',')
mean_cpu_usage = np.mean(array[:, 1])
max_cpu_usage = np.max(array[:, 1])
print("slave{}: {}, {}".format(i, mean_cpu_usage, max_cpu_usage))
if max_cpu_usage <= 90:
remove(file_path)
i += 1
master.get_connection().get(
master.get_log_path(),
"{folder_path}/master.csv".format(folder_path=folder_path, i=i))
print("The data files have been put into monitor_data/" + folder_name)
def main():
# os.system("./sbin/stop-port-forward.sh")
# os.system("./sbin/start-port-forward.sh")
with open("conf/master-port", "r") as f:
master = Master(int(f.read()))
master.upload_profile()
master.install_python3()
master.install_java()
# master.install_scala()
master.install_spark()
master.install_monitor(forth=False)
master.set_slaves(
) # set "$SPARK_HOME/conf/slaves" based on local config
master.set_ssh_config(
) # enable master to log in slaves with private key
with open("conf/slave-ports", "r") as f:
ports: List[int] = [int(line) for line in f.readlines()]
slaves: List[Server] = []
for port in ports:
print("")
slave = Server(port)
slaves.append(slave)
slave.install_python3()
slave.upload_profile()
slave.install_java()
slave.install_scala()
slave.install_spark()
slave.install_python3()
slave.install_monitor(forth=False)