def copy_logs(argv):
""" Copies logs back from a Spark cluster.
This script copies the JSON event log and JSON continuous monitor back from a Spark
driver and Spark executor, respectively, to the local machine. Returns a two-item
tuple with the name of the event log and the continuous monitor log.
"""
parser = OptionParser()
parser.add_option(
"-e", "--executor-host", help="Executor from which to copy continuous monitor")
parser.add_option(
"-d", "--driver-host", help="Hostname of driver")
parser.add_option(
"-f", "--filename-prefix", help="Filename prefix to use for files copied back")
parser.add_option(
"-u", "--username", default="root", help="Username to user when logging in")
parser.add_option(
"-i", "--identity-file", help="Identity file to use when logging in")
(opts, args) = parser.parse_args()
if not opts.executor_host:
parser.error("--executor-host must be specified")
if not opts.driver_host:
parser.error("--driver-host must be specified")
if not opts.identity_file:
parser.error("--identity-file must be specified")
if not opts.filename_prefix:
parser.error("--filename-prefix must be specified")
# Copy the event log from the driver back to the local machine.
ret = utils.ssh_get_stdout(
opts.driver_host,
opts.identity_file,
opts.username,
"ls -t /tmp/spark-events | head -n 1")
print ret
event_log_relative_filename = ret.strip("\n").strip("\r")
event_log_filename = "/tmp/spark-events/%s" % event_log_relative_filename
local_event_log_file = "%s_event_log" % opts.filename_prefix
print ("Copying event log from file %s on host %s back to %s" %
(event_log_filename, opts.driver_host, local_event_log_file))
utils.scp_from(
opts.driver_host,
opts.identity_file,
opts.username,
event_log_filename,
local_event_log_file)
# Copy the continuous monitor from the driver back to the local machine.
local_continuous_monitor_file = utils.copy_latest_continuous_monitor(
opts.executor_host,
opts.identity_file,
opts.filename_prefix,
opts.username)
return (local_event_log_file, local_continuous_monitor_file)
def copy_logs(argv):
""" Copies logs back from a Spark cluster.
This script copies the JSON event log and JSON continuous monitor back from a Spark
driver and Spark executor, respectively, to the local machine. Returns a two-item
tuple with the name of the event log and the continuous monitor log.
"""
parser = OptionParser()
parser.add_option("-e",
"--executor-host",
help="Executor from which to copy continuous monitor")
parser.add_option("-d", "--driver-host", help="Hostname of driver")
parser.add_option("-f",
"--filename-prefix",
help="Filename prefix to use for files copied back")
parser.add_option("-u",
"--username",
default="root",
help="Username to user when logging in")
parser.add_option("-i",
"--identity-file",
help="Identity file to use when logging in")
(opts, args) = parser.parse_args()
if not opts.executor_host:
parser.error("--executor-host must be specified")
if not opts.driver_host:
parser.error("--driver-host must be specified")
if not opts.identity_file:
parser.error("--identity-file must be specified")
if not opts.filename_prefix:
parser.error("--filename-prefix must be specified")
# Copy the event log from the driver back to the local machine.
ret = utils.ssh_get_stdout(opts.driver_host, opts.identity_file,
opts.username,
"ls -t /tmp/spark-events | head -n 1")
print ret
event_log_relative_filename = ret.strip("\n").strip("\r")
event_log_filename = "/tmp/spark-events/{}".format(
event_log_relative_filename)
local_event_log_file = "{}_event_log".format(opts.filename_prefix)
print "Copying event log from file {} on host {} back to {}".format(
event_log_filename, opts.driver_host, local_event_log_file)
utils.scp_from(opts.driver_host, opts.identity_file, opts.username,
event_log_filename, local_event_log_file)
# Copy the continuous monitor from the driver back to the local machine.
local_continuous_monitor_file = utils.copy_latest_continuous_monitor(
opts.executor_host, opts.identity_file, opts.filename_prefix,
opts.username)
return (local_event_log_file, local_continuous_monitor_file)
"""
This file allows a user to copy the most recent continuous monitor from
a remote executor.
"""
import sys
import utils
if __name__ == "__main__":
if len(sys.argv) < 4:
print "Usage: python copy_continuous_monitor.py hostname identity_file output_prefix"
sys.exit(1)
utils.copy_latest_continuous_monitor(sys.argv[1], sys.argv[2], sys.argv[3], "root")