def copy_app_to_host(cls, app_location, keyname, is_verbose):
"""Copies the given application to a machine running the Login service within
an AppScale deployment.
Args:
app_location: The location on the local filesystem where the application
can be found.
keyname: The name of the SSH keypair that uniquely identifies this
AppScale deployment.
is_verbose: A bool that indicates if we should print the commands we exec
to copy the app to the remote host to stdout.
Returns:
A str corresponding to the location on the remote filesystem where the
application was copied to.
"""
AppScaleLogger.log("Creating remote directory to copy app into")
app_id = AppEngineHelper.get_app_id_from_app_config(app_location)
remote_app_dir = "/var/apps/{0}/app".format(app_id)
cls.ssh(LocalState.get_login_host(keyname), keyname,
'mkdir -p {0}'.format(remote_app_dir), is_verbose)
AppScaleLogger.log("Tarring application")
rand = str(uuid.uuid4()).replace('-', '')[:8]
local_tarred_app = "/tmp/appscale-app-{0}-{1}.tar.gz".format(app_id, rand)
LocalState.shell("cd {0} && tar -czf {1} *".format(app_location,
local_tarred_app), is_verbose)
AppScaleLogger.log("Copying over application")
remote_app_tar = "{0}/{1}.tar.gz".format(remote_app_dir, app_id)
cls.scp(LocalState.get_login_host(keyname), keyname, local_tarred_app,
remote_app_tar, is_verbose)
os.remove(local_tarred_app)
return remote_app_tar
def rsync_files(cls, host, keyname, local_appscale_dir, is_verbose):
"""Copies over an AppScale source directory from this machine to the
specified host.
Args:
host: A str representing a host that should be accessible from this
machine.
keyname: A str representing the name of the SSH keypair that can log into
the specified machine.
local_appscale_dir: A str representing the path on the local filesystem
where the AppScale source to copy over can be found.
is_verbose: A bool that indicates if we should print the rsync commands
we exec to stdout.
Raises:
BadConfigurationException: If local_appscale_dir does not exist locally,
or if any of the standard AppScale module folders do not exist.
"""
ssh_key = LocalState.get_key_path_from_name(keyname)
local_path = os.path.expanduser(local_appscale_dir)
if not os.path.exists(local_path):
raise BadConfigurationException("The location you specified to copy " \
"from, {0}, doesn't exist.".format(local_path))
LocalState.shell("rsync -e 'ssh -i {0} {1}' -arv "
"--exclude='AppDB/logs/*' " \
"--exclude='AppDB/cassandra/cassandra/*' " \
"{2}/* root@{3}:/root/appscale/".format(ssh_key, cls.SSH_OPTIONS,
local_path, host), is_verbose)
def copy_app_to_host(cls, app_location, keyname, is_verbose):
"""Copies the given application to a machine running the Login service
within an AppScale deployment.
Args:
app_location: The location on the local filesystem where the application
can be found.
keyname: The name of the SSH keypair that uniquely identifies this
AppScale deployment.
is_verbose: A bool that indicates if we should print the commands we exec
to copy the app to the remote host to stdout.
Returns:
A str corresponding to the location on the remote filesystem where the
application was copied to.
"""
app_id = AppEngineHelper.get_app_id_from_app_config(app_location)
AppScaleLogger.log("Tarring application")
rand = str(uuid.uuid4()).replace('-', '')[:8]
local_tarred_app = "{0}/appscale-app-{1}-{2}.tar.gz".format(tempfile.gettempdir(),
app_id, rand)
LocalState.shell("cd '{0}' && COPYFILE_DISABLE=1 tar -czhf {1} --exclude='*.pyc' *".format(
app_location, local_tarred_app), is_verbose)
AppScaleLogger.log("Copying over application")
remote_app_tar = "{0}/{1}.tar.gz".format(cls.REMOTE_APP_DIR, app_id)
cls.scp(LocalState.get_login_host(keyname), keyname, local_tarred_app,
remote_app_tar, is_verbose)
os.remove(local_tarred_app)
return remote_app_tar
def rsync_files(cls, host, keyname, local_appscale_dir, is_verbose):
"""Copies over an AppScale source directory from this machine to the
specified host.
Args:
host: A str representing a host that should be accessible from this
machine.
keyname: A str representing the name of the SSH keypair that can log into
the specified machine.
local_appscale_dir: A str representing the path on the local filesystem
where the AppScale source to copy over can be found.
is_verbose: A bool that indicates if we should print the rsync commands
we exec to stdout.
Raises:
BadConfigurationException: If local_appscale_dir does not exist locally,
or if any of the standard AppScale module folders do not exist.
"""
ssh_key = LocalState.get_key_path_from_name(keyname)
appscale_dirs = ["lib", "AppController", "AppManager", "AppServer",
"AppLoadBalancer", "AppMonitoring", "Neptune", "InfrastructureManager"]
for dir_name in appscale_dirs:
local_path = os.path.expanduser(local_appscale_dir) + os.sep + dir_name
if not os.path.exists(local_path):
raise BadConfigurationException("The location you specified to copy " +
"from, {0}, doesn't contain a {1} folder.".format(local_appscale_dir,
local_path))
LocalState.shell("rsync -e 'ssh -i {0} {1}' -arv {2}/* root@{3}:/root/appscale/{4}" \
.format(ssh_key, cls.SSH_OPTIONS, local_path, host, dir_name), is_verbose)
# Rsync AppDB separately, as it has a lot of paths we may need to exclude
# (e.g., built database binaries).
local_app_db = os.path.expanduser(local_appscale_dir) + os.sep + "AppDB/*"
LocalState.shell("rsync -e 'ssh -i {0} {1}' -arv --exclude='logs/*' --exclude='hadoop-*' --exclude='hbase/hbase-*' --exclude='voldemort/voldemort/*' --exclude='cassandra/cassandra/*' {2} root@{3}:/root/appscale/AppDB".format(ssh_key, cls.SSH_OPTIONS, local_app_db, host), is_verbose)
def copy_app_to_host(cls, app_location, keyname, is_verbose):
"""Copies the given application to a machine running the Login service
within an AppScale deployment.
Args:
app_location: The location on the local filesystem where the application
can be found.
keyname: The name of the SSH keypair that uniquely identifies this
AppScale deployment.
is_verbose: A bool that indicates if we should print the commands we exec
to copy the app to the remote host to stdout.
Returns:
A str corresponding to the location on the remote filesystem where the
application was copied to.
"""
app_id = AppEngineHelper.get_app_id_from_app_config(app_location)
AppScaleLogger.log("Tarring application")
rand = str(uuid.uuid4()).replace('-', '')[:8]
local_tarred_app = "{0}/appscale-app-{1}-{2}.tar.gz".format(
tempfile.gettempdir(), app_id, rand)
LocalState.shell(
"cd '{0}' && COPYFILE_DISABLE=1 tar -czhf {1} --exclude='*.pyc' *".
format(app_location, local_tarred_app), is_verbose)
AppScaleLogger.log("Copying over application")
remote_app_tar = "{0}/{1}.tar.gz".format(cls.REMOTE_APP_DIR, app_id)
cls.scp(LocalState.get_login_host(keyname), keyname, local_tarred_app,
remote_app_tar, is_verbose)
os.remove(local_tarred_app)
return remote_app_tar
def rsync_files(cls, host, keyname, local_appscale_dir, is_verbose):
"""Copies over an AppScale source directory from this machine to the
specified host.
Args:
host: A str representing a host that should be accessible from this
machine.
keyname: A str representing the name of the SSH keypair that can log into
the specified machine.
local_appscale_dir: A str representing the path on the local filesystem
where the AppScale source to copy over can be found.
is_verbose: A bool that indicates if we should print the rsync commands
we exec to stdout.
Raises:
BadConfigurationException: If local_appscale_dir does not exist locally,
or if any of the standard AppScale module folders do not exist.
"""
ssh_key = LocalState.get_key_path_from_name(keyname)
local_path = os.path.expanduser(local_appscale_dir)
if not os.path.exists(local_path):
raise BadConfigurationException("The location you specified to copy " \
"from, {0}, doesn't exist.".format(local_path))
LocalState.shell("rsync -e 'ssh -i {0} {1}' -arv "
"--exclude='AppDB/logs/*' " \
"--exclude='AppDB/cassandra/cassandra/*' " \
"{2}/* root@{3}:/root/appscale/".format(ssh_key, cls.SSH_OPTIONS,
local_path, host), is_verbose)
def add_keypair(cls, options):
"""Sets up passwordless SSH login to the machines used in a virtualized
cluster deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Raises:
AppScaleException: If any of the machines named in the ips_layout are
not running, or do not have the SSH daemon running.
"""
LocalState.require_ssh_commands(options.auto, options.verbose)
LocalState.make_appscale_directory()
path = LocalState.LOCAL_APPSCALE_PATH + options.keyname
if options.add_to_existing:
public_key = path + ".pub"
private_key = path
else:
public_key, private_key = LocalState.generate_rsa_key(options.keyname,
options.verbose)
if options.auto:
if 'root_password' in options:
AppScaleLogger.log("Using the provided root password to log into " + \
"your VMs.")
password = options.root_password
else:
AppScaleLogger.log("Please enter the password for the root user on" + \
" your VMs:")
password = getpass.getpass()
node_layout = NodeLayout(options)
if not node_layout.is_valid():
raise BadConfigurationException("There were problems with your " + \
"placement strategy: " + str(node_layout.errors()))
all_ips = [node.public_ip for node in node_layout.nodes]
for ip in all_ips:
# first, make sure ssh is actually running on the host machine
if not RemoteHelper.is_port_open(ip, RemoteHelper.SSH_PORT,
options.verbose):
raise AppScaleException("SSH does not appear to be running at {0}. " \
"Is the machine at {0} up and running? Make sure your IPs are " \
"correct!".format(ip))
# next, set up passwordless ssh
AppScaleLogger.log("Executing ssh-copy-id for host: {0}".format(ip))
if options.auto:
LocalState.shell("{0} root@{1} {2} {3}".format(cls.EXPECT_SCRIPT, ip,
private_key, password), options.verbose)
else:
LocalState.shell("ssh-copy-id -i {0} root@{1}".format(private_key, ip),
options.verbose)
AppScaleLogger.success("Generated a new SSH key for this deployment " + \
"at {0}".format(private_key))
def add_keypair(cls, options):
"""Sets up passwordless SSH login to the machines used in a virtualized
cluster deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
LocalState.require_ssh_commands(options.auto, options.verbose)
LocalState.make_appscale_directory()
path = LocalState.LOCAL_APPSCALE_PATH + options.keyname
if options.add_to_existing:
public_key = path + ".pub"
private_key = path
else:
public_key, private_key = LocalState.generate_rsa_key(
options.keyname, options.verbose)
if options.auto:
if 'root_password' in options:
AppScaleLogger.log("Using the provided root password to log into " + \
"your VMs.")
password = options.root_password
else:
AppScaleLogger.log("Please enter the password for the root user on" + \
" your VMs:")
password = getpass.getpass()
node_layout = NodeLayout(options)
if not node_layout.is_valid():
raise BadConfigurationException("There were problems with your " + \
"placement strategy: " + str(node_layout.errors()))
all_ips = [node.public_ip for node in node_layout.nodes]
for ip in all_ips:
# first, set up passwordless ssh
AppScaleLogger.log(
"Executing ssh-copy-id for host: {0}".format(ip))
if options.auto:
LocalState.shell(
"{0} root@{1} {2} {3}".format(cls.EXPECT_SCRIPT, ip,
private_key, password),
options.verbose)
else:
LocalState.shell(
"ssh-copy-id -i {0} root@{1}".format(private_key, ip),
options.verbose)
# next, copy over the ssh keypair we generate
RemoteHelper.scp(ip, options.keyname, public_key,
'/root/.ssh/id_rsa.pub', options.verbose)
RemoteHelper.scp(ip, options.keyname, private_key,
'/root/.ssh/id_rsa', options.verbose)
AppScaleLogger.success("Generated a new SSH key for this deployment " + \
"at {0}".format(private_key))
def add_keypair(cls, options):
"""Sets up passwordless SSH login to the machines used in a virtualized
cluster deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
Raises:
AppScaleException: If any of the machines named in the ips_layout are
not running, or do not have the SSH daemon running.
"""
LocalState.require_ssh_commands(options.auto, options.verbose)
LocalState.make_appscale_directory()
path = LocalState.LOCAL_APPSCALE_PATH + options.keyname
if options.add_to_existing:
public_key = path + ".pub"
private_key = path
else:
public_key, private_key = LocalState.generate_rsa_key(options.keyname,
options.verbose)
if options.auto:
if 'root_password' in options:
AppScaleLogger.log("Using the provided root password to log into " + \
"your VMs.")
password = options.root_password
else:
AppScaleLogger.log("Please enter the password for the root user on" + \
" your VMs:")
password = getpass.getpass()
node_layout = NodeLayout(options)
if not node_layout.is_valid():
raise BadConfigurationException("There were problems with your " + \
"placement strategy: " + str(node_layout.errors()))
all_ips = [node.public_ip for node in node_layout.nodes]
for ip in all_ips:
# first, make sure ssh is actually running on the host machine
if not RemoteHelper.is_port_open(ip, RemoteHelper.SSH_PORT,
options.verbose):
raise AppScaleException("SSH does not appear to be running at {0}. " \
"Is the machine at {0} up and running? Make sure your IPs are " \
"correct!".format(ip))
# next, set up passwordless ssh
AppScaleLogger.log("Executing ssh-copy-id for host: {0}".format(ip))
if options.auto:
LocalState.shell("{0} root@{1} {2} {3}".format(cls.EXPECT_SCRIPT, ip,
private_key, password), options.verbose)
else:
LocalState.shell("ssh-copy-id -i {0} root@{1}".format(private_key, ip),
options.verbose)
AppScaleLogger.success("Generated a new SSH key for this deployment " + \
"at {0}".format(private_key))
def add_keypair(cls, options):
"""Sets up passwordless SSH login to the machines used in a virtualized
cluster deployment.
Args:
options: A Namespace that has fields for each parameter that can be
passed in via the command-line interface.
"""
LocalState.require_ssh_commands(options.auto, options.verbose)
LocalState.make_appscale_directory()
path = LocalState.LOCAL_APPSCALE_PATH + options.keyname
if options.add_to_existing:
public_key = path + ".pub"
private_key = path
else:
public_key, private_key = LocalState.generate_rsa_key(options.keyname,
options.verbose)
if options.auto:
if 'root_password' in options:
AppScaleLogger.log("Using the provided root password to log into " + \
"your VMs.")
password = options.root_password
else:
AppScaleLogger.log("Please enter the password for the root user on" + \
" your VMs:")
password = getpass.getpass()
node_layout = NodeLayout(options)
if not node_layout.is_valid():
raise BadConfigurationException("There were problems with your " + \
"placement strategy: " + str(node_layout.errors()))
all_ips = [node.public_ip for node in node_layout.nodes]
for ip in all_ips:
# first, set up passwordless ssh
AppScaleLogger.log("Executing ssh-copy-id for host: {0}".format(ip))
if options.auto:
LocalState.shell("{0} root@{1} {2} {3}".format(cls.EXPECT_SCRIPT, ip,
private_key, password), options.verbose)
else:
LocalState.shell("ssh-copy-id -i {0} root@{1}".format(private_key, ip),
options.verbose)
# next, copy over the ssh keypair we generate
RemoteHelper.scp(ip, options.keyname, public_key, '/root/.ssh/id_rsa.pub',
options.verbose)
RemoteHelper.scp(ip, options.keyname, private_key, '/root/.ssh/id_rsa',
options.verbose)
AppScaleLogger.success("Generated a new SSH key for this deployment " + \
"at {0}".format(private_key))
def scp_remote_to_local(cls,
host,
keyname,
source,
dest,
is_verbose,
user='******'):
"""Securely copies a file from a remote machine to this machine.
Args:
host: A str representing the machine that we should log into.
keyname: A str representing the name of the SSH keypair to log in with.
source: A str representing the path on the remote machine where the
file should be copied from.
dest: A str representing the path on the local machine where the file
should be copied to.
is_verbose: A bool that indicates if we should print the scp command to
stdout.
user: A str representing the user to log in as.
Returns:
A str representing the standard output of the secure copy and a str
representing the standard error of the secure copy.
"""
ssh_key = LocalState.get_key_path_from_name(keyname)
return LocalState.shell(
"scp -r -i {0} {1} {2}@{3}:{4} {5}".format(ssh_key,
cls.SSH_OPTIONS, user,
host, source, dest),
is_verbose)
def ssh(cls,
host,
keyname,
command,
is_verbose,
user='******',
num_retries=LocalState.DEFAULT_NUM_RETRIES):
"""Logs into the named host and executes the given command.
Args:
host: A str representing the machine that we should log into.
keyname: A str representing the name of the SSH keypair to log in with.
command: A str representing what to execute on the remote host.
is_verbose: A bool indicating if we should print the ssh command to
stdout.
user: A str representing the user to log in as.
Returns:
A str representing the standard output of the remote command and a str
representing the standard error of the remote command.
"""
ssh_key = LocalState.get_key_path_from_name(keyname)
return LocalState.shell(
"ssh -F /dev/null -i {0} {1} {2}@{3} bash".format(
ssh_key, cls.SSH_OPTIONS, user, host),
is_verbose,
num_retries,
stdin=command)
def ssh(cls, host, keyname, command, is_verbose, user='******'):
"""Logs into the named host and executes the given command.
Args:
host: A str representing the machine that we should log into.
keyname: A str representing the name of the SSH keypair to log in with.
command: A str representing what to execute on the remote host.
is_verbose: A bool indicating if we should print the ssh command to
stdout.
user: A str representing the user to log in as.
Returns:
A str representing the standard output of the remote command and a str
representing the standard error of the remote command.
"""
ssh_key = LocalState.get_key_path_from_name(keyname)
return LocalState.shell("ssh -i {0} {1} {2}@{3} '{4}'".format(ssh_key,
cls.SSH_OPTIONS, user, host, command), is_verbose)
def rsync_files(cls, host, keyname, local_appscale_dir, is_verbose):
"""Copies over an AppScale source directory from this machine to the
specified host.
Args:
host: A str representing a host that should be accessible from this
machine.
keyname: A str representing the name of the SSH keypair that can log into
the specified machine.
local_appscale_dir: A str representing the path on the local filesystem
where the AppScale source to copy over can be found.
is_verbose: A bool that indicates if we should print the rsync commands
we exec to stdout.
Raises:
BadConfigurationException: If local_appscale_dir does not exist locally,
or if any of the standard AppScale module folders do not exist.
"""
ssh_key = LocalState.get_key_path_from_name(keyname)
appscale_dirs = [
"lib", "AppController", "AppManager", "AppServer",
"AppServer_Java", "AppDashboard", "InfrastructureManager",
"AppTaskQueue", "XMPPReceiver"
]
for dir_name in appscale_dirs:
local_path = os.path.expanduser(
local_appscale_dir) + os.sep + dir_name
if not os.path.exists(local_path):
raise BadConfigurationException("The location you specified to copy " \
"from, {0}, doesn't contain a {1} folder.".format(local_appscale_dir,
local_path))
LocalState.shell("rsync -e 'ssh -i {0} {1}' -arv {2}/* "\
"root@{3}:/root/appscale/{4}".format(ssh_key, cls.SSH_OPTIONS,
local_path, host, dir_name), is_verbose)
# Rsync AppDB separately, as it has a lot of paths we may need to exclude
# (e.g., built database binaries).
local_app_db = os.path.expanduser(
local_appscale_dir) + os.sep + "AppDB/*"
LocalState.shell("rsync -e 'ssh -i {0} {1}' -arv --exclude='logs/*' " \
"--exclude='hadoop-*' --exclude='hbase/hbase-*' " \
"--exclude='cassandra/cassandra/*' {2} root@{3}:/root/appscale/AppDB" \
.format(ssh_key, cls.SSH_OPTIONS, local_app_db, host), is_verbose)
# And rsync the firewall configuration file separately, as it's not a
# directory (which the above all are).
local_firewall = os.path.expanduser(local_appscale_dir) + os.sep + \
"firewall.conf"
LocalState.shell("rsync -e 'ssh -i {0} {1}' -arv {2} root@{3}:" \
"/root/appscale/firewall.conf".format(ssh_key, cls.SSH_OPTIONS,
local_firewall, host), is_verbose)
def scp(cls, host, keyname, source, dest, is_verbose, user='******'):
"""Securely copies a file from this machine to the named machine.
Args:
host: A str representing the machine that we should log into.
keyname: A str representing the name of the SSH keypair to log in with.
source: A str representing the path on the local machine where the
file should be copied from.
dest: A str representing the path on the remote machine where the file
should be copied to.
is_verbose: A bool that indicates if we should print the scp command to
stdout.
user: A str representing the user to log in as.
Returns:
A str representing the standard output of the secure copy and a str
representing the standard error of the secure copy.
"""
ssh_key = LocalState.get_key_path_from_name(keyname)
return LocalState.shell("scp -i {0} {1} {2} {3}@{4}:{5}".format(ssh_key,
cls.SSH_OPTIONS, source, user, host, dest), is_verbose)
