def get_logs(self, unique_id, logs, directory, pattern='^$'):
""" Copies logs from the remote host that the process is running on to the provided directory
:Parameter unique_id the unique_id of the process in question
:Parameter logs a list of logs given by absolute path from the remote host
:Parameter directory the local directory to store the copied logs
:Parameter pattern a pattern to apply to files to restrict the set of logs copied
"""
hostname = self.processes[unique_id].hostname
install_path = self.processes[unique_id].install_path
if hostname is not None:
with get_sftp_client(hostname,
username=runtime.get_username(),
password=runtime.get_password()) as ftp:
for f in logs:
try:
mode = ftp.stat(f).st_mode
except IOError, e:
if e.errno == errno.ENOENT:
logger.error("Log file " + f +
" does not exist on " + hostname)
pass
else:
copy_dir(ftp, f, directory, unique_id)
if install_path is not None:
copy_dir(ftp, install_path, directory, unique_id, pattern)
def fetch_logs_from_host(hostname, install_path, prefix, logs, directory,
pattern):
""" Static method Copies logs from specified host on the specified install path
:Parameter hostname the remote host from where we need to fetch the logs
:Parameter install_path path where the app is installed
:Parameter prefix prefix used to copy logs. Generall the unique_id of process
:Parameter logs a list of logs given by absolute path from the remote host
:Parameter directory the local directory to store the copied logs
:Parameter pattern a pattern to apply to files to restrict the set of logs copied
"""
if hostname is not None:
with get_sftp_client(hostname,
username=runtime.get_username(),
password=runtime.get_password()) as ftp:
for f in logs:
try:
mode = ftp.stat(f).st_mode
except IOError, e:
if e.errno == errno.ENOENT:
logger.error("Log file " + f +
" does not exist on " + hostname)
pass
else:
copy_dir(ftp, f, directory, prefix)
if install_path is not None:
copy_dir(ftp, install_path, directory, prefix, pattern)
def install(self, package_id, configs={}):
"""
Installs a package (tarball, or zip) on to a list of remote hosts by
SFTP'ing the package to the remote install_path.
param: package_id -- A unique ID used to identify an installed YARN package.
param: configs -- Map of config key/values pairs. Valid keys include:
yarn_site_template: Jinja2 yarn-site.xml template local path.
yarn_driver_configs: Key/value pairs to be injected into the yarn-site.xml template.
yarn_nm_hosts: A list of YARN NM hosts to install the package onto.
install_path: An absolute path where the package will be installed.
executable: A local path pointing to the location of the package that should be installed on remote hosts.
"""
configs = self._get_merged_configs(configs)
self._validate_configs(configs, [
'yarn_site_template', 'yarn_driver_configs', 'yarn_nm_hosts',
'install_path', 'executable'
])
# Get configs.
nm_hosts = configs.get('yarn_nm_hosts')
install_path = configs.get('install_path')
executable = configs.get('executable')
# FTP and decompress job tarball to all NMs.
exec_file_location = os.path.join(
install_path, self._get_package_tgz_name(package_id))
exec_file_install_path = os.path.join(install_path, package_id)
for host in nm_hosts:
logger.info('Deploying {0} on host: {1}'.format(package_id, host))
with get_ssh_client(host, self.username, self.password) as ssh:
better_exec_command(
ssh, "mkdir -p {0}".format(install_path),
"Failed to create path: {0}".format(install_path))
with get_sftp_client(host, self.username, self.password) as ftp:
def progress(transferred_bytes, total_bytes_to_transfer):
logger.debug("{0} of {1} bytes transferred.".format(
transferred_bytes, total_bytes_to_transfer))
ftp.put(executable, exec_file_location, callback=progress)
# Extract archive locally so we can use run-job.sh.
executable_tgz = tarfile.open(executable, 'r:gz')
executable_tgz.extractall(package_id)
# Generate yarn-site.xml install it in package's local 'config' directory.
yarn_site_dir = self._get_yarn_conf_dir(package_id)
yarn_site_path = os.path.join(yarn_site_dir, 'yarn-site.xml')
logger.info("Installing yarn-site.xml to {0}".format(yarn_site_path))
if not os.path.exists(yarn_site_dir):
os.makedirs(yarn_site_dir)
templates.render_config(configs.get('yarn_site_template'),
yarn_site_path,
configs.get('yarn_driver_configs'))
def install(self, package_id, configs={}):
"""
Installs a package (tarball, or zip) on to a list of remote hosts by
SFTP'ing the package to the remote install_path.
param: package_id -- A unique ID used to identify an installed YARN package.
param: configs -- Map of config key/values pairs. Valid keys include:
yarn_site_template: Jinja2 yarn-site.xml template local path.
yarn_driver_configs: Key/value pairs to be injected into the yarn-site.xml template.
yarn_nm_hosts: A list of YARN NM hosts to install the package onto.
install_path: An absolute path where the package will be installed.
executable: A local path pointing to the location of the package that should be installed on remote hosts.
"""
configs = self._get_merged_configs(configs)
self._validate_configs(configs, ['yarn_site_template', 'yarn_driver_configs', 'yarn_nm_hosts', 'install_path', 'executable'])
# Get configs.
nm_hosts = configs.get('yarn_nm_hosts')
install_path = configs.get('install_path')
executable = configs.get('executable')
# FTP and decompress job tarball to all NMs.
exec_file_location = os.path.join(install_path, self._get_package_tgz_name(package_id))
exec_file_install_path = os.path.join(install_path, package_id)
for host in nm_hosts:
logger.info('Deploying {0} on host: {1}'.format(package_id, host))
with get_ssh_client(host, self.username, self.password) as ssh:
better_exec_command(ssh, "mkdir -p {0}".format(install_path), "Failed to create path: {0}".format(install_path))
with get_sftp_client(host, self.username, self.password) as ftp:
def progress(transferred_bytes, total_bytes_to_transfer):
logger.debug("{0} of {1} bytes transferred.".format(transferred_bytes, total_bytes_to_transfer))
ftp.put(executable, exec_file_location, callback=progress)
# Extract archive locally so we can use run-job.sh.
executable_tgz = tarfile.open(executable, 'r:gz')
executable_tgz.extractall(package_id)
# Generate yarn-site.xml install it in package's local 'config' directory.
yarn_site_dir = self._get_yarn_conf_dir(package_id)
yarn_site_path = os.path.join(yarn_site_dir, 'yarn-site.xml')
logger.info("Installing yarn-site.xml to {0}".format(yarn_site_path))
if not os.path.exists(yarn_site_dir):
os.makedirs(yarn_site_dir)
templates.render_config(configs.get('yarn_site_template'), yarn_site_path, configs.get('yarn_driver_configs'))
def test_copy_logs(self):
install_path = '/tmp/test_copy_dir'
if not os.path.exists(install_path):
os.mkdir(install_path)
output_path = '/tmp/test_copy_dir_output'
if not os.path.exists(output_path):
os.mkdir(output_path)
with open(os.path.join(install_path, 'test.log'), 'w') as f:
f.write('this is the test log')
with open(os.path.join(install_path, 'test.out'), 'w') as f:
f.write('this is the test out')
with open(os.path.join(install_path, 'test.foo'), 'w') as f:
f.write('this is the test foo')
with get_sftp_client('localhost') as ftp:
copy_dir(ftp, install_path, output_path, 'prefix', '.*out')
assert os.path.exists(os.path.join(output_path, "prefix_test_copy_dir-test.out"))
shutil.rmtree(output_path)
shutil.rmtree(install_path)
def test_copy_logs(self):
install_path = "/tmp/test_copy_dir"
if not os.path.exists(install_path):
os.mkdir(install_path)
output_path = "/tmp/test_copy_dir_output"
if not os.path.exists(output_path):
os.mkdir(output_path)
with open(os.path.join(install_path, "test.log"), "w") as f:
f.write("this is the test log")
with open(os.path.join(install_path, "test.out"), "w") as f:
f.write("this is the test out")
with open(os.path.join(install_path, "test.foo"), "w") as f:
f.write("this is the test foo")
with get_sftp_client("localhost") as ftp:
copy_dir(ftp, install_path, output_path, "prefix", ".*out")
assert os.path.exists(os.path.join(output_path, "prefix_test_copy_dir-test.out"))
shutil.rmtree(output_path)
shutil.rmtree(install_path)
def get_logs(self, unique_id, logs, directory):
""" Copies logs from the remote host that the process is running on to the provided directory
:Parameter unique_id the unique_id of the process in question
:Parameter logs a list of logs given by absolute path from the remote host
:Parameter directory the local directory to store the copied logs
"""
hostname = self.processes[unique_id].hostname
if hostname is not None:
# TODO(jehrlich) make 1 ssh client for use in subclasses
with get_sftp_client(hostname) as ftp:
for f in logs:
try:
ftp.stat(f)
except IOError, e:
if e.errno == errno.ENOENT:
logger.error("Log file " + f + " does not exist on " + hostname)
pass
else:
new_file = os.path.join(directory, "{0}-{1}".format(unique_id, os.path.basename(f)))
ftp.get(f, new_file)
def test_copy_logs(self):
install_path = '/tmp/test_copy_dir'
if not os.path.exists(install_path):
os.mkdir(install_path)
output_path = '/tmp/test_copy_dir_output'
if not os.path.exists(output_path):
os.mkdir(output_path)
with open(os.path.join(install_path, 'test.log'), 'w') as f:
f.write('this is the test log')
with open(os.path.join(install_path, 'test.out'), 'w') as f:
f.write('this is the test out')
with open(os.path.join(install_path, 'test.foo'), 'w') as f:
f.write('this is the test foo')
with get_sftp_client('localhost') as ftp:
copy_dir(ftp, install_path, output_path, 'prefix', '.*out')
assert os.path.exists(
os.path.join(output_path, "prefix_test_copy_dir-test.out"))
shutil.rmtree(output_path)
shutil.rmtree(install_path)
def get_logs(self, unique_id, logs, directory, pattern='^$'):
""" Copies logs from the remote host that the process is running on to the provided directory
:Parameter unique_id the unique_id of the process in question
:Parameter logs a list of logs given by absolute path from the remote host
:Parameter directory the local directory to store the copied logs
:Parameter pattern a pattern to apply to files to restrict the set of logs copied
"""
hostname = self.processes[unique_id].hostname
install_path = self.processes[unique_id].install_path
if hostname is not None:
with get_sftp_client(hostname, username=runtime.get_username(), password=runtime.get_password()) as ftp:
for f in logs:
try:
mode = ftp.stat(f).st_mode
except IOError, e:
if e.errno == errno.ENOENT:
logger.error("Log file " + f + " does not exist on " + hostname)
pass
else:
copy_dir(ftp, f, directory, unique_id)
if install_path is not None:
copy_dir(ftp, install_path, directory, unique_id, pattern)
def fetch_logs_from_host(self, hostname, install_path, prefix, logs, directory, pattern):
""" Copies logs from any host on the specified install path
:Parameter hostname the remote host from where we need to fetch the logs
:Parameter install_path path where the app is installed
:Parameter prefix prefix used to copy logs. Generall the unique_id of process
:Parameter logs a list of logs given by absolute path from the remote host
:Parameter directory the local directory to store the copied logs
:Parameter pattern a pattern to apply to files to restrict the set of logs copied
"""
if hostname is not None:
with get_sftp_client(hostname, username=runtime.get_username(), password=runtime.get_password()) as ftp:
for f in logs:
try:
mode = ftp.stat(f).st_mode
except IOError, e:
if e.errno == errno.ENOENT:
logger.error("Log file " + f + " does not exist on " + hostname)
pass
else:
copy_dir(ftp, f, directory, prefix, pattern)
if install_path is not None:
copy_dir(ftp, install_path, directory, prefix, pattern)
def get_logs(self, unique_id, logs, directory):
""" Copies logs from the remote host that the process is running on to the provided directory
:Parameter unique_id the unique_id of the process in question
:Parameter logs a list of logs given by absolute path from the remote host
:Parameter directory the local directory to store the copied logs
"""
hostname = self.processes[unique_id].hostname
if hostname is not None:
# TODO(jehrlich) make 1 ssh client for use in subclasses
with get_sftp_client(hostname) as ftp:
for f in logs:
try:
ftp.stat(f)
except IOError, e:
if e.errno == errno.ENOENT:
logger.error("Log file " + f +
" does not exist on " + hostname)
pass
else:
new_file = os.path.join(
directory,
"{0}-{1}".format(unique_id, os.path.basename(f)))
ftp.get(f, new_file)
def install(self, unique_id, configs=None):
"""
Copies the executable to the remote machine under install path. Inspects the configs for the possible keys
'hostname': the host to install on
'install_path': the location on the remote host
'executable': the executable to copy
'no_copy': if this config is passed in and true then this method will not copy the executable assuming that it is
already installed
'post_install_cmds': an optional list of commands that should be executed on the remote machine after the
executable has been installed. If no_copy is set to true, then the post install commands will not be run.
If the unique_id is already installed on a different host, this will perform the cleanup action first.
If either 'install_path' or 'executable' are provided the new value will become the default.
:param unique_id:
:param configs:
:return:
"""
# the following is necessay to set the configs for this function as the combination of the
# default configurations and the parameter with the parameter superceding the defaults but
# not modifying the defaults
if configs is None:
configs = {}
tmp = self.default_configs.copy()
tmp.update(configs)
configs = tmp
hostname = None
is_tarfile = False
is_zipfile = False
if unique_id in self.processes and 'hostname' in configs:
self.uninstall(unique_id, configs)
hostname = configs['hostname']
elif 'hostname' in configs:
hostname = configs['hostname']
elif unique_id not in self.processes:
# we have not installed this unique_id before and no hostname is provided in the configs so raise an error
raise DeploymentError("hostname was not provided for unique_id: " + unique_id)
env = configs.get("env", {})
install_path = configs.get('install_path') or self.default_configs.get('install_path')
pid_file = configs.get('pid_file') or self.default_configs.get('pid_file')
if install_path is None:
logger.error("install_path was not provided for unique_id: " + unique_id)
raise DeploymentError("install_path was not provided for unique_id: " + unique_id)
if not configs.get('no_copy', False):
with get_ssh_client(hostname, username=runtime.get_username(), password=runtime.get_password()) as ssh:
log_output(better_exec_command(ssh, "mkdir -p {0}".format(install_path),
"Failed to create path {0}".format(install_path)))
log_output(better_exec_command(ssh, "chmod 755 {0}".format(install_path),
"Failed to make path {0} writeable".format(install_path)))
executable = configs.get('executable') or self.default_configs.get('executable')
if executable is None:
logger.error("executable was not provided for unique_id: " + unique_id)
raise DeploymentError("executable was not provided for unique_id: " + unique_id)
#if the executable is in remote location copy to local machine
copy_from_remote_location = False;
if (":" in executable):
copy_from_remote_location = True
if ("http" not in executable):
remote_location_server = executable.split(":")[0]
remote_file_path = executable.split(":")[1]
remote_file_name = os.path.basename(remote_file_path)
local_temp_file_name = os.path.join(configs.get("tmp_dir","/tmp"),remote_file_name)
if not os.path.exists(local_temp_file_name):
with get_sftp_client(remote_location_server,username=runtime.get_username(), password=runtime.get_password()) as ftp:
try:
ftp.get(remote_file_path, local_temp_file_name)
executable = local_temp_file_name
except:
raise DeploymentError("Unable to load file from remote server " + executable)
#use urllib for http copy
else:
remote_file_name = executable.split("/")[-1]
local_temp_file_name = os.path.join(configs.get("tmp_dir","/tmp"),remote_file_name)
if not os.path.exists(local_temp_file_name):
try:
urllib.urlretrieve (executable, local_temp_file_name)
except:
raise DeploymentError("Unable to load file from remote server " + executable)
executable = local_temp_file_name
try:
exec_name = os.path.basename(executable)
install_location = os.path.join(install_path, exec_name)
with get_sftp_client(hostname, username=runtime.get_username(), password=runtime.get_password()) as ftp:
ftp.put(executable, install_location)
except:
raise DeploymentError("Unable to copy executable to install_location:" + install_location)
finally:
#Track if its a tarfile or zipfile before deleting it in case the copy to remote location fails
is_tarfile = tarfile.is_tarfile(executable)
is_zipfile = zipfile.is_zipfile(executable)
if (copy_from_remote_location and not configs.get('cache',False)):
os.remove(executable)
# only supports tar and zip (because those modules are provided by Python's standard library)
if configs.get('extract', False) or self.default_configs.get('extract', False):
if is_tarfile:
log_output(better_exec_command(ssh, "tar -xf {0} -C {1}".format(install_location, install_path),
"Failed to extract tarfile {0}".format(exec_name)))
elif is_zipfile:
log_output(better_exec_command(ssh, "unzip -o {0} -d {1}".format(install_location, install_path),
"Failed to extract zipfile {0}".format(exec_name)))
else:
logger.error(executable + " is not a supported filetype for extracting")
raise DeploymentError(executable + " is not a supported filetype for extracting")
post_install_cmds = configs.get('post_install_cmds', False) or self.default_configs.get('post_install_cmds', [])
for cmd in post_install_cmds:
relative_cmd = "cd {0}; {1}".format(install_path, cmd)
log_output(exec_with_env(ssh, relative_cmd,
msg="Failed to execute post install command: {0}".format(relative_cmd), env=env))
self.processes[unique_id] = Process(unique_id, self.service_name, hostname, install_path)
self.processes[unique_id].pid_file = pid_file
def install(self, unique_id, configs=None):
"""
Copies the executable to the remote machine under install path. Inspects the configs for te possible keys
'hostname': the host to install on
'install_path': the location on the remote host
'executable': the executable to copy
'no_copy': if this config is passed in and true then this method will not copy the executable assuming that it is
already installed
If the unique_id is already installed on a different host, this will perform the cleanup action first.
If either 'install_path' or 'executable' are provided the new value will become the default.
:param unique_id:
:param configs:
:return:
"""
# the following is necessay to set the configs for this function as the combination of the
# default configurations and the parameter with the parameter superceding the defaults but
# not modifying the defaults
if configs is None:
configs = {}
tmp = self.default_configs.copy()
tmp.update(configs)
configs = tmp
hostname = None
if unique_id in self.processes:
process = self.processes[unique_id]
prev_hostname = process.hostname
if 'hostname' in configs:
if prev_hostname is not configs['hostname']:
self.uninstall(unique_id, configs)
hostname = configs['hostname']
else:
self.uninstall(unique_id, configs)
hostname = prev_hostname
elif 'hostname' in configs:
hostname = configs['hostname']
else:
# we have not installed this unique_id before and no hostname is provided in the configs so raise an error
logger.error("hostname was not provided for unique_id: " + unique_id)
raise DeploymentError("hostname was not provided for unique_id: " + unique_id)
install_path = configs.get('install_path') or self.default_configs.get('install_path')
if install_path is None:
logger.error("install_path was not provided for unique_id: " + unique_id)
raise DeploymentError("install_path was not provided for unique_id: " + unique_id)
if not configs.get('no_copy', False):
with get_ssh_client(hostname) as ssh:
better_exec_command(ssh, "mkdir -p {0}".format(install_path), "Failed to create path {0}".format(install_path))
better_exec_command(ssh, "chmod a+w {0}".format(install_path), "Failed to make path {0} writeable".format(install_path))
executable = configs.get('executable') or self.default_configs.get('executable')
if executable is None:
logger.error("executable was not provided for unique_id: " + unique_id)
raise DeploymentError("executable was not provided for unique_id: " + unique_id)
exec_name = os.path.basename(executable)
install_location = os.path.join(install_path, exec_name)
with get_sftp_client(hostname) as ftp:
ftp.put(executable, install_location)
# only supports tar and zip (because those modules are provided by Python's standard library)
if configs.get('extract', False) or self.default_configs.get('extract', False):
if tarfile.is_tarfile(executable):
better_exec_command(ssh, "tar -xf {0} -C {1}".format(install_location, install_path), "Failed to extract tarfile {0}".format(exec_name))
elif zipfile.is_zipfile(executable):
better_exec_command(ssh, "unzip -o {0} -d {1}".format(install_location, install_path), "Failed to extract zipfile {0}".format(exec_name))
else:
logger.error(executable + " is not a supported filetype for extracting")
raise DeploymentError(executable + " is not a supported filetype for extracting")
self.processes[unique_id] = Process(unique_id, self.service_name, hostname, install_path)
def install(self, unique_id, configs=None):
"""
Copies the executable to the remote machine under install path. Inspects the configs for the possible keys
'hostname': the host to install on
'install_path': the location on the remote host
'executable': the executable to copy
'no_copy': if this config is passed in and true then this method will not copy the executable assuming that it is
already installed
'post_install_cmds': an optional list of commands that should be executed on the remote machine after the
executable has been installed. If no_copy is set to true, then the post install commands will not be run.
If the unique_id is already installed on a different host, this will perform the cleanup action first.
If either 'install_path' or 'executable' are provided the new value will become the default.
:param unique_id:
:param configs:
:return:
"""
# the following is necessay to set the configs for this function as the combination of the
# default configurations and the parameter with the parameter superceding the defaults but
# not modifying the defaults
if configs is None:
configs = {}
tmp = self.default_configs.copy()
tmp.update(configs)
configs = tmp
hostname = None
is_tarfile = False
is_zipfile = False
if unique_id in self.processes and 'hostname' in configs:
self.uninstall(unique_id, configs)
hostname = configs['hostname']
elif 'hostname' in configs:
hostname = configs['hostname']
elif unique_id not in self.processes:
# we have not installed this unique_id before and no hostname is provided in the configs so raise an error
raise DeploymentError("hostname was not provided for unique_id: " +
unique_id)
env = configs.get("env", {})
install_path = configs.get('install_path') or self.default_configs.get(
'install_path')
pid_file = configs.get('pid_file') or self.default_configs.get(
'pid_file')
if install_path is None:
logger.error("install_path was not provided for unique_id: " +
unique_id)
raise DeploymentError(
"install_path was not provided for unique_id: " + unique_id)
if not configs.get('no_copy', False):
with get_ssh_client(hostname,
username=runtime.get_username(),
password=runtime.get_password()) as ssh:
log_output(
better_exec_command(
ssh, "mkdir -p {0}".format(install_path),
"Failed to create path {0}".format(install_path)))
log_output(
better_exec_command(
ssh, "chmod 755 {0}".format(install_path),
"Failed to make path {0} writeable".format(
install_path)))
executable = configs.get(
'executable') or self.default_configs.get('executable')
if executable is None:
logger.error(
"executable was not provided for unique_id: " +
unique_id)
raise DeploymentError(
"executable was not provided for unique_id: " +
unique_id)
#if the executable is in remote location copy to local machine
copy_from_remote_location = False
if (":" in executable):
copy_from_remote_location = True
if ("http" not in executable):
remote_location_server = executable.split(":")[0]
remote_file_path = executable.split(":")[1]
remote_file_name = os.path.basename(remote_file_path)
local_temp_file_name = os.path.join(
configs.get("tmp_dir", "/tmp"), remote_file_name)
if not os.path.exists(local_temp_file_name):
with get_sftp_client(
remote_location_server,
username=runtime.get_username(),
password=runtime.get_password()) as ftp:
try:
ftp.get(remote_file_path,
local_temp_file_name)
executable = local_temp_file_name
except:
raise DeploymentError(
"Unable to load file from remote server "
+ executable)
#use urllib for http copy
else:
remote_file_name = executable.split("/")[-1]
local_temp_file_name = os.path.join(
configs.get("tmp_dir", "/tmp"), remote_file_name)
if not os.path.exists(local_temp_file_name):
try:
urllib.urlretrieve(executable,
local_temp_file_name)
except:
raise DeploymentError(
"Unable to load file from remote server " +
executable)
executable = local_temp_file_name
try:
exec_name = os.path.basename(executable)
install_location = os.path.join(install_path, exec_name)
with get_sftp_client(
hostname,
username=runtime.get_username(),
password=runtime.get_password()) as ftp:
ftp.put(executable, install_location)
except:
raise DeploymentError(
"Unable to copy executable to install_location:" +
install_location)
finally:
#Track if its a tarfile or zipfile before deleting it in case the copy to remote location fails
is_tarfile = tarfile.is_tarfile(executable)
is_zipfile = zipfile.is_zipfile(executable)
if (copy_from_remote_location
and not configs.get('cache', False)):
os.remove(executable)
# only supports tar and zip (because those modules are provided by Python's standard library)
if configs.get('extract', False) or self.default_configs.get(
'extract', False):
if is_tarfile:
log_output(
better_exec_command(
ssh, "tar -xf {0} -C {1}".format(
install_location, install_path),
"Failed to extract tarfile {0}".format(
exec_name)))
elif is_zipfile:
log_output(
better_exec_command(
ssh, "unzip -o {0} -d {1}".format(
install_location, install_path),
"Failed to extract zipfile {0}".format(
exec_name)))
else:
logger.error(
executable +
" is not a supported filetype for extracting")
raise DeploymentError(
executable +
" is not a supported filetype for extracting")
post_install_cmds = configs.get(
'post_install_cmds', False) or self.default_configs.get(
'post_install_cmds', [])
for cmd in post_install_cmds:
relative_cmd = "cd {0}; {1}".format(install_path, cmd)
log_output(
exec_with_env(
ssh,
relative_cmd,
msg="Failed to execute post install command: {0}".
format(relative_cmd),
env=env))
self.processes[unique_id] = Process(unique_id, self.service_name,
hostname, install_path)
self.processes[unique_id].pid_file = pid_file
def install(self, unique_id, configs=None):
"""
Copies the executable to the remote machine under install path. Inspects the configs for the possible keys
'hostname': the host to install on
'install_path': the location on the remote host
'executable': the executable to copy
'no_copy': if this config is passed in and true then this method will not copy the executable assuming that it is
already installed
'post_install_cmds': an optional list of commands that should be executed on the remote machine after the
executable has been installed. If no_copy is set to true, then the post install commands will not be run.
If the unique_id is already installed on a different host, this will perform the cleanup action first.
If either 'install_path' or 'executable' are provided the new value will become the default.
:param unique_id:
:param configs:
:return:
"""
# the following is necessay to set the configs for this function as the combination of the
# default configurations and the parameter with the parameter superceding the defaults but
# not modifying the defaults
if configs is None:
configs = {}
tmp = self.default_configs.copy()
tmp.update(configs)
configs = tmp
hostname = None
if unique_id in self.processes:
process = self.processes[unique_id]
prev_hostname = process.hostname
if 'hostname' in configs:
if prev_hostname is not configs['hostname']:
self.uninstall(unique_id, configs)
hostname = configs['hostname']
else:
self.uninstall(unique_id, configs)
hostname = prev_hostname
elif 'hostname' in configs:
hostname = configs['hostname']
else:
# we have not installed this unique_id before and no hostname is provided in the configs so raise an error
logger.error("hostname was not provided for unique_id: " + unique_id)
raise DeploymentError("hostname was not provided for unique_id: " + unique_id)
install_path = configs.get('install_path') or self.default_configs.get('install_path')
if install_path is None:
logger.error("install_path was not provided for unique_id: " + unique_id)
raise DeploymentError("install_path was not provided for unique_id: " + unique_id)
if not configs.get('no_copy', False):
with get_ssh_client(hostname) as ssh:
better_exec_command(ssh, "mkdir -p {0}".format(install_path), "Failed to create path {0}".format(install_path))
better_exec_command(ssh, "chmod a+w {0}".format(install_path), "Failed to make path {0} writeable".format(install_path))
executable = configs.get('executable') or self.default_configs.get('executable')
if executable is None:
logger.error("executable was not provided for unique_id: " + unique_id)
raise DeploymentError("executable was not provided for unique_id: " + unique_id)
exec_name = os.path.basename(executable)
install_location = os.path.join(install_path, exec_name)
with get_sftp_client(hostname) as ftp:
ftp.put(executable, install_location)
# only supports tar and zip (because those modules are provided by Python's standard library)
if configs.get('extract', False) or self.default_configs.get('extract', False):
if tarfile.is_tarfile(executable):
better_exec_command(ssh, "tar -xf {0} -C {1}".format(install_location, install_path), "Failed to extract tarfile {0}".format(exec_name))
elif zipfile.is_zipfile(executable):
better_exec_command(ssh, "unzip -o {0} -d {1}".format(install_location, install_path), "Failed to extract zipfile {0}".format(exec_name))
else:
logger.error(executable + " is not a supported filetype for extracting")
raise DeploymentError(executable + " is not a supported filetype for extracting")
post_install_cmds = configs.get('post_install_cmds', False) or self.default_configs.get('post_install_cmds', [])
for cmd in post_install_cmds:
relative_cmd = "cd {0}; {1}".format(install_path, cmd)
better_exec_command(ssh, relative_cmd, "Failed to execute post install command: {0}".format(relative_cmd))
self.processes[unique_id] = Process(unique_id, self.service_name, hostname, install_path)