def make_dirs(self, remote_dir, mode=777):
"""create all directories in remotedir as needed, setting their mode
to mode, if created.
If remotedir already exists, silently complete. If a regular file is
in the way, raise an exception.
:param str remotedir: the directory structure to create
:param int mode: *Default: 777* -
int representation of octal mode for directory
:returns: None
:raises: OSError
"""
logging_utils.log_func_details(remote_dir=remote_dir, mode=mode)
if self.is_dir(remote_dir):
pass
elif self.is_file(remote_dir):
raise OSError("a file with the same name as the remotedir, "
"'%s', already exists." % remote_dir)
else:
head, tail = os.path.split(remote_dir)
if head and not self.is_dir(head):
self.make_dirs(head, mode)
if tail:
self.mkdir(remote_dir, mode=mode)
def initiate_connection(self, verbose=False):
"""
connect telnet
:param hostname: str - ip for hostname to connect at.
:param username: str - username used to connection
:param password: str - password used to connection
:param port: int - port used for connection
:return: telnet prompt
"""
if verbose:
log_func_details(hostname=self.hostname,
password=self.password,
verbose=verbose)
try:
self._client = telnetlib.Telnet(self.hostname, Telnet.port)
if self._client is None:
return False
except Exception as e:
logger.error("error connecting to telnet!!")
raise ConnectionError(e)
self._client.write(b'\n')
self._client.read_until(b'login:'******'ascii') + b'\n')
self._client.read_until(b'Password:'******'ascii') + b'\n')
time.sleep(5)
self.telnet_prompt = self._client.read_very_eager().decode(
'ascii').splitlines()[-1]
logger.info('connected to Telnet!')
logger.info("the Telnet prompt is: {0}".format(self.telnet_prompt))
return True
def initiate_connection(self, verbose=False):
"""
Establish the SFTP connection.
:param host: str ip
:param username: str username
:param password:
:return:
"""
logging_utils.log_func_details(hostname=self.hostname,
username=self.username,
password=self.password)
if self._sftp is None:
logger.info("establishing new sftp connection... ")
cnopts = pysftp.CnOpts()
cnopts.hostkeys = None
self._sftp = pysftp.Connection(host=self.hostname,
username=self.username,
password=self.password,
cnopts=cnopts)
logger.info("new sftp connection was established : {0}".format(
self._sftp))
else:
logger.warning("already has sftp connection : {0}".format(
self._sftp))
def start_tunnel(self, getting_tunnel_details=False, verbose=False):
"""
start tunnel
:param getting_tunnel_details:
:param verbose
:return: True/False if tunnel started or not.
"""
if verbose:
log_func_details(getting_tunnel_details=getting_tunnel_details)
logger.info("starting server...")
self.ssh_tunnel.start()
logger.info("check if tunnel is active and up after starting tunnel...")
if (self.ssh_tunnel.tunnel_is_up is not None):
logger.info("Tunnel is up and ready to go!!!")
if getting_tunnel_details:
logger.info("here is tunnel connction details:")
logger.info("local_bind_address: {0} ".format(self.ssh_tunnel.local_bind_address))
logger.info("local_bind_host: {0} ".format(self.ssh_tunnel.local_bind_host))
logger.info("local_bind_port: {0}".format(self.ssh_tunnel.local_bind_port))
logger.info(
"tunnel_is_up: {0}".format(self.ssh_tunnel.tunnel_is_up[self.ssh_tunnel.local_bind_address]))
self.local_bind_address = self.ssh_tunnel.local_bind_address[0]
self.local_bind_port = self.ssh_tunnel.local_bind_port
return True
else:
raise Exception("Tunnel not been up after starting server !!!")
def terminate_connection(self, sleep_before=0, verbose=True):
"""
close connection in case the connection still opened until end of the script
:param sleep_before: in sec <int>
:param verbose: Boolean - activate the trace
:return: None
:Raises: Exception in case cannot close connection
"""
if verbose:
log_func_details(sleep_before=sleep_before, verbose=verbose)
time.sleep(sleep_before)
try:
if self._client is not None:
if self._client.get_transport() is not None:
self._client.close()
logger.info("connection is closed successfully!")
else:
logger.info("connection is closed!")
else:
logger.error(
"there is no open connection to close, you should did establish_connection before !!"
.format(self._client))
raise ValueError("no open connection to close!")
except Exception as e:
raise Exception("connection <{0}> cannot be closed: {1}".format(
self._client, e))
def run_command_status_via_ssh(command, device=None, verbose=False):
"""
Run Shell command on local or remote device
Return command rc, stdout, stderr.
:param device: None if Local , String if remote
:param verbose: true for debug mode , False for info
:param command: command string
:type command: str
:type device: str
:type verbose: bool
:return: rc, stdout, stderr
:rtype: tuple
"""
if verbose:
log_func_details(command=command, device=device, verbose=verbose)
if device is not None:
if verbose:
logger.info("runCommandStatus (%s) : %s" % (device, command))
try:
logger.info(
"trying connection using the using default <pg,q1w2e3r4>....")
ssh_params = ConnParamsFactory.get_params(
ConnectionType.SSH,
hostname=device,
username=SSHCredentials.PG_USERNAME,
password=SSHCredentials.PG_PASSWORD)
if verbose:
ssh_params.get_params_dict(True)
conn = ConnectionFactory.get_connection(ConnectionType.SSH,
ssh_params)
ssh_conn = ssh_casting(conn)
ssh_conn.initiate_connection(verbose=verbose)
_, stdout, stderr, return_code = ssh_conn.exec_command(
command=command, verbose=verbose)
logger.info("run command passed successfully!!")
conn.terminate_connection(verbose=verbose)
logger.info("connection closed!")
except Exception as e:
raise Exception("Error running command status: {0}".format(e))
return return_code, stdout, stderr
else:
if verbose:
logger.info("runCommandStatus : " + str(command))
proc = subprocess.Popen(command,
shell=True,
close_fds=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = proc.communicate()
if verbose:
logger.info("rc: " + str(proc.returncode))
logger.info("stdout: " + str(stdout).strip())
if str(stderr.strip()) != '':
logger.info("stderr: " + str(stderr).strip())
return proc.returncode, str(stdout).strip(), str(stderr).strip()
def rmdir(self, remote_path):
"""remove remote directory
:param str remote_path: the remote directory to remove
:returns: None
"""
logging_utils.log_func_details(remote_path=remote_path)
self._sftp.rmdir(remote_path)
def stat(self, remote_path):
"""return information about file/directory for the given remote path
:param str remote_path: path to stat
:returns: (obj) SFTPAttributes
"""
logging_utils.log_func_details(remote_path=remote_path)
return self._sftp.stat(remote_path)
def list_dir(self, remote_path='.'):
"""return a list of files/directories for the given remote path.
Unlike, paramiko, the directory listing is sorted.
:param str remote_path: path to list on the server
:returns: (list of str) directory entries, sorted
"""
logging_utils.log_func_details(remote_path=remote_path)
return sorted(self._sftp.listdir(remote_path))
def lstat(self, remote_path):
"""return information about file/directory for the given remote path,
without following symbolic links. Otherwise, the same as .stat()
:param str remote_path: path to stat
:returns: (obj) SFTPAttributes object
"""
logging_utils.log_func_details(remote_path=remote_path)
return self._sftp.lstat(remote_path)
def terminate_connection(self, sleep_before=0, verbose=False):
"""
kill connection to telnet
:param sleep_before: time in sec before kill connection <int>
:return: None
"""
if verbose:
log_func_details(sleep_before=sleep_before, verbose=verbose)
time.sleep(sleep_before)
self._client.close()
logger.info("Telnet connection is closed!")
def read_link(self, remotelink):
"""Return the target of a symlink (shortcut). The result will be
an absolute pathname.
:param str remotelink: remote path of the symlink
:return: (str) absolute path to target
"""
logging_utils.log_func_details(remotelink=remotelink)
return self._sftp.normalize(self._sftp.read_link(remotelink))
def chdir(self, remote_path):
"""change the current working directory on the remote
:param str remote_path: the remote path to change to
:returns: None
:raises: IOError, if path does not exist
"""
logging_utils.log_func_details(remote_path=remote_path)
self._sftp.chdir(remote_path)
def put_r(self,
localpath,
remote_path,
confirm=True,
preserve_mtime=False):
"""Recursively copies a local directory's contents to a remote_path
:param str localpath: the local path to copy (source)
:param str remote_path:
the remote path to copy to (target)
:param bool confirm:
whether to do a stat() on the file afterwards to confirm the file
size
:param bool preserve_mtime:
*Default: False* - make the modification time(st_mtime) on the
remote file match the time on the local. (st_atime can differ
because stat'ing the localfile can/does update it's st_atime)
:returns: None
:raises IOError: if remote_path doesn't exist
:raises OSError: if localpath doesn't exist
"""
logging_utils.log_func_details(localpath=localpath,
remote_path=remote_path,
confirm=confirm,
preserve_mtime=preserve_mtime)
wtcb = WTCallbacks()
cur_local_dir = os.getcwd()
os.chdir(localpath)
walktree('.', wtcb.file_cb, wtcb.dir_cb, wtcb.unk_cb)
# restore local directory
os.chdir(cur_local_dir)
for dname in wtcb.dlist:
if dname != '.':
pth = reparent(remote_path, dname)
if not self.is_dir(pth):
self.mkdir(pth)
for fname in wtcb.flist:
head, _ = os.path.split(fname)
if head not in wtcb.dlist:
for subdir in path_advance(head):
if subdir not in wtcb.dlist and subdir != '.':
self.mkdir(reparent(remote_path, subdir))
wtcb.dlist = wtcb.dlist + [
subdir,
]
src = os.path.join(localpath, fname)
dest = reparent(remote_path, fname)
# print('put', src, dest)
self.put(src, dest, confirm=confirm, preserve_mtime=preserve_mtime)
def remove(self, remote_file):
"""remove the file @ remotefile, remotefile may include a path, if no
path, then :attr:`.pwd` is used. This method only works on files
:param str remotefile: the remote file to delete
:returns: None
:raises: IOError
"""
logging_utils.log_func_details(remote_file=remote_file)
self._sftp.remove(remote_file)
def mkdir(self, remote_path, mode=777):
"""Create a directory named remote_path with mode. On some systems,
mode is ignored. Where it is used, the current umask value is first
masked out.
:param str remote_path: directory to create`
:param int mode: *Default: 777* -
int representation of octal mode for directory
:returns: None
"""
logging_utils.log_func_details(remote_path=remote_path, mode=mode)
self._sftp.mkdir(remote_path, mode=int(str(mode), 8))
def run_command_via_ssh(command, device=None, verbose=False):
"""
running command using default credentials, Run Shell command
on local or remote device Return command exit code.
:param device: None if Local , String if remote
:param verbose: true for debug mode , False for info
:param command: command string
:type command: str
:type device: str
:type verbose: bool
:return: return code
:rtype: int
:TODO adding more than try statement to try multiple scenarios from username/pass
"""
if verbose:
log_func_details(command=command, device=device, verbose=verbose)
if device is not None:
try:
logger.debug("< Device: %s > runCommand : %s" % (device, command))
logger.debug(
"trying connection using the using default <pg,q1w2e3r4>....")
ssh_params = ConnParamsFactory.get_params(
ConnectionType.SSH,
hostname=device,
username=SSHCredentials.PG_USERNAME,
password=SSHCredentials.PG_PASSWORD)
ssh_params.get_params_dict(True)
conn = ConnectionFactory.get_connection(ConnectionType.SSH,
ssh_params)
ssh_conn = ssh_casting(conn)
ssh_conn.initiate_connection(verbose=True)
_, _, _, rc = ssh_conn.exec_command(command=command)
logger.info("run command passed successfully!!")
conn.terminate_connection()
logger.info("connection closed!")
except Exception as e:
raise Exception("Error running command: {0}".format(e))
return rc
else:
if verbose:
logger.info("runCommand : " + str(command))
with open(os.devnull, 'w') as tempf:
proc = subprocess.Popen(command,
shell=True,
stdout=tempf,
stderr=tempf)
proc.communicate()
return proc.returncode
def put(self,
localpath,
remote_path=None,
callback=None,
confirm=True,
preserve_mtime=False):
"""Copies a file between the local host and the remote host.
:param str localpath: the local path and filename
:param str remote_path:
the remote path, else the remote :attr:`.pwd` and filename is used.
:param callable callback:
optional callback function (form: ``func(int, int``)) that accepts
the bytes transferred so far and the total bytes to be transferred.
:param bool confirm:
whether to do a stat() on the file afterwards to confirm the file
size
:param bool preserve_mtime:
*Default: False* - make the modification time(st_mtime) on the
remote file match the time on the local. (st_atime can differ
because stat'ing the localfile can/does update it's st_atime)
:returns:
(obj) SFTPAttributes containing attributes about the given file
:raises IOError: if remote_path doesn't exist
:raises OSError: if localpath doesn't exist
"""
logging_utils.log_func_details(localpath=localpath,
remote_path=remote_path,
callback=callback,
confirm=confirm,
preserve_mtime=preserve_mtime)
if not remote_path:
remote_path = os.path.split(localpath)[1]
if preserve_mtime:
local_stat = os.stat(localpath)
times = (local_stat.st_atime, local_stat.st_mtime)
sftpattrs = self._sftp.put(localpath,
remote_path,
callback=callback,
confirm=confirm)
if preserve_mtime:
self._sftp.utime(remote_path, times)
sftpattrs = self._sftp.stat(remote_path)
return sftpattrs
def is_file(self, remote_path):
"""return true if remote_path is a file
:param str remote_path: the path to test
:returns: (bool)
"""
logging_utils.log_func_details(remote_path=remote_path)
try:
result = S_ISREG(self._sftp.stat(remote_path).st_mode)
except IOError: # no such file
result = False
return result
def rename(self, remote_src, remote_dest):
"""rename a file or directory on the remote host.
:param str remote_src: the remote file/directory to rename
:param str remote_dest: the remote file/directory to put it
:returns: None
:raises: IOError
"""
logging_utils.log_func_details(remote_src=remote_src,
remote_dest=remote_dest)
self._sftp.rename(remote_src, remote_dest)
def terminate_connection(self, sleep_before=0, verbose=False):
"""
abstract method to enforce inherited classes to implement it
:param sleep_before: time before terminate in sec
:type sleep_before: int
:param verbose: to enable or disable debug mode
:type verbose: bool
:return: None
"""
import time
logging_utils.log_func_details(sleep_before=sleep_before,
verbose=verbose)
time.sleep(sleep_before)
self._sftp.close()
logger.info("the sftp connection is closed!")
def run_uninterruptible_command(command, device=None, verbose=False):
"""
Use nohup command to run another uninterruptible command in background that
can't be terminated if its session was closed or terminated.
:param command: command to execute
:param device: None if Local , String if remote
:param verbose: true for debug mode , False for info
:type command: str
:type device: str
:type verbose: bool
:return: rc, stdout, stderr
:rtype: tuple
"""
if verbose:
log_func_details(command=command, device=device)
command = 'nohup sh -c "' + command + '" > /dev/null 2>&1 &'
return run_command_status_via_ssh(command, device)
def cd(self, remote_path=None): # pylint:disable=c0103
"""context manager that can change to a optionally specified remote
directory and restores the old pwd on exit.
:param str|None remote_path: *Default: None* -
remote_path to temporarily make the current directory
:returns: None
:raises: IOError, if remote path doesn't exist
"""
logging_utils.log_func_details(remote_path=remote_path)
original_path = self.pwd
try:
if remote_path is not None:
self.chdir(remote_path)
yield
finally:
self.chdir(original_path)
def chmod(self, remote_path, mode=777):
"""set the mode of a remote_path to mode, where mode is an integer
representation of the octal mode to use.
:param str remote_path: the remote path/file to modify
:param int mode: *Default: 777* -
int representation of octal mode for directory
:returns: None
:raises: IOError, if the file doesn't exist
"""
logging_utils.log_func_details(remote_path=remote_path, mode=mode)
self._sftp.chmod(remote_path, mode=int(str(mode), 8))
logger.info("changed mode of file <{0}> to mode : {1} ".format(
remote_path, mode))
def get_r(self, remotedir, localdir, preserve_mtime=False):
"""recursively copy remotedir structure to localdir
:param str remotedir: the remote directory to copy from
:param str localdir: the local directory to copy to
:param bool preserve_mtime: *Default: False* -
preserve modification time on files
:returns: None
:raises:
"""
logging_utils.log_func_details(remotedir=remotedir,
localdir=localdir,
preserve_mtime=preserve_mtime)
wtcb = WTCallbacks()
self.walktree(remotedir, wtcb.file_cb, wtcb.dir_cb, wtcb.unk_cb)
# handle directories we recursed through
for dname in wtcb.dlist:
for subdir in path_advance(dname):
try:
os.mkdir(reparent(localdir, subdir))
# force result to a list for setter,
wtcb.dlist = wtcb.dlist + [
subdir,
]
except OSError: # dir exists
pass
for fname in wtcb.flist:
# they may have told us to start down farther, so we may not have
# recursed through some, ensure local dir structure matches
head, _ = os.path.split(fname)
if head not in wtcb.dlist:
for subdir in path_advance(head):
if subdir not in wtcb.dlist and subdir != '.':
os.mkdir(reparent(localdir, subdir))
wtcb.dlist = wtcb.dlist + [
subdir,
]
self.get(fname,
reparent(localdir, fname),
preserve_mtime=preserve_mtime)
def exec_tail_log(self,
cmd,
name,
thread_id,
dict,
map,
timeout=10,
enable_wait=True,
verbose=True,
logger=None):
"""
this function send the command for the channel, and wait for prompt
:param cmd: the text to send to the channel
:param prompt: prompt to wait after sending the command. default value is gp prompt
:param timeout: timeout for the command to finish
:param enable_wait: when true, the function will run in blocking mode and wait for given prompt
when false, the function will run the command and return.
:return: buffer: [interactive mode] the output of the command
stdin, stdout, stderr: [non interactive mode] channel standard output
"""
if verbose:
log_func_details(cmd=cmd,
name=name,
thread_id=thread_id,
dict=dict,
map=map,
timeout=timeout,
enable_wait=enable_wait,
verbose=verbose,
logger=logger)
_id = threading.get_ident()
thread_id[_id] = True
dict[name] = ''
map[name] = _id
shell = self._client.invoke_shell()
shell.send(cmd + '\n')
start_time = time.time()
while timeout > time.time() - start_time or (timeout == -1
and thread_id[_id]):
if shell.recv_ready():
line = shell.recv(9999)
if verbose and logger:
logger.debug(line)
dict[name] += line.decode("utf-8")
return dict[name]
def exec_command(self,
command,
bufsize=-1,
timeout=None,
sudo_required=False,
get_pty=False,
sudo_passwd='q1w2e3r4',
verbose=True):
"""
exec command via ssh
:param command: <str>
:param bufsize: <int>
:param timeout: <int> second , None in None
:param sudo: <boolean> if sudo needed
:param get_pty: <boolean>
:param sudo_passwd: <string> sudo password
:param verbose <boolean>
:return: stdin, stdout, stderr , exit_code
"""
if verbose:
log_func_details(command=command,
bufsize=bufsize,
timeout=timeout,
sudo=sudo_required,
get_pty=get_pty,
sudo_passwd=sudo_passwd)
if sudo_required or "sudo" in command.lower():
logger.info(
"sudo command was detected,the password used for sudo is: {0}".
format(sudo_passwd))
command = "echo '{0}' | sudo -S {1}".format(sudo_passwd, command)
stdin, stdout, stderr = self._client.exec_command(
command, bufsize, timeout, get_pty)
exit_status = stdout.channel.recv_exit_status()
stdout = "".join(stdout.readlines())
stderr = "".join(stderr.readlines())
if verbose:
logger.debug("stdout:{0}".format(stdout))
logger.debug("stderr:{0}".format(stderr))
logger.debug("exit_status: {0}".format(exit_status))
return stdin, stdout, stderr, exit_status
def terminate_connection(self, sleep_before=0, verbose=False):
"""
terminate ssh tunnnel
:param sleep_before: time before terminate in sec
:type sleep_before: int
:param verbose: to enable or disable debug mode
:type verbose: bool
:return:
"""
import time
if verbose:
log_func_details(sleep_before=sleep_before, verbose=verbose)
time.sleep(sleep_before)
logger.info("stopping server...")
self.ssh_tunnel.close()
logger.info("check if tunnel is in-active and down after stopping tunnel...")
if self.ssh_tunnel.tunnel_is_up is not {}:
logger.info("Tunnel has been closed!!")
return True
else:
raise Exception("Tunnel not been down after stopping it !!!")
def initiate_connection(self, verbose=False):
"""
initiate_tunnel via ssh (abstract method)
:param verbose: boolean - for debug purpose
:return: True/False if tunnel initiated ot not.
:rtype : bool
"""
if verbose:
log_func_details(ssh_address_or_host=self.hostname, ssh_username=self.username,
ssh_pkey_file_path=self.ssh_pkey_file_path, remote_bind_address=self.remote_bind_address,
remote_bind_port=self.remote_bind_port, verbose=verbose)
logger.info("start initiating ssh Tunnel between localhost and remote-host: %s " % self.hostname)
try:
self.ssh_tunnel = SSHTunnelForwarder(ssh_address_or_host=self.hostname,
ssh_username=self.username,
ssh_password=self.password,
ssh_pkey=self.ssh_pkey_file_path,
remote_bind_address=(self.remote_bind_address, self.remote_bind_port),
ssh_port=self.port)
except Exception as e:
logger.error("error getting the tunnel connection via SSHTunnelForwarder")
raise ConnectionError(e)
logger.info("starting tunnel...")
self.ssh_tunnel.start()
logger.info("check if tunnel is active and up after starting tunnel...")
if (self.ssh_tunnel.tunnel_is_up is not None):
logger.info("Tunnel is up and ready to go!!!")
if verbose:
logger.info("here is tunnel connction details:")
logger.info("local_bind_address: {0} ".format(self.ssh_tunnel.local_bind_address))
logger.info("local_bind_host: {0} ".format(self.ssh_tunnel.local_bind_host))
logger.info("local_bind_port: {0}".format(self.ssh_tunnel.local_bind_port))
logger.info(
"tunnel_is_up: {0}".format(self.ssh_tunnel.tunnel_is_up[self.ssh_tunnel.local_bind_address]))
self.local_bind_address = self.ssh_tunnel.local_bind_address[0]
self.local_bind_port = self.ssh_tunnel.local_bind_port
logger.info("Initiating SSH Tunnel Passed Greatly !!!")
return True
return False
def initiate_connection(self, verbose=True):
"""
open a connection to the given machine, the function set a channel for interactive shell.
if it fail to open a connection the function will print an error to logger an then raise an exception
:param verbose <boolean> trace option
:return: connection (self._client)
"""
if verbose:
logger.debug("[*] initiate the ssh connection... ")
log_func_details(hostname=self.hostname,
username=self.username,
passwor=self.password,
port=self.port,
timeout=self.timeout,
verbose=verbose)
self._client = paramiko.SSHClient()
self._client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
connection_success = False
for i in range(self.retries):
try:
logger.debug("retries #:{0}".format(i + 1))
self._client.connect(hostname=self.hostname,
port=self.port,
username=self.username,
password=self.password,
timeout=self.timeout)
connection_success = True
break
except Exception as e:
logger.error(e)
time.sleep(3)
pass
if not connection_success:
logger.error(
"fail to connect to hostname: {0} username: {1} password: {2}".
format(self.hostname, self.username, self.password))
raise Exception
logger.info("connected to host: {0}".format(self.hostname))
