def get_file_from_host(self, from_path, to_path):
'''
A wrapper api on top of paramiko scp module, to scp
a remote file to the local.
'''
sshcon = self._get_client()
scpcon = scp.SCPClient(sshcon.get_transport())
try:
scpcon.get(from_path, to_path)
except scp.SCPException as exp:
LOG.error("Receive failed: [%s]", exp)
return 0
return 1
def create_scp_client(ssh_client):
"""
Creates a client for SCP transfer, given an open SSH client.
See `create_ssh_client` to create an SSH client.
"""
# sanitize argument lets us pass wildcards to get calls
scp_client = scp.SCPClient(ssh_client.get_transport(),
sanitize=lambda x: x,
socket_timeout=1)
return scp_client
def put_file_to_host(self, from_path, to_path):
'''
A wrapper api on top of paramiko scp module, to scp
a local file to the remote.
'''
sshcon = self._get_client()
scpcon = scp.SCPClient(sshcon.get_transport())
try:
scpcon.put(from_path, remote_path=to_path)
except scp.SCPException as exp:
LOG.error("Send failed: [%s]", exp)
return 0
return 1
def sftp_put(self, src, dest):
""" Puts a file to the remote server
Arguments:
- src: Source local file to copy.
- dest: Destination path in the remote server.
"""
client, proxy = self.connect()
transport = client.get_transport()
try:
sftp = paramiko.SFTPClient.from_transport(transport)
if not transport.active:
sftp = scp.SCPClient(transport)
except Exception:
# in case of failure try to use scp
sftp = scp.SCPClient(transport)
sftp.put(src, dest)
if self.auto_close:
sftp.close()
if proxy:
proxy.close()
transport.close()
def establish_scp_conn(self):
"""Establish the secure copy connection."""
self.scp_conn = paramiko.SSHClient()
self.scp_conn.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.scp_conn.connect(hostname=self.ssh_ctl_chan.host,
port=self.ssh_ctl_chan.port,
username=self.ssh_ctl_chan.username,
password=self.ssh_ctl_chan.password,
key_filename=self.ssh_ctl_chan.key_file,
look_for_keys=self.ssh_ctl_chan.use_keys,
allow_agent=self.ssh_ctl_chan.allow_agent,
timeout=self.ssh_ctl_chan.timeout)
self.scp_client = scp.SCPClient(self.scp_conn.get_transport())
def __init__(self,
hostname,
username=None,
password=None,
pkey=None,
timeout=None):
self.ssh = paramiko.SSHClient()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
if type(hostname) is bytes:
self.hostname = hostname.decode('utf-8')
else:
self.hostname = hostname
if type(username) is bytes:
self.username = username.decode('utf-8')
else:
self.username = username
self.timeout = timeout
if type(password) is bytes:
password = password.decode('utf-8')
if type(pkey) is bytes:
pkey = pkey.decode('utf-8')
if password == "":
password = None
if pkey is None or pkey == "":
self.pkey = None
else:
self.pkey = paramiko.rsakey.RSAKey(file_obj=io.StringIO(pkey),
password=password)
password = None
self.password = password
self.path = "~/"
try:
#logging.debug('connect_y')
self.ssh.connect(hostname=self.hostname,
username=self.username,
password=self.password,
pkey=self.pkey,
timeout=self.timeout)
#logging.debug('connect_n')
except Exception as e:
logging.debug('Excepetion: {}'.format(e))
raise e
self.scp = scp.SCPClient(self.ssh.get_transport())
self.connected = True
def main():
with open('C:\\Users\\ayc08\\Documents\\GitHub\\python_study\\src\\nonpass\\sample\\sample.csv', 'r') as f:
read = csv.DictReader(f)
for row in read:
source_host = row.get('source_host')
print(source_host)
source_host_fqdn = source_host + '.b.' + source_host[-3:] + '.server'
print(source_host_fqdn)
with paramiko.SSHClient() as ssh:
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=source_host_fqdn, port=22, username="", password="")
with scp.SCPClient(ssh.get_transport()) as scp:
scp.put('','')
source_dir = row.get('source_directory') + '/.ssh'
stdin, stdout, stderr = ssh.exec_command('python ' + GENERATE_PUB_KEY + ' ' + source_dir)
scp.get('','')
ssh.exec_command('rm ' + GENERATE_PUB_KEY)
target_host = row.get('target_host')
print(target_host)
target_host_fqdn = target_host + '.b.' + target_host[-3:] + '.server'
print(target_host_fqdn)
with paramiko.SSHClient() as ssh:
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=target_host_fqdn, port=22, username="", password="")
with scp.SCPClient(ssh.get_transport()) as scp:
scp.put('','')
target_dir = row.get('target_directory') + '/.ssh'
stdin, stdout, stderr = ssh.exec_command('python ' + SETTING_PUB_KEY + ' ' + target_dir)
ssh.exec_command('rm ' + SETTING_PUB_KEY)
def get_files(self, remote_file, local_file):
try:
#Could use SFTP
#self.ftp=self.ssh.open_sftp()
#self.ftp.get(remote_file,local_file)
self.scp = scp.SCPClient(self.ssh.get_transport())
self.scp.get(remote_file, local_path=local_file)
self.logfile.info("Successfully received file from %s to %s" %
(remote_file, local_file))
return True
except Exception, e:
self.logfile.error("get_files caught exception %s - %s" %
(e.__class__, e))
return False
def connect(self):
if self.keys:
self.ssh.connect(self.host,
username=self.user,
password=self.password)
else:
self.ssh.connect(self.host,
username=self.user,
password=self.password,
look_for_keys=False)
connected = self.test()
if connected:
self.scp = scp.SCPClient(self.ssh.get_transport())
return connected
def put_dirs(self, local_dir, remote_dir):
try:
self.scp = scp.SCPClient(self.ssh.get_transport())
self.scp.put(local_dir, remote_path=remote_dir, recursive=True)
#SCPClient cant control permissions on copy from Windows to Linux
time.sleep(2)
self.ssh.exec_command("chmod -R 755 " + remote_dir)
self.logfile.info("Successfully put directory to %s" %
(remote_dir))
return True
except Exception, e:
self.logfile.error("put_dirs caught exception %s - %s" %
(e.__class__, e))
return False
def request_scp_transfer(self):
""" Begins SCP file transfer with progress """
self.ensure_scp()
self._init_transfer()
source = self.upgrade_image_src_path
dest = self.upgrade_image_dest_path
ssh_connect_params = self.ft.ssh_ctl_chan._connect_params_dict()
self.ft.scp_conn = self.ft.ssh_ctl_chan._build_ssh_client()
self.ft.scp_conn.connect(**ssh_connect_params)
with tqdm(unit="b", unit_scale=True, ascii=True) as t:
self.progress = self._scp_tqdm(t)
self.ft.scp_client = scp.SCPClient(
self.ft.scp_conn.get_transport(), progress=self.progress)
self.ft.scp_client.put(source, dest)
def prepare_download(self, files):
# Checks whether the VM is running
try:
info = get_ssh_parameters(self.target)
except subprocess.CalledProcessError:
logging.critical("Failed to get the status of the machine -- is "
"it running?")
sys.exit(1)
# Connect with scp
self.ssh = paramiko.SSHClient()
self.ssh.set_missing_host_key_policy(IgnoreMissingKey())
self.ssh.connect(**info)
self.client_scp = scp.SCPClient(self.ssh.get_transport())
def sftp_get_files(self, src, dest):
""" Gets a list of files from the remote server
Arguments:
- src: A list with the source files in the remote server.
- dest: A list with the local destination paths to copy.
"""
client, proxy = self.connect()
transport = client.get_transport()
try:
sftp = paramiko.SFTPClient.from_transport(transport)
if not transport.active:
sftp = scp.SCPClient(transport)
except Exception:
# in case of failure try to use scp
sftp = scp.SCPClient(transport)
for file0, file1 in zip(src, dest):
sftp.get(file0, file1)
sftp.close()
if proxy:
proxy.close()
transport.close()
def sftp_put_files(self, files):
""" Puts a list of files to the remote server
Arguments:
- files: A tuple where the first elements is the local source file to copy and the second
element the destination paths in the remote server.
"""
client, proxy = self.connect()
transport = client.get_transport()
try:
sftp = paramiko.SFTPClient.from_transport(transport)
if not transport.active:
sftp = scp.SCPClient(transport)
except Exception:
# in case of failure try to use scp
sftp = scp.SCPClient(transport)
for src, dest in files:
sftp.put(src, dest)
sftp.close()
if proxy:
proxy.close()
transport.close()
def move_torrents(hostname,
username,
password,
torrents,
target_dir='/volume1/Shared/torrents'):
"""Given a list of files, will move them to remote using scp"""
with paramiko.SSHClient() as client:
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(hostname=hostname, username=username, password=password)
for torrent in torrents:
with scp.SCPClient(client.get_transport()) as scp_session:
scp_session.put(torrent, target_dir)
def send_file(self, user, password, ip, source, dest):
try:
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(ip, username=user, password=password)
scp_client = scp.SCPClient(client.get_transport())
scp_client.put(source, dest)
return n4d.responses.build_successful_call_response()
except Exception as e:
return n4d.responses.build_failed_call_response(ret_msg=str(e))
def establish_scp_conn(self):
'''
Establish the SCP connection
'''
self.scp_conn = paramiko.SSHClient()
self.scp_conn.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.scp_conn.connect(hostname=self.ssh_ctl_chan.ip,
port=self.ssh_ctl_chan.port,
username=self.ssh_ctl_chan.username,
password=self.ssh_ctl_chan.password,
look_for_keys=False,
allow_agent=False,
timeout=8)
self.scp_client = scp.SCPClient(self.scp_conn.get_transport())
def download_backup_file(file, host):
ssh_client = Client.get_instance(host, 'serverpilot')
# SCPCLient takes a paramiko transport as an argument
scp_client = scp.SCPClient(ssh_client.get_transport(), progress=progress)
try:
scp_client.get(file, environ.get('HOME'))
except SCPException as scpe:
print('Error fetching the file: {}'.format(str(scpe)))
exit(-3)
scp_client.close()
def transfer_to(self, local_path, remote_path, recursive=False):
"""Copy file or directory to default connection.
Args:
local_path: Location of local file or directory.
remote_path: Location of remote file or directory.
recursive (bool): Must be set to True if 'local_path' is a directory.
"""
_LOGGER.debug("copying %s -> %s:%s", self._host, local_path, remote_path)
_local_path = str(local_path)
_remote_path = str(remote_path)
with self as ssh_client:
ssh_client.connect()
with scp.SCPClient(ssh_client.get_transport()) as scp_client:
scp_client.put(_local_path, _remote_path, recursive=recursive)
def scp_copy_file(self, local_file_path, remote_file_path):
"""Copy file to remote via SCP."""
ssh_client = self.get_client()
if self.verbose:
log.verbose("Copy file to remote: -")
log.verbose(" %s" % local_file_path)
log.verbose(" %s" % remote_file_path)
with scp.SCPClient(ssh_client.get_transport()) as scp_client:
scp_client.put(local_file_path, remote_file_path)
if self.ssh_client is None:
ssh_client.close()
def scpJob(c):
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
if c.get("password", "") != "":
client.connect(c["host"],
port=c["port"],
username=c["username"],
password=c["password"])
else:
client.load_system_host_keys()
client.connect(c["host"], port=c["port"], username=c["username"])
# stdin, stdout, stderr = client.exec_command('ls -l')
cp = scp.SCPClient(client.get_transport())
logging.info(c.get("name", "connecting to server..."))
local_time = datetime.now(timezone.utc).replace(microsecond=0).astimezone()
# now = local_time.isoformat()
now = local_time.strftime("%Y-%m-%dT%H%M_%s")
parent = os.path.dirname(os.path.abspath(c["dst"]))
basename = os.path.basename(c["dst"]) + "_" + now
dst = os.path.join(parent, basename)
os.makedirs(dst, mode=511, exist_ok=True)
for src in c["srcList"]:
tmp = os.path.join(dst, os.path.basename(src))
cp.get(src, local_path=tmp, recursive=True)
cp.close()
client.close()
if c.get("format", "") == "": return
if c["format"] == "tar.gz":
shutil.make_archive(dst, "gztar", parent, basename)
else:
shutil.make_archive(dst, c["format"], parent, basename)
shutil.rmtree(dst)
logging.info("saved {}!".format(dst + "." + c["format"]))
n = c.get("keep", 0)
if n <= 0: return
fs = glob(c["dst"] + "_*." + c["format"])
fs = sorted(fs, key=os.path.getctime, reverse=True)
for f in fs[n:]:
logging.warning("deleting " + f)
os.remove(f)
def uploadFile(sourcePath: str,
targetPath: str,
sshc: paramiko.client.SSHClient,
compress_method: str = None,
verbose: bool = True) -> pathlib.Path:
def show_progress(filename, size, sent):
print(f"Uploading {filename} progress: " +
f"{float(sent)/float(size)*100:.2f}%",
end="\r")
progress = show_progress if verbose else None
try:
if compress_method:
fileName = pathlib.Path(sourcePath).name
# change targetPath for uploading to
# targetPath's directory / sourcePath's name + ext.
targetPath = pathlib.Path(
str(pathlib.Path(targetPath).parent / fileName) + "." +
compress_method)
sourcePath = archiveFile(sourcePath,
verbose=verbose,
method=compress_method)
isArchived = True
with scp.SCPClient(sshc.get_transport(), progress=progress) as scpc:
# in case Path is PosixPath, casting them to str
scpc.put(str(sourcePath), str(targetPath))
print("\n") # nextline
if compress_method:
unarchiveSSH(targetPath,
sshc,
method=compress_method,
verbose=verbose)
isUnarchived = True
# change targetPath to uploaded raw file
uploadedPath = str(pathlib.Path(targetPath).parent / fileName)
finally: # delete archive files
if 'isArchived' in locals():
with verbosity_context(f"Deleting archive {sourcePath}", verbose):
os.remove(sourcePath)
if 'isUnarchived' in locals():
sftp = sshc.open_sftp()
with verbosity_context(f"Deleting archive {targetPath} via SCP",
verbose):
sftp.remove(str(targetPath))
return uploadedPath if compress_method else targetPath
def deploy(log):
log('Updating web rulebook')
ssh_key_path = find_ssh_key_path()
# Swap out the style sheet inclusion with one that flask will understand
with open(GENERATED_INDEX_PATH, 'r', encoding='utf8') as f:
index_html = f.read()
index_html = index_html.replace(LOCAL_STYLE_SHEET, REMOTE_STYLE_SHEET)
with open(GENERATED_INDEX_PATH, 'w+', encoding='utf8') as f:
f.write(index_html)
ssh_client = None
scp_client = None
try:
ssh_key = paramiko.RSAKey.from_private_key_file(ssh_key_path)
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(hostname=INSTANCE_HOSTNAME,
username=INSTANCE_USER,
pkey=ssh_key,
timeout=TIMEOUT_SEC)
scp_client = scp.SCPClient(ssh_client.get_transport(),
socket_timeout=TIMEOUT_SEC)
scp_client.put(GENERATED_INDEX_PATH,
remote_path='/home/ubuntu/barbs/templates/index.html')
scp_client.put(GENERATED_STYLE_PATH,
remote_path='/home/ubuntu/barbs/static/style.css')
ssh_client.exec_command('pm2 restart barbs')
except scp.SCPException:
log('Failed to upload files')
except paramiko.SSHException:
log('Failed to restart the web server')
else:
log('Updated web rulebook')
finally:
if scp_client:
scp_client.close()
if ssh_client:
ssh_client.close()
# Revert style file
with open(GENERATED_INDEX_PATH, 'r', encoding='utf8') as f:
index_html = f.read()
index_html = index_html.replace(REMOTE_STYLE_SHEET, LOCAL_STYLE_SHEET)
with open(GENERATED_INDEX_PATH, 'w+', encoding='utf8') as f:
f.write(index_html)
def file(self):
try:
client = paramiko.SSHClient()
client.set_missing_host_key_policy(AllowAnythingPolicy())
client.connect(self.ip,
username=self.username,
password=self.password,
port=22,
timeout=self.timeout)
with scp.SCPClient(client.get_transport()) as scps:
scps.put(self.info, self.drectory)
return True
except:
return False
def connect(self, *args, remoteTries=0, **kwargs):
remoteTries += 1
try:
self.ssh.connect(
self._host, username=self._username, password=self._password
)
self.transport = self.ssh.get_transport()
self._scp = _scp.SCPClient(self.transport)
except ConnectionResetError:
if remoteTries > self.retryLimit:
raise ConnectionError(
"Number of retries, %d, greater than retry limit %d, \
cannot connect"%(remoteTries, self.retryLimit))
logging.info('Retrying Connect')
self.connect(remoteTries=remoteTries)
def transfer_from(self, remote_path, local_path, recursive=False):
"""Copy file or directory from default connection.
Args:
remote_path: Location of remote directory or file.
local_path: Location of local directory or file.
recursive (bool): Must be set to True when 'remote_path' is a directory.
"""
_LOGGER.debug("copying %s:%s -> %s", self._host, remote_path, local_path)
_remote_path = str(remote_path)
_local_path = str(local_path)
with self as ssh_client:
ssh_client.connect()
with scp.SCPClient(ssh_client.get_transport()) as scp_client:
scp_client.get(_remote_path, _local_path, recursive=recursive)
def _scp_get(self, hostname=None, username=None, password=None, srcfile=None, destfile=None):
"""SCP Get file
:param hostname: FQDN/IP
:param username: Username
:param password: Password
:param srcfile: Source file
:param destfile: Destination file
"""
self.ssh_client.connect(hostname=hostname,
username=username,
password=password)
scp_client = scp.SCPClient(self.ssh_client.get_transport(), sanitize=lambda x: x)
scp_client.get(srcfile, destfile, recursive=True)
scp_client.close()
def exec_jars(ssh, jars, remote_folder, debug_run_jars):
if not debug_run_jars:
return
scp_client = scp.SCPClient(ssh.get_transport())
scp_client.put(os.path.dirname(os.path.abspath(__file__)) + "/start.sh", remote_folder)
remote_wait_run(ssh, "ps -ef | grep \'java -jar vwvo-\'| grep -v grep | awk \'{print $2}\' | xargs kill -9")
for jar, xxxxx in jars:
jar_name = os.path.basename(jar)
pos = re.search(r'-[0-9]', jar_name)
log_name = jar_name[:pos.start()]
remote_wait_run(ssh, "/bin/sh " + remote_folder + "/start.sh " + remote_folder + " " + os.path.basename(jar) + " " + log_name + ".log")
def upload_scene(scenes_dir, host, remote_path, client):
"""
:param remote_path
"""
temp_dir = "temp/"
remote_files = dict()
remote_files[host['address']] = list()
for scene in find_scenes(scenes_dir):
name = scenes_params.get_scene_name(scene)
remote_scene_dir = os.path.join(remote_path, name)
client.exec_command(str("mkdir -p {}".format(remote_scene_dir)))
arc_name = name + ".tgz"
local_arc_name = temp_dir + arc_name
ensure_dir(local_arc_name)
make_tarfile(local_arc_name, os.path.dirname(scene))
scp_client = scp.SCPClient(client.get_transport())
# or
#client = scp.Client(host=host, user=user, keyfile=keyfile)
# or
#client = scp.Client(host=host, user=user)
#client.use_system_keys()
remote_file = os.path.join(remote_scene_dir, arc_name)
remote_file = remote_file.replace("\\", "/")
remote_blender_file = os.path.join(remote_scene_dir,
os.path.basename(scene))
remote_blender_file = remote_blender_file.replace("\\", "/")
scp_client.put(local_arc_name, remote_file)
print(remote_file)
print(remote_blender_file)
remote_files[host['address']].append(
(remote_file, remote_blender_file))
shutil.rmtree(temp_dir)
return remote_files
def scp_get(self,
remote_path=None,
local_dir=None,
*,
local_path=None,
info=True,
limit_bytes=None,
if_exists=None):
""" 文档参考 self.scp_put
:param local_path: 可以不输入远程remote_path，仅输入local_path来映射、运行
get应该会比put更慢一点，因为get需要使用命令从服务器获得更多参考信息，而put很多文件信息可以在本地直接获得
>> self.scp_get('/home/datasets/doc3D', '/home/dataset')
"""
if local_path:
local_path = XlPath(local_path)
remote_path = self.__remote_dir(local_path, None) / local_path.name
local_dir = local_path.parent
else:
remote_path = self.Path(remote_path)
local_dir = self.__local_dir(remote_path, local_dir)
local_path = local_dir / remote_path.name
# 远程文件不存在，不用运行
if not remote_path.exists_type():
return
if not local_dir.exists():
local_dir.mkdir(parents=True, exist_ok=True)
if info == 0:
progress = ScpProgress(info, limit_bytes=limit_bytes)
elif info == 1:
progress = ScpProgress(info,
desc=f'↓{local_path}',
total=remote_path.size(),
limit_bytes=limit_bytes)
else:
raise NotImplementedError
scp = scplib.SCPClient(self.get_transport(), progress=progress)
try:
self.__scp_base(scp.get, progress, remote_path, local_dir,
local_path, if_exists)
except ScpLimitError:
pass
