def copyFileToComputer(ftp_client: paramiko.SFTPClient, remoteFolder: str,
resFolder: str, filename: str) -> pref.Error:
'''
Copies a file to a remote computer ftp_client
@param ftp_client:parmiko.STFPClient the ssh ftp connection what the file will be transferred over
@param remoteFolder:str
'''
err = pref.Success
#creates folder if it doesn't exist
try:
ftp_client.mkdir(remoteFolder, mode=0o777)
ftp_client.chmod(remoteFolder, mode=0o777)
except: #This is happen if the folder already exists .
pass
#copies file
try:
ftp_client.put("{}{}".format(resFolder, filename),
"{}{}".format(remoteFolder, filename))
except Exception as e:
print(e)
#couldn't find script on sever.
err = pref.getError(pref.ERROR_FILE_NOT_FOUND,
args=(resFolder + filename))
logger.error(err)
try:
ftp_client.chmod("{}{}".format(remoteFolder, filename), 0o777)
except:
pass
if (ftp_client):
ftp_client.close()
return err
async def push_file(ftp: paramiko.SFTPClient, ftp_path: str, local_path: str):
"""Pushes file to remote if local is more up to date, else raises error."""
if not compare_files(ftp, ftp_path, local_path):
print(f'Pushing {local_path} to {ftp_path}')
ftp.put(local_path, ftp_path)
else:
raise Exception("Error! FTP file was updated.")
def _send_updates(files: Union[LogFiles, PidFiles, SeqFiles],
sftp: paramiko.SFTPClient) -> None:
"""Sends update files to the remote FTP server.
This function is used for logfiles, pid update files and seq update files
(*.log, *.pid, *.seq) which can be transferred directly with no further
processing needed.
Args:
files: list of CurationFiles objects (LogFiles, PidFiles, SeqFiles).
sftp: paramiko SFTPClient object.
"""
if files:
for f in files:
localpath = str(f)
remotepath = str(PurePosixPath(str(files.remote_dir), f.name))
sftp.put(localpath=localpath, remotepath=remotepath)
click.echo(f"Sent {f.name} to {files.remote_dir}")
click.secho(f"Sent all {str(files)} to {REMOTE_HOST_NAME}.",
fg="green")
click.echo("---")
else:
click.echo(f"No {str(files)} to send.")
click.echo("---")
def _put_recursive(self, path: Path, dest: SFTPPath,
client: SFTPClient) -> None:
''' copy local files to the remote host '''
if path.is_file():
client.put(str(path), str(dest))
else:
dest.mkdir()
for f in path.iterdir():
new_dest = dest.joinpath(f.name)
self._put_recursive(f, new_dest, client)
def upload_file(sftp: SFTPClient, local_path, remote_path):
"""Copy local file to remote. if remote dir is not exists should create it.
:param sftp:
:param local_path: a file
:param remote_path: a file path, is not exists and included file name
:return:
"""
p = Path(remote_path).parent.as_posix()
makedirs(sftp, p)
sftp.put(local_path, remote_path)
def sftp_sync(local_dir: str, sftp: paramiko.SFTPClient, remote_dir: str):
"""Synchronizes local_dir towards remote_dir (i.e. files in remote_dir
not in local_dir won't be copied to local_dir)"""
info_ = ColorLogger.getLogger('blue')
warn_ = ColorLogger.getLogger('yellow')
info = info_.log
warn = warn_.log
o1 = LocalFileOps()
o2 = SftpFileOps(sftp)
# ensure both base paths exist and are actual folders
o1.mkpath(local_dir)
o2.mkpath(remote_dir)
# stack of (local_relpath_including_base_dir, remote_relpath_including_remote_dir)
S = [(local_dir, remote_dir)]
while S:
relpath, remote_relpath = S.pop()
efd = o1.existsIsFileIsDir(relpath)
if efd == 1:
p1 = relpath
p2 = remote_relpath
o2.mkpath(os.path.dirname(p2))
# upload only if modification time of p1 > modification time of p2
try:
tmp = o2.stat(p2)
remote_times = (int(tmp.st_atime), int(tmp.st_mtime))
except:
# assume remote file not existing
remote_times = (0, 0)
tmp = o1.stat(p1)
local_times = (int(tmp.st_atime), int(tmp.st_mtime))
local_mtime = local_times[1]
remote_mtime = remote_times[1]
if local_mtime > remote_mtime:
warn(
f"Updating remote path {p2} with mtime {remote_mtime} from local path {p1} with mtime {local_mtime}"
)
sftp.put(p1, p2)
sftp.utime(p2, local_times)
else:
info(
f"Won't update remote path {p2} with mtime {remote_mtime}, equal or more recent than local path {p1} with mtime {local_mtime}"
)
elif efd == 2:
S.extend(((pathConcat(relpath, filename, '/')),
(pathConcat(remote_relpath, filename, '/')))
for filename in o1.listdir(relpath))
def ssh_cpdir(sftp: SFTPClient,
local_dir: str,
remote_dir: str,
cur: str = ''):
"""
どうやら paramiko には再帰コピー機能がないようなので地道にコピー
"""
local_cur_dir = path.join(local_dir, cur)
remote_cur_dir = remote_dir if '' == cur else remote_dir + '/' + cur
sftp.mkdir(remote_cur_dir, 0o755)
for child_name in os.listdir(local_cur_dir):
child_path = path.join(local_cur_dir, child_name)
if path.isdir(child_path):
ssh_cpdir(sftp, local_dir, remote_dir, cur + '/' + child_name)
else:
sftp.put(child_path, remote_cur_dir + '/' + child_name)
def _send_new_entries(
files: List[Union[TrEMBLFiles, NewFiles, PepFiles, SubFiles]],
sftp: paramiko.SFTPClient,
) -> None:
"""Sends new entries to the remote FTP server.
"New entries" includes curated TrEMBL entries plus other new entries such
as direct submissions. These files must be concatenated into a single file
named "allnew" with a date stamp appended to the filename before transfer.
A temporary directory is used to make the "allnew" file.
Args:
trembl_files: TrEMBLFiles object
new_files: NewFiles object
sftp: paramiko SFTPClient object
"""
if any(files):
date_string = datetime.date.today().strftime("%Y%m%d")
# Remote directory is the same for all new files so just look at the
# first object in the list
remote_dir = files[0].remote_dir
with tempfile.TemporaryDirectory() as tmp:
# Concatenate new, sub and pep files to allnew
allnew = Path(tmp, f"allnew_{date_string}.swp")
for f in files:
if f:
f.write_files(allnew, mode="a")
for file in f:
click.echo(
f"Adding file {file.name} to concatenated file {allnew.name}"
)
# Transfer allnew file to remote server
localpath = str(allnew)
remotepath = str(PurePosixPath(str(remote_dir), allnew.name))
sftp.put(localpath=localpath, remotepath=remotepath)
click.echo(f"Sent {allnew.name} to {remote_dir}")
click.secho(f"Sent all new files to {REMOTE_HOST_NAME}.",
fg="green")
click.echo("---")
else:
click.echo("No new files to send.")
click.echo("---")
def upload_file(sftp: paramiko.SFTPClient, local_path,
remote_path) -> paramiko.SFTPClient:
local_all = files_of_dir(local_path)
for file in local_all:
real_remote = os.path.join(
remote_path,
file.replace(os.path.dirname(local_path),
'').lstrip('\\')).replace("\\", "/")
remote_p = os.path.dirname(real_remote)
# 远程创建文件夹
try:
stat = sftp.stat(remote_p)
except FileNotFoundError:
sftp.mkdir(remote_p)
stat = sftp.stat(remote_p)
if S_ISDIR(stat.st_mode):
# sftp 的 put命令，需要在 remotepath 中，指定文件的名字
# f.put(r'D:\Python.zip', r'/10.130.218.69 sftp (10.130.218.69)/nginx/home/nginx/webapps/Python.zip')
sftp.put(file, real_remote)
return sftp
def send(self, sftp: SFTPClient) -> None:
for file in os.listdir(self.local_path):
if self.send_iteration % self.update_history_every != 0 and (
file.startswith("history") and file.endswith(".json")):
self.log(level=DEBUG,
msg="Skipping file [{0}]; [{1}/{2}]...".format(
file, self.send_iteration,
self.update_history_every))
continue
local_full_path = os.path.join(self.local_path, file)
remote_full_path = os.path.join(self.remote_path, file)
try:
self.log(
level=DEBUG,
msg="Copying local file[{}] to remote file [{}]".format(
local_full_path, remote_full_path))
sftp.put(local_full_path, remote_full_path)
except IOError:
self.log(
level=ERROR,
msg=
"IOError trying to copy local file[{}] to remote file [{}]"
.format(local_full_path, remote_full_path))
self.send_iteration += 1
def copy_from_local_to_remote(sftp: SFTPClient, local_path, remote_path):
p = Path(remote_path).parent.as_posix()
makedirs(sftp, p)
sftp.put(local_path, remote_path)
def sftp_upload(sftp: SFTPClient, filename: str, remote_filename: str):
file_size = Path(filename).stat().st_size
progress = ProgressBar(max_value=file_size)
sftp.put(filename, remote_filename, callback=lambda completed, total: progress.update(completed))
progress.finish()
