def test_sync_fs_fs(self):
local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "local"))
remote = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))
opts = {"dry_run": False, "verbose": 3}
s = BiDirSynchronizer(local, remote, opts)
s.run()
stats = s.get_stats()
# pprint(stats)
self.assertEqual(stats["local_dirs"], 2)
self.assertEqual(
stats["local_files"],
4) # currently files are not counted, when inside a *new* folder
self.assertEqual(stats["remote_dirs"], 0)
self.assertEqual(stats["remote_files"], 0)
self.assertEqual(stats["files_written"], 6)
self.assertEqual(stats["dirs_created"], 2)
self.assertEqual(stats["bytes_written"], 16403)
# file times are preserved
self.assertEqual(_get_test_file_date("local/file1.txt"),
STAMP_20140101_120000)
self.assertEqual(_get_test_file_date("remote/file1.txt"),
STAMP_20140101_120000)
# Again: nothing to do
s = BiDirSynchronizer(local, remote, opts)
s.run()
stats = s.get_stats()
# pprint(stats)
self.assertEqual(stats["local_dirs"], 2)
self.assertEqual(stats["local_files"], 6)
self.assertEqual(stats["remote_dirs"], 2)
self.assertEqual(stats["remote_files"], 6)
self.assertEqual(stats["files_created"], 0)
self.assertEqual(stats["files_deleted"], 0)
self.assertEqual(stats["files_written"], 0)
self.assertEqual(stats["dirs_created"], 0)
self.assertEqual(stats["bytes_written"], 0)
# Modify remote and/or remote
_touch_test_file("local/file1.txt")
_touch_test_file("remote/file2.txt")
# file3.txt will cause a conflict:
_touch_test_file("local/file3.txt")
dt = datetime.datetime.now() - datetime.timedelta(seconds=10)
_touch_test_file("remote/file3.txt", dt=dt)
s = BiDirSynchronizer(local, remote, opts)
s.run()
stats = s.get_stats()
# pprint(stats)
self.assertEqual(stats["entries_seen"], 18)
self.assertEqual(stats["entries_touched"], 2)
self.assertEqual(stats["files_created"], 0)
self.assertEqual(stats["files_deleted"], 0)
self.assertEqual(stats["files_written"], 2)
self.assertEqual(stats["dirs_created"], 0)
self.assertEqual(stats["download_files_written"], 1)
self.assertEqual(stats["upload_files_written"], 1)
self.assertEqual(stats["conflict_files"], 1)
self.assertEqual(stats["bytes_written"], 6)
def test_download_fs_fs(self):
# Download files from local to remote (which is empty)
local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))
remote = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "local"))
opts = {"force": False, "delete": False, "dry_run": False}
s = DownloadSynchronizer(local, remote, opts)
s.run()
stats = s.get_stats()
# pprint(stats)
self.assertEqual(stats["local_dirs"], 0)
self.assertEqual(stats["local_files"], 0)
self.assertEqual(stats["remote_dirs"], 2)
self.assertEqual(stats["remote_files"], 4) # currently files are not counted, when inside a *new* folder
self.assertEqual(stats["files_written"], 6)
self.assertEqual(stats["dirs_created"], 2)
self.assertEqual(stats["bytes_written"], 16403)
# Again: nothing to do
s = DownloadSynchronizer(local, remote, opts)
s.run()
stats = s.get_stats()
# pprint(stats)
self.assertEqual(stats["local_dirs"], 2)
self.assertEqual(stats["local_files"], 6)
self.assertEqual(stats["remote_dirs"], 2)
self.assertEqual(stats["remote_files"], 6)
self.assertEqual(stats["files_written"], 0)
self.assertEqual(stats["dirs_created"], 0)
self.assertEqual(stats["bytes_written"], 0)
# file times are preserved
self.assertEqual(_get_test_file_date("local/file1.txt"), STAMP_20140101_120000)
self.assertEqual(_get_test_file_date("remote/file1.txt"), STAMP_20140101_120000)
def _sync_test_folders(options):
"""Run bi-dir sync with fresh objects."""
local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "local"))
remote = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))
opts = {"dry_run": False, "verbose": 3}
if options:
opts.update(options)
s = BiDirSynchronizer(local, remote, opts)
s.run()
return s.get_stats()
def _transfer_files(self, count, size):
temp1_path = os.path.join(PYFTPSYNC_TEST_FOLDER, "local")
empty_folder(temp1_path) # remove standard test files
local = FsTarget(temp1_path)
remote = self.remote
for i in range(count):
write_test_file("local/file_{}.txt".format(i), size=size)
# Upload all of temp/local to remote
opts = {"force": False, "delete": False, "verbose": 3}
s = UploadSynchronizer(local, remote, opts)
s.run()
stats = s.get_stats()
# pprint(stats)
self.assertEqual(stats["files_written"], count)
self.assertEqual(stats["bytes_written"], count * size)
# pprint(stats)
print(
"Upload {} x {} bytes took {}: {}".format(
count, size, stats["upload_write_time"],
stats["upload_rate_str"]),
file=sys.stderr,
)
# Download all of remote to temp/remote
local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))
opts = {"force": False, "delete": True, "verbose": 3}
s = DownloadSynchronizer(local, remote, opts)
s.run()
stats = s.get_stats()
# pprint(stats)
self.assertEqual(stats["files_written"], count)
self.assertEqual(stats["bytes_written"], count * size)
# pprint(stats)
print(
"Download {} x {} bytes took {}: {}".format(
count, size, stats["download_write_time"],
stats["download_rate_str"]),
file=sys.stderr,
)
def run(self):
from ftpsync.targets import FsTarget, UploadSynchronizer
from ftpsync.ftp_target import FtpTarget
ftp_configuration = self.deployment['FTP']
local = FsTarget(os.path.join(SETUP_DIR, 'keys_of_peace'))
remote = FtpTarget(ftp_configuration['PATH'],
self.deployment['SERVER'],
ftp_configuration['USER'],
ftp_configuration['PASSWORD'])
s = UploadSynchronizer(
local, remote, {
'delete_unmatched': True,
'dry_run': False,
'force': False,
'omit': '*.sqlite3,*.pyc',
})
s.run()
import paramiko
ssh_configuration = self.deployment['SSH']
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(self.deployment['SERVER'],
username=ssh_configuration['USER'],
password=ssh_configuration['PASSWORD'])
_in, _out, _err = client.exec_command(';'.join(
ssh_configuration['COMMANDS']))
_in.write(ssh_configuration['IN'])
client.close()
def copy_to_ftp(source_folder, dest_host):
logging.info('Copying %s to %s' % (source_folder, dest_host))
local = FsTarget(source_folder)
remote = FtpTarget(settings.DEFAULT_FTP_PATH, dest_host,
settings.DEFAULT_FTP_USER, settings.DEFAULT_FTP_PASS)
s = UploadSynchronizer(local, remote, settings.DEFAULT_FTP_SYNC_OPTS)
s.run()
def sync_gridsome() -> None:
"""
指定のローカルとリモートディレクトリを同期する
"""
cfg = configparser.ConfigParser()
cfg.read("config.ini")
# ローカルとリモートの設定
local = FsTarget(cfg["PATH"]["LOCAL"])
user = cfg["FTPS"]["USER"]
passwd = cfg["FTPS"]["PASSWORD"]
remote = FtpTarget(
cfg["PATH"]["REMOTE"], # リモートディレクトリパス
cfg["FTPS"]["SERVER"], # FTPサーバ
username=user,
password=passwd,
tls=True, # FTPS有効
)
# オプション設定
# ローカル優先／--deleteオプション有効／指定ディレクトリは同期除外
# opts = {"resolve": "local", "delete": True, "force": True}
opts = {"resolve": "local"}
# 同期の実行
sync = BiDirSynchronizer(local, remote, opts)
sync.run()
def _make_remote_target(cls):
"""Return the remote target instance, depending on `use_ftp_target`."""
if cls.use_ftp_target:
check_ftp_test_connection(PYFTPSYNC_TEST_FOLDER, PYFTPSYNC_TEST_FTP_URL)
remote = make_target(PYFTPSYNC_TEST_FTP_URL)
else:
remote = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))
return remote
def test_upload_fs_fs(self):
local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "local"))
remote = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))
opts = {"force": False, "delete": False, "verbose": self.verbose}
s = UploadSynchronizer(local, remote, opts)
s.run()
stats = s.get_stats()
# pprint(stats)
self.assertEqual(stats["local_dirs"], 2)
# currently files are not counted, when inside a *new* folder:
self.assertEqual(stats["local_files"], 4)
self.assertEqual(stats["remote_dirs"], 0)
self.assertEqual(stats["remote_files"], 0)
self.assertEqual(stats["files_written"], 6)
self.assertEqual(stats["dirs_created"], 2)
self.assertEqual(stats["bytes_written"], 16403)
# file times are preserved
self.assertEqual(get_test_file_date("local/file1.txt"), STAMP_20140101_120000)
self.assertEqual(get_test_file_date("remote/file1.txt"), STAMP_20140101_120000)
def upload_dir(local, remote, dry_run=False, opts=None):
env = environment()
if 'ssh' in env:
ssh = env['ssh']
rsync_opts = [
'--recursive', '--archive', '--compress', '--itemize-changes'
]
if dry_run:
rsync_opts.append('--dry-run')
if state['verbose']:
rsync_opts.append('--verbose')
# trailing slash is important, see rsync documentation
src = local + '/'
abs_remote = os.path.join(ssh['document_root'], remote)
dest = '{}@{}:{}'.format(ssh['user'], ssh['host'], abs_remote)
args = rsync_opts + ([] if opts is None else opts) + [src, dest]
# TODO rsync still prompts for the password
with shell_env(RSYNC_PASSWORD=ssh['password']):
fab.local('rsync {}'.format(' '.join(args)))
else:
ftp = env['ftp']
host = ftp_host(env)
extra_opts = {'ftp_debug': 1 if state['verbose'] else 0}
url = urlparse(ftp['url'])
if url.scheme != 'ftp':
fab.warn('non-ftp url scheme, may not be supported')
abs_remote = os.path.join(url.path, remote)
if not host.path.exists(abs_remote):
fab.warn('remote path is missing: {}'.format(abs_remote))
if console.confirm('create missing directories?'):
host.makedirs(abs_remote)
local_target = FsTarget(local, extra_opts=extra_opts)
remote_target = FtpTarget(path=abs_remote,
host=url.netloc,
username=ftp['user'],
password=ftp['password'],
extra_opts=extra_opts)
opts = {
'force': False,
# TODO
'delete_unmatched': False,
'verbose': 3,
'execute': True,
'dry_run': dry_run
}
s = UploadSynchronizer(local_target, remote_target, opts)
try:
s.run()
except KeyboardInterrupt:
fab.warn('aborted by user')
finally:
s.local.close()
s.remote.close()
stats = s.get_stats()
pprint(stats)
def test_sync_conflicts(self):
local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "local"))
remote = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))
opts = {
"dry_run": False,
"verbose": self.verbose
} # , "resolve": "ask"}
# Copy local -> remote
s = BiDirSynchronizer(local, remote, opts)
s.run()
stats = s.get_stats()
self.assertEqual(stats["files_written"], 6)
self.assertEqual(stats["dirs_created"], 2)
# Modify local and remote
# conflict 1: local is newer
dt = datetime.datetime.utcnow()
touch_test_file("local/file1.txt", dt)
dt = datetime.datetime.utcnow() - datetime.timedelta(seconds=10)
touch_test_file("remote/file1.txt", dt=dt)
# path = os.path.join(PYFTPSYNC_TEST_FOLDER, "remote/file1.txt")
# stamp = time.time() - 10
# os.utime(path, (stamp, stamp))
# conflict 2: remote is newer
touch_test_file("remote/file2.txt")
dt = datetime.datetime.utcnow() - datetime.timedelta(seconds=10)
touch_test_file("local/file2.txt", dt=dt)
s = BiDirSynchronizer(local, remote, opts)
s.run()
stats = s.get_stats()
# pprint(stats)
self.assertEqual(stats["entries_seen"], 18)
self.assertEqual(stats["entries_touched"], 0)
self.assertEqual(stats["bytes_written"], 0)
self.assertEqual(stats["conflict_files"], 2)
def synchronize(user, password):
print "syncing..."
try:
local = FsTarget(DEPLOY_FOLDER)
remote = FtpTarget(FTP_TARGET, FTP_HOST, user, password, tls=True)
opts = {"resolve": "skip", "verbose": 1, "dry_run": False}
s = BiDirSynchronizer(local, remote, opts)
s.run()
print "sync complete..."
except Exception as e:
print type(e)
print e.args
print e
def _sync_test_folders(cls, synchronizer_class, options, remote=None):
"""Run synchronizer with fresh objects and custom options."""
local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "local"))
if remote is None:
remote = cls._make_remote_target()
opts = {"verbose": 1}
if options:
opts.update(options)
s = synchronizer_class(local, remote, opts)
s.run()
s.close()
return s.get_stats()
def download_prolog():
localpath = path_config.local_ftp_path # 本地保存的ftp目录
os.makedirs(localpath, 0o777) # 创建本地日志文件夹
local = FsTarget(localpath)
remote = FtpTarget(path_config.ftp_remotepath,
path_config.ftp_server,
username=path_config.ftp_username,
password=path_config.ftp_password)
opts = {"resolve": "skip", "verbose": 1}
try:
s = DownloadSynchronizer(local, remote, opts) # 下载ftp日志文件夹
s.run()
except SyntaxError:
pass
def test_sync_fs_ftp(self):
local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "local"))
remote = self.remote
# Upload all of temp/local to remote
opts = {"force": False, "delete": True, "verbose": 3}
s = UploadSynchronizer(local, remote, opts)
s.run()
stats = s.get_stats()
# pprint(stats)
self.assertEqual(stats["local_dirs"], 2)
# currently files are not counted, when inside a *new* folder:
self.assertEqual(stats["local_files"], 4)
self.assertEqual(stats["remote_dirs"], 0)
self.assertEqual(stats["remote_files"], 0)
self.assertEqual(stats["files_written"], 6)
self.assertEqual(stats["dirs_created"], 2)
self.assertEqual(stats["bytes_written"], 16403)
# Change one file and upload again
touch_test_file("local/file1.txt")
opts = {"force": False, "delete": True, "verbose": 3}
s = UploadSynchronizer(local, remote, opts)
s.run()
stats = s.get_stats()
# pprint(stats)
# assert False
self.assertEqual(stats["entries_seen"], 18) # ???
self.assertEqual(stats["entries_touched"], 1)
self.assertEqual(stats["files_created"], 0)
self.assertEqual(stats["files_deleted"], 0)
self.assertEqual(stats["files_written"], 1)
self.assertEqual(stats["dirs_created"], 0)
self.assertEqual(stats["download_files_written"], 0)
self.assertEqual(stats["upload_files_written"], 1)
self.assertEqual(stats["conflict_files"], 0)
self.assertEqual(stats["bytes_written"], 3)
# Download all from remote to temp/remote
local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))
opts = {"force": False, "delete": True, "verbose": 3}
s = DownloadSynchronizer(local, remote, opts)
s.run()
stats = s.get_stats()
# pprint(stats)
self.assertEqual(stats["entries_seen"], 8)
self.assertEqual(stats["entries_touched"], 8)
# self.assertEqual(stats["files_created"], 6)
self.assertEqual(stats["files_deleted"], 0)
self.assertEqual(stats["files_written"], 6)
self.assertEqual(stats["dirs_created"], 2)
self.assertEqual(stats["download_files_written"], 6)
self.assertEqual(stats["upload_files_written"], 0)
self.assertEqual(stats["conflict_files"], 0)
self.assertEqual(stats["bytes_written"], 16403)
# Original file times are preserved, even when retrieved from FTP
self.assertNotEqual(get_test_file_date("local/file1.txt"),
STAMP_20140101_120000)
self.assertEqual(
get_test_file_date("local/file1.txt"),
get_test_file_date("local//file1.txt"),
)
self.assertEqual(get_test_file_date("local/file2.txt"),
STAMP_20140101_120000)
self.assertEqual(get_test_file_date("remote//file2.txt"),
STAMP_20140101_120000)
# Synchronize temp/local <=> remote : nothing to do
local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "local"))
opts = {"verbose": 3}
s = BiDirSynchronizer(local, remote, opts)
s.run()
stats = s.get_stats()
pprint(stats)
self.assertEqual(stats["entries_touched"], 0)
self.assertEqual(stats["conflict_files"], 0)
self.assertEqual(stats["bytes_written"], 0)
# Synchronize temp/remote <=> remote : nothing to do
local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))
opts = {"verbose": 3}
s = BiDirSynchronizer(local, remote, opts)
s.run()
stats = s.get_stats()
pprint(stats)
self.assertEqual(stats["entries_touched"], 0)
self.assertEqual(stats["conflict_files"], 0)
self.assertEqual(stats["bytes_written"], 0)
# ftpsync library shuld be installed
from ftpsync.targets import FsTarget
from ftpsync.ftp_target import FtpTarget
from ftpsync.syncronizers import UploadSynchronizer
local = FsTarget('C:\\foo')
user = '******'
passwd = 'pwd'
remote = FtpTarget("/foo", "svr.com", username=user, password=passwd)
opts = {"force": False, "delete_unmatched": True, "verbose": 3}
s = UploadSynchronizer(local, remote, opts)
s.run()
#!/usr/bin/env python3.6
from ftpsync.targets import FsTarget
from ftpsync.ftp_target import FtpTarget
from ftpsync.synchronizers import UploadSynchronizer
from ftpsetupfile import *
print('\n\nOpening an FTP connection for the upload...\n')
local = FsTarget(LOCAL_DIR)
remote = FtpTarget(REMOTE_DIR,
REMOTE_HOST,
username=FTP_USERNAME,
password=FTP_PASSWD)
opts = {"force": True, "delete_unmatched": True, "verbose": 2}
s = UploadSynchronizer(local, remote, opts)
s.run()
print('\n\nUpload complete!')
from ftpsync.targets import FsTarget
from ftpsync.ftp_target import FtpTarget
from ftpsync.synchronizers import UploadSynchronizer
local = FsTarget('C:\\temp')
user = '******'
passwd = 'pwd'
remote = FtpTarget("/foo", "ftp.svr.com", username=user, password=passwd)
opts = {"force": True, "delete_unmatched": True, "verbose": 3}
s = UploadSynchronizer(local, remote, opts)
s.run()
