def test_put_noverify(self, monkeypatch: MonkeyPatch):
scput = SplitCopyPut()
scput.scs = MockSplitCopyShared()
scput.progress = MockProgress()
def validate_remote_path_put():
pass
def check_target_exists():
return True
def delete_target_remote():
pass
def determine_local_filesize():
return 1000000
def split_file_local(*args):
pass
def get_chunk_info():
return [["chunk0", 1000], ["chunk1", 1000]]
def put_files(*args):
pass
def join_files_remote(*args):
pass
def compare_file_sizes(*args):
pass
def inc_percentage():
for n in range(90, 101):
time.sleep(0.1)
scput.progress.totals["percent_done"] = n
scput.noverify = True
monkeypatch.setattr(scput, "validate_remote_path_put",
validate_remote_path_put)
monkeypatch.setattr(scput, "check_target_exists", check_target_exists)
monkeypatch.setattr(scput, "delete_target_remote",
delete_target_remote)
monkeypatch.setattr(scput, "determine_local_filesize",
determine_local_filesize)
monkeypatch.setattr(scput, "split_file_local", split_file_local)
monkeypatch.setattr(scput, "get_chunk_info", get_chunk_info)
monkeypatch.setattr(scput, "put_files", put_files)
monkeypatch.setattr(scput, "join_files_remote", join_files_remote)
monkeypatch.setattr(scput, "compare_file_sizes", compare_file_sizes)
thread = Thread(
name="inc_percentage_done",
target=inc_percentage,
)
thread.start()
result = scput.put()
thread.join()
assert isinstance(result[0], datetime.datetime) and isinstance(
result[1], datetime.datetime)
def test_put_fail(self, monkeypatch: MonkeyPatch):
scput = SplitCopyPut()
scput.scs = MockSplitCopyShared()
scput.progress = MockProgress()
def validate_remote_path_put():
pass
def check_target_exists():
return True
def delete_target_remote():
pass
def determine_local_filesize():
return 1000000
def local_sha_put():
return "sha384sum", 384, "abcdef0123456789"
def split_file_local(*args):
pass
def get_chunk_info():
return [["chunk0", 1000], ["chunk1", 1000]]
def put_files(*args):
raise TransferError
monkeypatch.setattr(scput, "validate_remote_path_put",
validate_remote_path_put)
monkeypatch.setattr(scput, "check_target_exists", check_target_exists)
monkeypatch.setattr(scput, "delete_target_remote",
delete_target_remote)
monkeypatch.setattr(scput, "determine_local_filesize",
determine_local_filesize)
monkeypatch.setattr(scput, "local_sha_put", local_sha_put)
monkeypatch.setattr(scput, "split_file_local", split_file_local)
monkeypatch.setattr(scput, "get_chunk_info", get_chunk_info)
monkeypatch.setattr(scput, "put_files", put_files)
with raises(SystemExit):
scput.put()
def main():
"""body of script"""
def handlesigint(sigint, stack):
raise SystemExit
signal.signal(signal.SIGINT, handlesigint)
start_time = datetime.datetime.now()
parser = argparse.ArgumentParser()
parser.add_argument(
"source", help="either <path> or user@<host>:<path> or <host>:<path>"
)
parser.add_argument(
"target", help="either <path> or user@<host>:<path> or <host>:<path>"
)
parser.add_argument(
"--pwd", nargs=1, help="password to authenticate on remote host"
)
parser.add_argument(
"--ssh_key",
nargs=1,
help="path to ssh private key (only if in non-default location)",
)
parser.add_argument(
"--scp", action="store_true", help="use scp to copy files instead of ftp"
)
parser.add_argument(
"--noverify",
action="store_true",
help="skip sha hash comparison of src and dst file",
)
parser.add_argument(
"--split_timeout",
nargs=1,
help="time to wait for remote file split operation to complete, default 120s",
)
parser.add_argument(
"--ssh_port",
nargs=1,
help="ssh port number to connect to",
)
parser.add_argument("--nocurses", action="store_true", help="disable curses output")
parser.add_argument("--log", nargs=1, help="log level, eg DEBUG")
args = parser.parse_args()
if not args.log:
loglevel = "WARNING"
else:
loglevel = args.log[0]
numeric_level = getattr(logging, loglevel.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError(f"Invalid log level: {loglevel}")
logging.basicConfig(
format="%(asctime)s %(name)s %(lineno)s %(funcName)s %(levelname)s:%(message)s",
level=numeric_level,
)
user = None
host = None
passwd = None
ssh_key = None
ssh_port = 22
remote_dir = None
remote_file = None
remote_path = None
local_dir = None
local_path = None
copy_proto = None
get = False
noverify = args.noverify
source = args.source
target = args.target
use_curses = True
if args.nocurses:
use_curses = False
if os.path.isfile(source):
local_path = os.path.abspath(os.path.expanduser(source))
try:
with open(local_path, "rb"):
pass
except PermissionError:
raise SystemExit(
f"source file {local_path} exists but is not readable - cannot proceed"
)
local_file = os.path.basename(local_path)
local_dir = os.path.dirname(local_path)
elif re.search(r".*:", source):
if re.search(r"@", source):
user = source.split("@")[0]
host = source.split("@")[1]
host = host.split(":")[0]
else:
user = getpass.getuser()
host = source.split(":")[0]
remote_path = source.split(":")[1]
remote_file = os.path.basename(remote_path)
remote_dir = os.path.dirname(remote_path)
if remote_dir == "" or remote_dir == ".":
remote_dir = "~"
remote_path = f"{remote_dir}/{remote_file}"
if not remote_file:
raise SystemExit("src path doesn't specify a file name")
get = True
else:
raise SystemExit(
"specified source is not a valid path to a local "
"file, or is not in the format <user>@<host>:<path> "
"or <host>:<path>"
)
if os.path.isdir(target):
local_dir = os.path.abspath(os.path.expanduser(target))
local_file = remote_file
elif os.path.isdir(os.path.dirname(target)):
# we've been passed in a filename, may not exist yet
local_dir = os.path.dirname(os.path.abspath(os.path.expanduser(target)))
if os.path.basename(target) != remote_file:
# have to honour the change of name
local_file = os.path.basename(target)
else:
local_file = remote_file
elif re.search(r".*:", target):
if re.search(r"@", target):
user = target.split("@")[0]
host = target.split("@")[1]
host = host.split(":")[0]
else:
user = getpass.getuser()
host = target.split(":")[0]
remote_path = target.split(":")[1]
if remote_path == "":
remote_dir = "~"
remote_file = local_file
remote_path = f"{remote_dir}/{remote_file}"
elif os.path.dirname(remote_path) == "":
remote_dir = "~"
remote_file = remote_path
remote_path = f"{remote_dir}/{remote_file}"
else:
raise SystemExit(
"specified target is not a valid path to a local "
"file or directory, or is not in the format <user>@<host>:<path> "
"or <host>:<path>"
)
try:
host = gethostbyname(host)
except (gaierror, herror):
raise SystemExit("hostname resolution failed")
if args.pwd:
passwd = args.pwd[0]
if not args.scp:
copy_proto = "ftp"
else:
copy_proto = "scp"
if args.ssh_key is not None:
ssh_key = os.path.abspath(args.ssh_key[0])
if not os.path.isfile(ssh_key):
raise SystemExit("specified ssh key not found")
if args.ssh_port is not None:
try:
ssh_port = int(args.ssh_port[0])
except ValueError:
raise SystemExit("ssh_port must be an integer")
split_timeout = 120
if args.split_timeout is not None:
try:
split_timeout = int(args.split_timeout[0])
except ValueError:
raise SystemExit("split_timeout must be an integer")
kwargs = {
"user": user,
"host": host,
"passwd": passwd,
"ssh_key": ssh_key,
"ssh_port": ssh_port,
"remote_dir": remote_dir,
"remote_file": remote_file,
"remote_path": remote_path,
"local_dir": local_dir,
"local_file": local_file,
"local_path": local_path,
"copy_proto": copy_proto,
"get": get,
"noverify": noverify,
"split_timeout": split_timeout,
"use_curses": use_curses,
}
logger.info(kwargs)
if get:
splitcopyget = SplitCopyGet(**kwargs)
loop_start, loop_end = splitcopyget.get()
else:
splitcopyput = SplitCopyPut(**kwargs)
loop_start, loop_end = splitcopyput.put()
# and we are done...
end_time = datetime.datetime.now()
time_delta = end_time - start_time
transfer_delta = loop_end - loop_start
print(f"data transfer = {transfer_delta}\ntotal runtime = {time_delta}")
