def test_put_noverify(self, monkeypatch: MonkeyPatch): scput = SplitCopyPut() scput.scs = MockSplitCopyShared() scput.progress = MockProgress() def validate_remote_path_put(): pass def check_target_exists(): return True def delete_target_remote(): pass def determine_local_filesize(): return 1000000 def split_file_local(*args): pass def get_chunk_info(): return [["chunk0", 1000], ["chunk1", 1000]] def put_files(*args): pass def join_files_remote(*args): pass def compare_file_sizes(*args): pass def inc_percentage(): for n in range(90, 101): time.sleep(0.1) scput.progress.totals["percent_done"] = n scput.noverify = True monkeypatch.setattr(scput, "validate_remote_path_put", validate_remote_path_put) monkeypatch.setattr(scput, "check_target_exists", check_target_exists) monkeypatch.setattr(scput, "delete_target_remote", delete_target_remote) monkeypatch.setattr(scput, "determine_local_filesize", determine_local_filesize) monkeypatch.setattr(scput, "split_file_local", split_file_local) monkeypatch.setattr(scput, "get_chunk_info", get_chunk_info) monkeypatch.setattr(scput, "put_files", put_files) monkeypatch.setattr(scput, "join_files_remote", join_files_remote) monkeypatch.setattr(scput, "compare_file_sizes", compare_file_sizes) thread = Thread( name="inc_percentage_done", target=inc_percentage, ) thread.start() result = scput.put() thread.join() assert isinstance(result[0], datetime.datetime) and isinstance( result[1], datetime.datetime)
def test_put_fail(self, monkeypatch: MonkeyPatch): scput = SplitCopyPut() scput.scs = MockSplitCopyShared() scput.progress = MockProgress() def validate_remote_path_put(): pass def check_target_exists(): return True def delete_target_remote(): pass def determine_local_filesize(): return 1000000 def local_sha_put(): return "sha384sum", 384, "abcdef0123456789" def split_file_local(*args): pass def get_chunk_info(): return [["chunk0", 1000], ["chunk1", 1000]] def put_files(*args): raise TransferError monkeypatch.setattr(scput, "validate_remote_path_put", validate_remote_path_put) monkeypatch.setattr(scput, "check_target_exists", check_target_exists) monkeypatch.setattr(scput, "delete_target_remote", delete_target_remote) monkeypatch.setattr(scput, "determine_local_filesize", determine_local_filesize) monkeypatch.setattr(scput, "local_sha_put", local_sha_put) monkeypatch.setattr(scput, "split_file_local", split_file_local) monkeypatch.setattr(scput, "get_chunk_info", get_chunk_info) monkeypatch.setattr(scput, "put_files", put_files) with raises(SystemExit): scput.put()
def main(): """body of script""" def handlesigint(sigint, stack): raise SystemExit signal.signal(signal.SIGINT, handlesigint) start_time = datetime.datetime.now() parser = argparse.ArgumentParser() parser.add_argument( "source", help="either <path> or user@<host>:<path> or <host>:<path>" ) parser.add_argument( "target", help="either <path> or user@<host>:<path> or <host>:<path>" ) parser.add_argument( "--pwd", nargs=1, help="password to authenticate on remote host" ) parser.add_argument( "--ssh_key", nargs=1, help="path to ssh private key (only if in non-default location)", ) parser.add_argument( "--scp", action="store_true", help="use scp to copy files instead of ftp" ) parser.add_argument( "--noverify", action="store_true", help="skip sha hash comparison of src and dst file", ) parser.add_argument( "--split_timeout", nargs=1, help="time to wait for remote file split operation to complete, default 120s", ) parser.add_argument( "--ssh_port", nargs=1, help="ssh port number to connect to", ) parser.add_argument("--nocurses", action="store_true", help="disable curses output") parser.add_argument("--log", nargs=1, help="log level, eg DEBUG") args = parser.parse_args() if not args.log: loglevel = "WARNING" else: loglevel = args.log[0] numeric_level = getattr(logging, loglevel.upper(), None) if not isinstance(numeric_level, int): raise ValueError(f"Invalid log level: {loglevel}") logging.basicConfig( format="%(asctime)s %(name)s %(lineno)s %(funcName)s %(levelname)s:%(message)s", level=numeric_level, ) user = None host = None passwd = None ssh_key = None ssh_port = 22 remote_dir = None remote_file = None remote_path = None local_dir = None local_path = None copy_proto = None get = False noverify = args.noverify source = args.source target = args.target use_curses = True if args.nocurses: use_curses = False if os.path.isfile(source): local_path = os.path.abspath(os.path.expanduser(source)) try: with open(local_path, "rb"): pass except PermissionError: raise SystemExit( f"source file {local_path} exists but is not readable - cannot proceed" ) local_file = os.path.basename(local_path) local_dir = os.path.dirname(local_path) elif re.search(r".*:", source): if re.search(r"@", source): user = source.split("@")[0] host = source.split("@")[1] host = host.split(":")[0] else: user = getpass.getuser() host = source.split(":")[0] remote_path = source.split(":")[1] remote_file = os.path.basename(remote_path) remote_dir = os.path.dirname(remote_path) if remote_dir == "" or remote_dir == ".": remote_dir = "~" remote_path = f"{remote_dir}/{remote_file}" if not remote_file: raise SystemExit("src path doesn't specify a file name") get = True else: raise SystemExit( "specified source is not a valid path to a local " "file, or is not in the format <user>@<host>:<path> " "or <host>:<path>" ) if os.path.isdir(target): local_dir = os.path.abspath(os.path.expanduser(target)) local_file = remote_file elif os.path.isdir(os.path.dirname(target)): # we've been passed in a filename, may not exist yet local_dir = os.path.dirname(os.path.abspath(os.path.expanduser(target))) if os.path.basename(target) != remote_file: # have to honour the change of name local_file = os.path.basename(target) else: local_file = remote_file elif re.search(r".*:", target): if re.search(r"@", target): user = target.split("@")[0] host = target.split("@")[1] host = host.split(":")[0] else: user = getpass.getuser() host = target.split(":")[0] remote_path = target.split(":")[1] if remote_path == "": remote_dir = "~" remote_file = local_file remote_path = f"{remote_dir}/{remote_file}" elif os.path.dirname(remote_path) == "": remote_dir = "~" remote_file = remote_path remote_path = f"{remote_dir}/{remote_file}" else: raise SystemExit( "specified target is not a valid path to a local " "file or directory, or is not in the format <user>@<host>:<path> " "or <host>:<path>" ) try: host = gethostbyname(host) except (gaierror, herror): raise SystemExit("hostname resolution failed") if args.pwd: passwd = args.pwd[0] if not args.scp: copy_proto = "ftp" else: copy_proto = "scp" if args.ssh_key is not None: ssh_key = os.path.abspath(args.ssh_key[0]) if not os.path.isfile(ssh_key): raise SystemExit("specified ssh key not found") if args.ssh_port is not None: try: ssh_port = int(args.ssh_port[0]) except ValueError: raise SystemExit("ssh_port must be an integer") split_timeout = 120 if args.split_timeout is not None: try: split_timeout = int(args.split_timeout[0]) except ValueError: raise SystemExit("split_timeout must be an integer") kwargs = { "user": user, "host": host, "passwd": passwd, "ssh_key": ssh_key, "ssh_port": ssh_port, "remote_dir": remote_dir, "remote_file": remote_file, "remote_path": remote_path, "local_dir": local_dir, "local_file": local_file, "local_path": local_path, "copy_proto": copy_proto, "get": get, "noverify": noverify, "split_timeout": split_timeout, "use_curses": use_curses, } logger.info(kwargs) if get: splitcopyget = SplitCopyGet(**kwargs) loop_start, loop_end = splitcopyget.get() else: splitcopyput = SplitCopyPut(**kwargs) loop_start, loop_end = splitcopyput.put() # and we are done... end_time = datetime.datetime.now() time_delta = end_time - start_time transfer_delta = loop_end - loop_start print(f"data transfer = {transfer_delta}\ntotal runtime = {time_delta}")