def test_make_dir_local(self): TEST_DIR = "/tmp/some_dir_123/1234" if executor.path_exists(TEST_DIR): executor.delete_dir(TEST_DIR) executor.make_dirs(TEST_DIR) assert os.path.exists(TEST_DIR), "Failed tp create directory {}".format(TEST_DIR) shutil.rmtree("/tmp/some_dir_123")
def test_make_dir_local(self): TEST_DIR = '/tmp/some_dir_123/1234' if executor.path_exists(TEST_DIR): executor.delete_dir(TEST_DIR) executor.make_dirs(TEST_DIR) assert os.path.exists(TEST_DIR),\ 'Failed tp create directory {}'.format(TEST_DIR) shutil.rmtree('/tmp/some_dir_123')
def transfer(src, dst, creds, upstream=True,\ tries=3, include=None, parallelism=10, extract=False): """ @extract: boolean - whether to extract tar or zip files after transfer @parallelism(default=10): number of parallel processes to use """ if isinstance(creds, dict): creds = Bunch(creds) if 'key' in creds: creds.key = os.path.expanduser(creds.key) if 'key_filename' in creds: creds.key = os.path.expanduser(creds.key_filename[0]) if upstream: srcs = executor.find_files(src, None, include=include) else: srcs = executor.find_files(src, creds, include=include) if len(srcs) < 1: print('No source files found to transfer.') return src_dirs = set([os.path.dirname(path) for path in srcs]) dst_dirs = [path.replace(src, dst) for path in src_dirs] dst_dirs = [path for path in dst_dirs if path not in ['', '/']] if upstream: executor.make_dirs(dst_dirs, creds=creds) else: executor.make_dirs(dst_dirs) dests = [] for path in srcs: if path[:len(src)].endswith('/'): path = os.path.join(dst, path[len(src):]) else: path = os.path.join(dst, path[len(src) + 1:]) dests.append(path) rsync = "rsync -raz -e 'ssh"\ " -o StrictHostKeyChecking=no"\ " -o ServerAliveInterval=100"\ " -i {}'".format(creds.key) cmds = [] for ind, path in enumerate(srcs): cmd = "{} {}@{}:{} {}".format(rsync, creds.user, creds.host, path, dests[ind]) if upstream: cmd = "{} {} {}@{}:{}".format(rsync, path, creds.user, creds.host, dests[ind]) cmds.append(cmd) pool = Pool(parallelism, init_worker) func = partial(executor._local, None, tries) pool.map(func, cmds) pool.close() pool.join() if extract: compression.extract(dst, creds=creds)
def __make_dirs(paths, creds, upstream): dirs = [os.path.dirname(path[1]) for path in paths] if upstream: executor.make_dirs(dirs, creds=creds) else: executor.make_dirs(dirs)