def test_md5(self) -> None: url = "{}/{}".format(TEST_GSUTILWRAP_URL_PREFIX, uuid.uuid4()) another_url = "{}/{}".format(TEST_GSUTILWRAP_URL_PREFIX, uuid.uuid4()) nonexisting_url = "{}/{}/i-dont-exist/really/not-at-all".format(TEST_GSUTILWRAP_URL_PREFIX, uuid.uuid4()) with tempfile.TemporaryDirectory() as tmp_dir_name: tmp_dir = pathlib.Path(tmp_dir_name) pth = tmp_dir / 'testme.txt' pth.write_text('tested') another_pth = tmp_dir / 'another_testme.txt' another_pth.write_text('so different') try: # same file gsutilwrap.copy(pattern=pth, target=url, quiet=True) self.assertTrue( gsutilwrap.same_md5(path=pth, url=url), "Expected md5 to be the same, but they were different.") # different file gsutilwrap.copy(pattern=another_pth.as_posix(), target=url, quiet=True) self.assertFalse( gsutilwrap.same_md5(path=pth, url=url), "Expected md5 to be different, but they were same.") # non-existing remote object self.assertFalse( gsutilwrap.same_md5(path=pth, url=nonexisting_url), "Expected md5 to be different when the object doesn't exist, but they were same.") # non-existing local file nonexisting_pth = tmp_dir / 'so-does-not-exist.txt' self.assertFalse( gsutilwrap.same_md5(path=nonexisting_pth, url=url), "Expected md5 to be different when the local file doesn't exist, but they were same.") # check md5 hex digests gsutilwrap.copy(pattern=pth, target=url, quiet=True) gsutilwrap.copy(pattern=another_pth, target=another_url, quiet=True) expected = ['d941191e51e81390343e12b159bb123f', '226c758acdfb13ed5f6e5bea3aef5a4d'] # multi-threaded md5_hexdigests = gsutilwrap.md5_hexdigests(urls=[url, another_url], multithreaded=True) self.assertListEqual(expected, md5_hexdigests) # single-threaded md5_hexdigests = gsutilwrap.md5_hexdigests(urls=[url, another_url]) self.assertListEqual(expected, md5_hexdigests) finally: if len(gsutilwrap.ls(pattern=url)) > 0: gsutilwrap.remove(pattern=url, quiet=True, multithreaded=True) if len(gsutilwrap.ls(pattern=another_url)) > 0: gsutilwrap.remove(pattern=another_url, quiet=True, multithreaded=True)
def benchmark_stat_many(self) -> None: for testcase in [10, 100]: with temppathlib.TemporaryDirectory() as tmp_dir: for index in range(testcase): file = tmp_dir.path / "file{}".format(index) file.write_text("hello") try: _setup(url=self.url_prefix, path=tmp_dir.path) time_gsutilwrap = 0.0 urls = gsutilwrap.ls(self.url_prefix + "**") for url in urls: time_gsutilwrap += timer(gsutilwrap.stat, url=url) client = gswrap.Client() time_gswrap = 0.0 urls = client.ls(url=self.url_prefix, recursive=True) for url in urls: time_gswrap += timer(client.stat, url=url) finally: _tear_down(url=self.url_prefix) print_benchmark( benchmark="Benchmark stat {} files".format(testcase), time_other_libraries=[(LibraryChecked.gsutilwrap, time_gsutilwrap)], time_gswrap=time_gswrap)
def _gsutilwrap_list_for_cp_many_to_many(src: str, dst: str) -> List[Tuple[str, str]]: lst = gsutilwrap.ls(src + "**") srcs_dsts = [] for file in lst: srcs_dsts.append((file, file.replace(src, dst))) return srcs_dsts
def test_write_read_text(self) -> None: quiet = True url = "{}/{}/file.txt".format(TEST_GSUTILWRAP_URL_PREFIX, uuid.uuid4()) expected_text = 'oi' try: self.assertEqual(0, len(gsutilwrap.ls(pattern=url))) gsutilwrap.write_text(url=url, text=expected_text, quiet=quiet) self.assertEqual(1, len(gsutilwrap.ls(pattern=url))) text = gsutilwrap.read_text(url=url) self.assertEqual(expected_text, text) finally: if len(gsutilwrap.ls(pattern=url)) > 0: gsutilwrap.remove(pattern=url, quiet=True)
def test_copy(self) -> None: quiet = True base_url = "{}/{}".format(TEST_GSUTILWRAP_URL_PREFIX, uuid.uuid4()) with tempfile.TemporaryDirectory() as tmp_dir_name: tmp_dir = pathlib.Path(tmp_dir_name) try: # Create some files in a temporary directory and copy them to the Google Storage tmp_file_1 = tmp_dir / "file_1.txt" tmp_file_1.write_text('File 1') tmp_file_2 = tmp_dir / "file_2.txt" tmp_file_2.write_text('File 2') # Make sure that nothing is stored in the test_url self.assertEqual(0, len(gsutilwrap.ls(pattern=base_url))) gsutilwrap.copy(tmp_file_1, "{}/file_0.txt".format(base_url), quiet=quiet) self.assertListEqual(['{}/file_0.txt'.format(base_url)], gsutilwrap.ls(base_url)) gsutilwrap.copy_many_to_one(patterns=[tmp_file_1, tmp_file_2], target=base_url, quiet=quiet) self.assertListEqual([ '{}/file_0.txt'.format(base_url), '{}/file_1.txt'.format(base_url), '{}/file_2.txt'.format(base_url) ], gsutilwrap.ls(base_url)) gsutilwrap.copy_many_to_many( patterns_targets=[(tmp_file_1, "{}/file_3.txt".format(base_url)), (tmp_file_2, "{}/file_4.txt".format(base_url))], quiet=quiet) self.assertListEqual([ '{}/file_0.txt'.format(base_url), '{}/file_1.txt'.format(base_url), '{}/file_2.txt'.format(base_url), '{}/file_3.txt'.format(base_url), '{}/file_4.txt'.format(base_url) ], gsutilwrap.ls(base_url)) finally: if len(gsutilwrap.ls(pattern=base_url)) > 0: gsutilwrap.remove(pattern=base_url, quiet=True, multithreaded=True, recursive=True)
def test_write_read_bytes(self) -> None: quiet = True url = "{}/{}/file.bin".format(TEST_GSUTILWRAP_URL_PREFIX, uuid.uuid4()) expected_data = b'\xDE\xAD\xBE\xEF' try: self.assertEqual(0, len(gsutilwrap.ls(pattern=url))) gsutilwrap.write_bytes(url=url, data=expected_data, quiet=quiet) self.assertEqual(1, len(gsutilwrap.ls(pattern=url))) data = gsutilwrap.read_bytes(url=url) self.assertEqual(expected_data, data) finally: if len(gsutilwrap.ls(pattern=url)) > 0: gsutilwrap.remove(pattern=url, quiet=True)
def test_ls(self) -> None: quiet = True base_url = "{}/{}".format(TEST_GSUTILWRAP_URL_PREFIX, uuid.uuid4()) url_doesnt_exist = "{}/i-dont-exist/really/not-at-all".format(base_url) self.assertEqual(0, len(gsutilwrap.ls(pattern=url_doesnt_exist))) urls = ['{}/one.txt'.format(base_url), '{}/two.txt'.format(base_url)] try: for url in urls: gsutilwrap.write_text(url=url, text="some dummy content", quiet=quiet) for url in urls: self.assertEqual(1, len(gsutilwrap.ls(pattern=url))) listed_urls = gsutilwrap.ls(pattern='{}/*.txt'.format(base_url)) self.assertListEqual(urls, listed_urls) # Check if dont_recurse is working self.assertEqual(2, len(gsutilwrap.ls(pattern="{}/".format(base_url)))) self.assertEqual(1, len(gsutilwrap.ls(pattern="{}/".format(base_url), dont_recurse=True))) finally: if len(gsutilwrap.ls(pattern=base_url)) > 0: gsutilwrap.remove(pattern=base_url, quiet=True, multithreaded=True, recursive=True)
def test_copy_no_clobber(self) -> None: quiet = True base_url = "{}/{}".format(TEST_GSUTILWRAP_URL_PREFIX, uuid.uuid4()) url = '{}/file.txt'.format(base_url) with tempfile.TemporaryDirectory() as tmp_dir_name: tmp_dir = pathlib.Path(tmp_dir_name) try: tmp_file = tmp_dir / "file.txt" original_content = "original content" new_content = "new content" tmp_file.write_text(original_content) gsutilwrap.copy(pattern=tmp_file, target=url, quiet=quiet) content = gsutilwrap.read_text(url=url) self.assertEqual(original_content, content) # update the file tmp_file.write_text(new_content) # copy gsutilwrap.copy(pattern=tmp_file, target=url, no_clobber=True, quiet=quiet) content = gsutilwrap.read_text(url=url) self.assertEqual(original_content, content) # copy many to one gsutilwrap.copy_many_to_one(patterns=[tmp_file], target=base_url + "/", no_clobber=True, quiet=quiet) content = gsutilwrap.read_text(url=url) self.assertEqual(original_content, content) # copy many to many gsutilwrap.copy_many_to_many(patterns_targets=[(tmp_file, url)], no_clobber=True, quiet=quiet) content = gsutilwrap.read_text(url=url) self.assertEqual(original_content, content) finally: if len(gsutilwrap.ls(pattern=base_url)) > 0: gsutilwrap.remove(pattern=base_url, quiet=True, multithreaded=True, recursive=True)
def test_many_ls(self) -> None: quiet = True url_doesnt_exist = "{}/i-dont-exist/really/not-at-all".format(TEST_GSUTILWRAP_URL_PREFIX) base_url = "{}/{}".format(TEST_GSUTILWRAP_URL_PREFIX, uuid.uuid4()) urls = ['{}/one.txt'.format(base_url), '{}/two.txt'.format(base_url)] try: for url in urls: gsutilwrap.write_text(url=url, text="some dummy content", quiet=quiet) patterns = [url_doesnt_exist] + urls result = gsutilwrap.ls_many(patterns=patterns) self.assertListEqual([[], [base_url + '/one.txt'], [base_url + '/two.txt']], result) finally: if len(gsutilwrap.ls(pattern=base_url)) > 0: gsutilwrap.remove(pattern=base_url, quiet=True, multithreaded=True, recursive=True)
def test_stat(self) -> None: url = "{}/{}".format(TEST_GSUTILWRAP_URL_PREFIX, uuid.uuid4()) with tempfile.TemporaryDirectory() as tmp_dir_name: tmp_dir = pathlib.Path(tmp_dir_name) pth = tmp_dir / 'testme.txt' pth.write_text('tested') atime = 1 mtime = 2 os.utime(pth.as_posix(), (atime, mtime)) try: gsutilwrap.copy(pattern=pth.as_posix(), target=url, preserve_posix=True, quiet=True) stat = gsutilwrap.stat(url=url) self.assertIsNotNone(stat) # cast assert stat is not None file_stat = pth.stat() self.assertIsNotNone(stat.file_mtime) self.assertIsNotNone(stat.content_length) # casts assert stat.file_mtime is not None assert stat.content_length is not None self.assertEqual(datetime.datetime.utcfromtimestamp(file_stat.st_mtime), stat.file_mtime) self.assertEqual(file_stat.st_size, stat.content_length) finally: if len(gsutilwrap.ls(pattern=url)) > 0: gsutilwrap.remove(pattern=url, quiet=True, multithreaded=True, recursive=True)
def test_long_ls(self) -> None: quiet = True base_url = "{}/{}".format(TEST_GSUTILWRAP_URL_PREFIX, uuid.uuid4()) url_doesnt_exist = "{}/i-dont-exist/really/not-at-all".format(base_url) self.assertEqual(0, len(gsutilwrap.long_ls(pattern=url_doesnt_exist))) urls = ['{}/one.txt'.format(base_url), '{}/two.txt'.format(base_url)] try: gsutilwrap.write_text(url=urls[0], text="some dummy content", quiet=quiet) gsutilwrap.write_text(url=urls[1], text="another dummy content", quiet=quiet) for url in urls: self.assertEqual(1, len(gsutilwrap.long_ls(pattern=url))) entries = gsutilwrap.long_ls(pattern='{}/*.txt'.format(base_url)) entries.sort(key=lambda entry: entry.url) self.assertEqual(urls[0], entries[0].url) self.assertEqual(urls[1], entries[1].url) self.assertEqual(18, entries[0].size) self.assertEqual(21, entries[1].size) # Check dont_recurse entries = gsutilwrap.long_ls(pattern="{}/".format(base_url), dont_recurse=True) self.assertEqual(1, len(entries)) self.assertTrue(entries[0].url.endswith('/')) self.assertIsNone(entries[0].update_time) self.assertIsNone(entries[0].size) finally: if len(gsutilwrap.ls(pattern=base_url)) > 0: gsutilwrap.remove(pattern=base_url, quiet=True, multithreaded=True, recursive=True)