def test_returns_two_shards_for_one_ambiguous(self): pattern = os.path.join(test_path, "*****@*****.**") sf = ShardedFile(pattern) self.assertEqual( sf.get_filenames(), [os.path.join(test_path, "*****@*****.**")], )
def test_returns_two_shards_for_two(self): pattern = os.path.join(test_path, "*****@*****.**") sf = ShardedFile(pattern) self.assertEqual( sf.get_filenames(), [ os.path.join(test_path, "*****@*****.**"), os.path.join(test_path, "*****@*****.**"), ], )
def test_fails_for_inconsistent_set_star(self): pattern = os.path.join(test_path, "inconsistent@*.baz") with self.assertRaisesRegex( ValueError, f"Shard {test_path}/[email protected] does not exist.", ): ShardedFile(pattern)
def from_file(cls, file_name: str): """Pass in either a single file name or a sharded file pattern. Performs early validation by 1) opening the file if it is a single file, or 2) computing and checking the file shards. """ result = cls() result.name = file_name if "@" in file_name: try: result.file_shards = ShardedFile(file_name) except ValueError as e: raise ArgumentTypeError("can't open '{}': {}".format( file_name, e)) else: result.single_file = file_name try: with open(file_name, "r"): # Just check validity. pass except OSError as e: raise ArgumentTypeError("can't open '{}': {}".format( file_name, e)) return result
def test_fails_for_ambiguous_star_pattern(self): pattern = os.path.join(test_path, "ambiguous@*.ext") with self.assertRaisesRegex( ValueError, "@* matches ambiguous shard sets: @1 and @2"): ShardedFile(pattern)
def test_fails_for_bad_sharding_pattern(self): pattern = os.path.join(test_path, "*****@*****.**") with self.assertRaisesRegex(ValueError, "Invalid shard specification: baz"): ShardedFile(pattern)
def test_fails_for_no_sharding(self): pattern = os.path.join(test_path, "foo.bar") with self.assertRaisesRegex(ValueError, "Not a sharded file"): ShardedFile(pattern)