def test_returns_two_shards_for_one_ambiguous(self):
pattern = os.path.join(test_path, "*****@*****.**")
sf = ShardedFile(pattern)
self.assertEqual(
sf.get_filenames(),
[os.path.join(test_path, "*****@*****.**")],
)
def test_returns_two_shards_for_two(self):
pattern = os.path.join(test_path, "*****@*****.**")
sf = ShardedFile(pattern)
self.assertEqual(
sf.get_filenames(),
[
os.path.join(test_path, "*****@*****.**"),
os.path.join(test_path, "*****@*****.**"),
],
)
def test_fails_for_inconsistent_set_star(self):
pattern = os.path.join(test_path, "inconsistent@*.baz")
with self.assertRaisesRegex(
ValueError,
f"Shard {test_path}/[email protected] does not exist.",
):
ShardedFile(pattern)
def from_file(cls, file_name: str):
"""Pass in either a single file name or a sharded file pattern.
Performs early validation by 1) opening the file if it is a single file,
or 2) computing and checking the file shards.
"""
result = cls()
result.name = file_name
if "@" in file_name:
try:
result.file_shards = ShardedFile(file_name)
except ValueError as e:
raise ArgumentTypeError("can't open '{}': {}".format(
file_name, e))
else:
result.single_file = file_name
try:
with open(file_name, "r"):
# Just check validity.
pass
except OSError as e:
raise ArgumentTypeError("can't open '{}': {}".format(
file_name, e))
return result
def test_fails_for_ambiguous_star_pattern(self):
pattern = os.path.join(test_path, "ambiguous@*.ext")
with self.assertRaisesRegex(
ValueError, "@* matches ambiguous shard sets: @1 and @2"):
ShardedFile(pattern)
def test_fails_for_bad_sharding_pattern(self):
pattern = os.path.join(test_path, "*****@*****.**")
with self.assertRaisesRegex(ValueError,
"Invalid shard specification: baz"):
ShardedFile(pattern)
def test_fails_for_no_sharding(self):
pattern = os.path.join(test_path, "foo.bar")
with self.assertRaisesRegex(ValueError, "Not a sharded file"):
ShardedFile(pattern)