def test_get_all_images_with_pattern(pattern, tmpdir):
"""It gets all image files"""
image_extensions = ["tif", "tiff", "jpg", "jpeg", "png"]
a_dir = tmpdir.mkdir("image_dir")
for number in range(2):
for ext in image_extensions:
file = a_dir.join(f"file{pattern}_{number}.{ext}")
file.ensure()
for ext in image_extensions:
matches = core.get_image_files_from_pattern(a_dir, None, ext)
files = [f for f in Path(a_dir).rglob(f"**/{ext}") if f.is_file()]
assert len(list(matches)) == 2
matches = core.get_image_files_from_pattern(a_dir, pattern, None)
files = [f for f in Path(a_dir).rglob("**/*") if f.is_file()]
assert len(files) == 2 * len(image_extensions)
assert len(list(matches)) == 2 * len(image_extensions) # all image files matched
def predict_directory(
directory: Path = typer.Argument(
...,
readable=True,
resolve_path=True,
help="Directory to start searching for images from",
),
csv_save_dir: Path = typer.Argument(
...,
writable=True,
resolve_path=True,
help="Directory used to store the csv report",
),
pattern: str = typer.Option("fs", help="Pattern used to filter image filenames"),
bs: int = typer.Option(16, help="Batch Size"),
image_format: str = typer.Option(
".tif",
help="Image format for flyswot to use for predictions, defaults to `*.tif`",
),
model_name: str = typer.Option(
"latest", help="Which model flyswot should use for making predictions"
),
model_path: str = None,
):
"""Predicts against all images stored under DIRECTORY which match PATTERN in the filename.
By default searches for filenames containing 'fs'.
Creates a CSV report saved to `csv_save_dir`
"""
start_time = time.perf_counter()
model_dir = models.ensure_model_dir()
model = models.ensure_model(model_dir)
# if model_name != "latest" and not model_path:
# model_parts = models._get_model_parts(Path(model_dir / Path(model_name)))
# if model_name != "latest" and model_path:
# model_parts = models._get_model_parts(Path(model_path / Path(model_name)))
onnxinference = OnnxInferenceSession(model.model, model.vocab)
files = sorted(core.get_image_files_from_pattern(directory, pattern, image_format))
check_files(files, pattern, directory)
typer.echo(f"Found {len(files)} files matching {pattern} in {directory}")
csv_fname = create_csv_fname(csv_save_dir)
with typer.progressbar(length=len(files)) as progress:
images_checked = 0
for i, batch in enumerate(itertoolz.partition_all(bs, files)):
batch_predictions = onnxinference.predict_batch(batch, bs)
if i == 0: # pragma: no cover
create_csv_header(batch_predictions, csv_fname)
write_batch_preds_to_csv(batch_predictions, csv_fname)
progress.update(len(batch))
images_checked += len(batch)
delta = timedelta(seconds=time.perf_counter() - start_time)
print_inference_summary(
str(delta), pattern, directory, csv_fname, image_format, images_checked
)
def test_get_all_images(ext, tmpdir):
"""It gets all image files"""
a_dir = tmpdir.mkdir("image_dir")
for number in range(50):
file = a_dir.join(f"file_{number}.{ext}")
file.ensure()
for i in range(5): # create 25 files in 5 subfolders
a_sub_dir = a_dir.mkdir(f"{i}_dir")
for i in range(5):
file = a_sub_dir.join(f"file_{i}.{ext}")
file.ensure()
matches = core.get_image_files_from_pattern(a_dir, None, None)
files = [f for f in Path(a_dir).rglob("**/*") if f.is_file()]
assert len(files) == 50 + 25
assert len(list(matches)) == 50 + 25 # all image files matched
def test_filter_with_front(fname, tmpdir):
"""It filters files from pattern"""
a_dir = tmpdir.mkdir("image_dir")
for number in range(50):
file = a_dir.join(f"file_{fname}_{number}.tif")
file.ensure()
file2 = a_dir.join(f"file_{number}.jpg")
file2.ensure()
for i in range(5): # create 25 files in 5 subfolders
a_sub_dir = a_dir.mkdir(f"{i}_dir")
for i in range(5):
file = a_sub_dir.join(f"file_{fname}_{i}.tif")
file.ensure()
matches = core.get_image_files_from_pattern(a_dir, fname, ".tif")
files = [f for f in Path(a_dir).rglob("**/*") if f.is_file()]
assert len(files) == (50 * 2) + 25
assert len(list(matches)) == 50 + 25
def predict_directory(
directory: Path = typer.Argument(
...,
readable=True,
resolve_path=True,
help="Directory to start searching for images from",
),
csv_save_dir: Path = typer.Argument(
...,
writable=True,
resolve_path=True,
help="Directory used to store the csv report",
),
model_id: str = typer.Option(
"flyswot/convnext-tiny-224_flyswot",
help="The model flyswot should use for making predictions",
),
pattern: str = typer.Option(None,
help="Pattern used to filter image filenames"),
bs: int = typer.Option(16, help="Batch Size"),
image_formats: List[str] = typer.Option(
default=[".tif"],
help="Image format(s) to check",
),
):
"""Predicts against all images stored under DIRECTORY which match PATTERN in the filename.
By default searches for filenames containing 'fs'.
Creates a CSV report saved to `csv_save_dir`
"""
start_time = time.perf_counter()
huggingfaceinference = HuggingFaceInferenceSession(model=model_id)
files = sorted(
itertoolz.concat(
core.get_image_files_from_pattern(directory, pattern, image_format)
for image_format in image_formats))
check_files(files, pattern, directory)
if not pattern:
pattern = "any pattern"
print(
f"Found {len(files)} files matching {pattern} in {directory} with extension(s) {image_formats}"
)
csv_fname = create_csv_fname(csv_save_dir)
corrupt_images, images_checked = predict_files(
files,
inference_session=huggingfaceinference,
bs=bs,
csv_fname=csv_fname)
if corrupt_images:
print(corrupt_images)
delta = timedelta(seconds=time.perf_counter() - start_time)
print_inference_summary(
str(delta),
pattern,
directory,
csv_fname,
image_formats,
images_checked,
model_id,
)