def directory_with_csv_files():
with tempfile.TemporaryDirectory() as tempdir:
with chdir(tempdir):
subprocess.check_output(["sh", sh_filepath("train_data.sh")])
subprocess.check_output(["sh", sh_filepath("test_data.sh")])
subprocess.check_output(["sh", sh_filepath("predict_data.sh")])
yield tempdir
def directory_with_csv_files():
with tempfile.TemporaryDirectory() as tempdir:
with chdir(tempdir):
subprocess.check_output([
"bash",
os.path.join(
os.path.dirname(os.path.dirname(__file__)),
"examples",
"autoclassifier",
"train_data.sh",
),
])
subprocess.check_output([
"bash",
os.path.join(
os.path.dirname(os.path.dirname(__file__)),
"examples",
"autoclassifier",
"test_data.sh",
),
])
subprocess.check_output([
"bash",
os.path.join(
os.path.dirname(os.path.dirname(__file__)),
"examples",
"autoclassifier",
"predict_data.sh",
),
])
yield tempdir
def directory_with_data_files():
with tempfile.TemporaryDirectory() as tempdir:
with chdir(tempdir):
subprocess.check_output(["bash", sh_filepath("image_data.sh")])
subprocess.check_output(["bash", sh_filepath("image_file.sh")])
for image in pathlib.Path(__file__).parent.glob("*.mnistpng"):
shutil.copy(str(image.absolute()), image.name)
yield tempdir
async def test_records(self):
records = []
tmpdir = tempfile.mkdtemp()
handlejson, fileobj = tempfile.mkstemp(dir=tmpdir)
handlepy, pyfile = tempfile.mkstemp(
prefix="op_source_test_", suffix=".py", dir=tmpdir
)
# Initial check to make sure parser works
with open(fileobj, mode="w") as f:
json.dump(FILE_DATA, f)
f.seek(0)
self.assertEqual(len(parser(fileobj)), 1)
# Write parser to file
with open(pyfile, mode="w") as p:
p.write(
inspect.cleandoc(
"""
import json
"""
)
+ "\n\n"
+ inspect.getsource(parser)
)
p.seek(0)
# Now test that operation source works
with chdir(pathlib.Path(pyfile).parent):
# Load the records
records = await CLI.cli(
"list",
"records",
"-sources",
"a=op",
"-source-opimp",
f"{pathlib.Path(pyfile).stem}:{parser.__name__}",
"-source-args",
fileobj,
)
# Make sure they're correct
self.assertEqual(len(records), 1)
self.assertDictEqual(
records[0].export(),
{
"key": "42",
"features": {
"question": "babebabe?",
"context": "Feeda da facea",
"answer_text": "dead beef",
"start_pos_char": 10,
"title": "feedface",
"is_impossible": False,
"answers": [],
},
"extra": {},
},
)
os.close(handlejson)
os.close(handlepy)
shutil.rmtree(tmpdir)
def setUpLabelledDirectory(self):
with tempfile.TemporaryDirectory() as tempdir:
with chdir(tempdir):
for dirname in ["a", "b", "c"]:
dirpath = pathlib.Path(dirname)
if not os.path.exists(dirpath):
dirpath.mkdir()
for image_name in ["x", "y", "z"]:
self.blank_image.save(dirpath / (image_name + ".png"))
yield tempdir
def directory_with_data_files():
with tempfile.TemporaryDirectory() as tempdir:
with chdir(tempdir):
subprocess.check_output(["bash", sh_filepath("dataset.sh")])
subprocess.check_output(["tar", "-xzf", "17flowers.tgz"])
subprocess.check_output(["python3", sh_filepath("split.py")])
subprocess.check_output(["bash", sh_filepath("unknown_data.sh")])
for image in pathlib.Path(__file__).parent.glob("*.jpg"):
shutil.copy(str(image.absolute()), image.name)
yield tempdir
async def test_records(self):
records = []
with tempfile.NamedTemporaryFile(
mode="w+t"
) as fileobj, tempfile.NamedTemporaryFile(
mode="w+t", prefix="op_source_test_", suffix=".py"
) as pyfile:
# Initial check to make sure parser works
json.dump(FILE_DATA, fileobj)
fileobj.seek(0)
self.assertEqual(len(parser(fileobj.name)), 1)
# Write parser to file
pyfile.write(
inspect.cleandoc(
"""
import json
"""
)
+ "\n\n"
+ inspect.getsource(parser)
)
pyfile.seek(0)
# Now test that operation source works
with chdir(pathlib.Path(pyfile.name).parent):
# Load the records
records = await CLI.cli(
"list",
"records",
"-sources",
"a=op",
"-source-opimp",
f"{pathlib.Path(pyfile.name).stem}:{parser.__name__}",
"-source-args",
fileobj.name,
)
# Make sure they're correct
self.assertEqual(len(records), 1)
self.assertDictEqual(
records[0].export(),
{
"key": "42",
"features": {
"question": "babebabe?",
"context": "Feeda da facea",
"answer_text": "dead beef",
"start_pos_char": 10,
"title": "feedface",
"is_impossible": False,
"answers": [],
},
"extra": {},
},
)
async def test_shell(self):
tempdir = await cached_download_unpack_archive(
"https://download.pytorch.org/tutorial/hymenoptera_data.zip",
"hymenoptera_data.zip",
"tempdir",
"491db45cfcab02d99843fbdcf0574ecf99aa4f056d52c660a39248b5524f9e6e8f896d9faabd27ffcfc2eaca0cec6f39",
)
self.required_plugins("dffml-model-pytorch", "dffml-config-image")
def clean_args(fd, directory):
cmnd = " ".join(fd.readlines()).split("\\\n")
cmnd = " ".join(cmnd).split()
for idx, word in enumerate(cmnd):
cmnd[idx] = word.strip()
cmnd[cmnd.index("-model-directory") + 1] = directory
if "-model-epochs" in cmnd:
cmnd[cmnd.index("-model-epochs") + 1] = "1"
return cmnd
with chdir(tempdir):
os.unlink(
os.path.join(
"hymenoptera_data", "train", "ants", "imageNotFound.gif"
)
)
subprocess.check_output(
["bash", sh_filepath("resnet18", "unknown_data.sh")]
)
shutil.copy(
sh_filepath("resnet18", "layers.yaml"),
os.path.join(os.getcwd(), "layers.yaml"),
)
with open(sh_filepath("resnet18", "train.sh"), "r") as f:
train_command = clean_args(f, str(tempdir))
await CLI.cli(*train_command[1:])
with open(sh_filepath("resnet18", "accuracy.sh"), "r") as f:
accuracy_command = clean_args(f, str(tempdir))
await CLI.cli(*accuracy_command[1:])
with open(sh_filepath("resnet18", "predict.sh"), "r") as f:
predict_command = clean_args(f, str(tempdir))
results = await CLI.cli(*predict_command[1:-1])
self.assertTrue(isinstance(results, list))
self.assertTrue(results)
results = results[0]
self.assertTrue(results.prediction("label"))
results = results.prediction("label")
self.assertIn("value", results)
self.assertIn("confidence", results)
self.assertIn(isinstance(results["value"], str), [True])
self.assertTrue(results["confidence"])
async def test_run(self):
await Develop.cli("skel", "link")
common_files = [
path.relative_to(self.skel.common)
for path in self.skel.common_files()
]
# At time of writing there are 4 plugins in skel/ change this as needed
plugins = self.skel.plugins()
self.assertGreater(len(plugins), 3)
for plugin in plugins:
for check in COMMON_FILES:
with chdir(plugin):
self.assertTrue(
check.is_symlink(),
f"{check.resolve()} is not a symlink",
)
def directory_with_csv_files():
with tempfile.TemporaryDirectory() as tempdir:
with chdir(tempdir):
subprocess.check_output(
[
"bash",
os.path.join(ROOT, "examples", "nlp", "train_data.sh",),
]
)
subprocess.check_output(
[
"bash",
os.path.join(ROOT, "examples", "nlp", "test_data.sh",),
]
)
yield tempdir
async def generic_test(self, name, package_specific_files):
package_name = "test-package"
# Acquire the CreateCMD class of specified type
cli_class = getattr(Develop.create, name)
# Create tempdir to copy files to
with tempfile.TemporaryDirectory() as tempdir:
# Create directories in tempdir, one we cd into and one we use as
# target parameter
for target in [
os.path.join(tempdir, "dot"),
os.path.join(tempdir, "within"),
]:
with self.subTest(target=target):
# Create the directory within the tempdir
os.mkdir(target)
# Instantiate an instance of the CreateCMD class
cli = cli_class(
package=package_name,
description=None,
target=target
if target[::-1].startswith(("dot")[::-1])
else None,
)
# Change directories
with chdir(target):
# Call create command
await cli.run()
# Verify that all went as planned
if target[::-1].startswith(("dot")[::-1]):
self.verify(
target, package_name, package_specific_files
)
elif target[::-1].startswith(("within")[::-1]):
self.verify(
os.path.join(target, package_name),
package_name,
package_specific_files,
)
else: # pragma: no cov
pass
async def test_shell(self, tempdir):
self.required_plugins("dffml-model-pytorch", "dffml-config-image")
def clean_args(fd, directory):
cmnd = " ".join(fd.readlines()).split("\\\n")
cmnd = " ".join(cmnd).split()
for idx, word in enumerate(cmnd):
cmnd[idx] = word.strip()
cmnd[cmnd.index("-model-directory") + 1] = directory
return cmnd
with chdir(tempdir):
os.unlink(
os.path.join("hymenoptera_data", "train", "ants",
"imageNotFound.gif"))
subprocess.check_output(
["bash", sh_filepath("resnet18", "unknown_data.sh")])
with open(sh_filepath("resnet18", "train.sh"), "r") as f:
train_command = clean_args(f, str(tempdir))
await CLI.cli(*train_command[1:])
with open(sh_filepath("resnet18", "accuracy.sh"), "r") as f:
accuracy_command = clean_args(f, str(tempdir))
await CLI.cli(*accuracy_command[1:])
with open(sh_filepath("resnet18", "predict.sh"), "r") as f:
predict_command = clean_args(f, str(tempdir))
results = await CLI.cli(*predict_command[1:-1])
self.assertTrue(isinstance(results, list))
self.assertTrue(results)
results = results[0]
self.assertTrue(results.prediction("label"))
results = results.prediction("label")
self.assertIn("value", results)
self.assertIn("confidence", results)
self.assertIn(isinstance(results["value"], str), [True])
self.assertTrue(results["confidence"])
async def test_run(self):
# Skip if not in development mode
if not is_develop("dffml"):
self.skipTest("dffml not installed in development mode")
await Develop.cli("skel", "link")
common_files = [
path.relative_to(self.skel.common)
for path in self.skel.common_files()
]
# At time of writing there are 4 plugins in skel/ change this as needed
plugins = self.skel.plugins()
self.assertGreater(len(plugins), 3)
for plugin in plugins:
for check in COMMON_FILES:
# We skip the setup.cfg symlink check for blank because it has a
# custom setup.cfg
if plugin.name == "blank" and check.name == "setup.cfg":
continue
with chdir(plugin):
self.assertTrue(
check.is_symlink(),
f"{check.resolve()} is not a symlink",
)
def setUpUnlabelledDirectory(self):
with tempfile.TemporaryDirectory() as tempdir:
with chdir(tempdir):
for image_name in ["l", "m", "n", "o", "k"]:
self.blank_image.save(image_name + ".png")
yield tempdir
async def setUp(self):
await super().setUp()
self._stack.enter_context(chdir(REPO_ROOT))
def testcase(self):
with chdir(NOTEBOOK_DATA_PATH):
with testbook(path, timeout=-1, execute=True) as tb:
pass
def directory_with_csv_files():
with tempfile.TemporaryDirectory() as tempdir:
with chdir(tempdir):
subprocess.check_output(["bash", sh_filepath("dataset.sh")])
yield tempdir