def compile_custom_op(path):
"""Runs the make command to build the custom op objects"""
files_to_generate = ["custom_codelet.gp", "libcustom_op.so"]
if check_data_exists(path, files_to_generate):
print("Objects already present, cleaning...")
subprocess.run(["make", "clean"], cwd=path)
completed = subprocess.run("make", cwd=path)
if completed.returncode != 0 or not check_data_exists(
path, files_to_generate):
raise Exception("Custom op compilation failed")
print("Successfully compiled custom op")
def _check_file_exists_helper(self, report_path, filename):
"""Helper function to check whether report_generation.md exists
in location report_path and raises an assertion error if it
doesn't exist."""
if not check_data_exists(report_path, [filename]):
raise AssertionError("{} does not exist in location {}".format(
filename, report_path))
def generate_data(path_to_generation_script):
"""Runs the data generation scripts assumes path"""
files_to_generate = ["training.csv", "validation.csv"]
if test_util.check_data_exists(path_to_generation_script,
files_to_generate):
print("Data already generated, skipping...")
return
cwd = os.path.dirname(os.path.abspath(inspect.stack()[0][1]))
test_util.run_python_script_helper(cwd, "data_gen/generate.py")
if not test_util.check_data_exists(path_to_generation_script,
files_to_generate):
raise Exception("Dataset generation failed")
print("Successfully generated datasets")
def _check_file_exists_helper(self, path_to_file):
"""Helper function to check whether report_generation.md exists
in location path_to_file and raises an assertion error if it
doesn't exist"""
files_to_generate = ["report.txt"]
if not test_util.check_data_exists(path_to_file, files_to_generate):
raise AssertionError(
"report.txt does not exist in location {}".format(
path_to_file))
def generate_data(path_to_generation_script):
"""Runs the data generation scripts assumes path"""
files_to_generate = ["training.csv", "validation.csv"]
if check_data_exists(
path_to_generation_script,
files_to_generate
):
print("Data already generated, skipping...")
return
run_python_script_helper(os.path.dirname(__file__), "data_gen/generate.py")
if not check_data_exists(
path_to_generation_script,
files_to_generate
):
raise Exception("Dataset generation failed")
print("Successfully generated datasets")
def check_all_data_present(file_path):
"""Checks the data exists in location file_path"""
filenames = [
"t10k-images-idx3-ubyte",
"t10k-labels-idx1-ubyte",
"train-images-idx3-ubyte",
"train-labels-idx1-ubyte",
]
data_path = os.path.join(file_path, "data")
return check_data_exists(data_path, filenames)
def generate_nmt_data():
"""Runs the data generation script which creates the data
required for the NMT tests."""
subprocess.run(
["./generate_data.sh"],
universal_newlines=True,
cwd=DATA_GEN_SCRIPT_PATH
)
if not test_util.check_data_exists(
DATA_PATH, FILES_TO_GENERATE
):
raise Exception(
f"Dataset generation failed. Cannot find files"
f" {FILES_TO_GENERATE} in location {DATA_PATH}."
)
print("Successfully generated datasets")
def _remove_any_existing_report(self, report_path, filename):
"""Helper function to check whether filename exists in report_path.
If it does, the file is removed."""
if check_data_exists(report_path, [filename]):
os.remove(os.path.join(report_path, filename))