def make_import_all_script(ell_models, outdir, language="cpp", target="host", profile=False): mkdir_str = "mkdir {}" ell_root = find_ell.get_ell_root() wrap_path = os.path.abspath( os.path.join(ell_root, "tools", "wrap", "wrap.py")) wrap_str = "python {} -t {} -l {} --llvm_format ir -od {}/{} {} -f {} \n" profile_str = "--profile" if profile else "" str_list = [] for (model_name, model_filename, model_relpath, model_path) in ell_models: str_list.append(mkdir_str.format(model_name)) last_str = model_path if model_filename in special_model_args: last_str += " " + " ".join(special_model_args[model_filename]) str_list.append( wrap_str.format(wrap_path, target, language, model_name, model_name, profile_str, last_str)) win_outfile = os.path.join(outdir, "import_all.cmd") unix_outfile = os.path.join(outdir, "import_all.sh") with open(win_outfile, 'w') as f: f.write(win_script_header) f.write("\n".join(str_list)) with open(unix_outfile, 'w', newline='\n') as f: f.write("\n".join(str_list))
def test(): testing = Testing() # -dd auto -sw 1 -sb 1 -sz 1 -pd 10 -l 2 -mp 5 -v --evaluationFrequency 1 -plf L2 args = ell.ProtoNNTrainerParameters() args.projectedDimension = 10 args.numPrototypesPerLabel = 5 args.numLabels = 2 args.sparsityW = 1 args.sparsityB = 1 args.sparsityZ = 1 args.gamma = -1 args.lossFunction = ell.ProtoNNLossFunction.L2 args.numInnerIterations = 1 args.numFeatures = 0 args.verbose = True trainer = ell.ProtoNNTrainer(args) dataset = ell.AutoSupervisedDataset() testFile = os.path.join(find_ell.get_ell_root(), "examples/data/protonnTestData.txt") print("Loading: " + testFile) dataset.Load(testFile) total = dataset.NumExamples() features = dataset.NumFeatures() testing.ProcessTest("Proton dataset loaded", testing.IsEqual(int(total), 200)) trainer.SetDataset(dataset) numIterations = 20 print("Training...") for i in range(numIterations): trainer.Update() predictor = trainer.GetPredictor() accuracy = get_accuracy(predictor, dataset, features) print("Accuracy %f" % (accuracy)) testing.ProcessTest("Proton accuracy test", testing.IsEqual(int(accuracy), 1)) map = predictor.GetMap() map.Save("protonnTestData.ell") testing.ProcessTest( "Saving protonnTestData.ell", testing.IsEqual(os.path.isfile("protonnTestData.ell"), True)) if testing.DidTestFail(): raise Exception("protonn_trainer_test failed") return 0
def make_project(self, target_dir): build_dir = os.path.join(target_dir, "build") if os.path.isdir(build_dir): rmtree(build_dir) os.makedirs(build_dir) current_path = os.getcwd() os.chdir(build_dir) cmd = buildtools.EllBuildTools(find_ell.get_ell_root()) cmd.cmake_generate("..") make = ["make"] if os.name == 'nt': make = ["cmake", "--build", ".", "--config", "Release"] cmd.run(make, print_output=True) os.chdir(current_path)
def make_project(target_dir): build_dir = os.path.join(target_dir, "build") if os.path.isdir(build_dir): rmtree(build_dir) os.makedirs(build_dir) current_path = os.getcwd() os.chdir(build_dir) cmd = buildtools.EllBuildTools(find_ell.get_ell_root(), verbose=True) cmake = ["cmake", ".."] if os.name == 'nt': cmake = ["cmake", "-G", "Visual Studio 15 2017 Win64", ".."] cmd.run(cmake, print_output=True) make = ["make"] if os.name == 'nt': make = ["cmake", "--build", ".", "--config", "Release"] cmd.run(make, print_output=True) os.chdir(current_path)
def test_cpp(model_path): target_dir = os.path.join(os.path.dirname(model_path), "tutorial_cpp") if os.path.isdir(target_dir): rmtree(target_dir) os.makedirs(target_dir) copyfile(os.path.join(script_path, "tutorialCMakeLists.txt"), os.path.join(target_dir, "CMakeLists.txt")) copyfile(os.path.join(script_path, "tutorial.cpp"), os.path.join(target_dir, "tutorial.cpp")) # invoke "wrap.py" helper to create a compilable C++ project wrap_model(model_path, target_dir, "cpp") # compile the project using cmake. make_project(target_dir) # did it actually build? binary = os.path.join(target_dir, "build") if os.name == 'nt': binary = os.path.join(binary, "release", "tutorial.exe") else: binary = os.path.join(binary, "tutorial") if not os.path.isfile(binary): print("### wrap_test failed, binary 'tutorial' was not produced") return 1 # execute the compiled tutorial.exe binary and check the output cmd = buildtools.EllBuildTools(find_ell.get_ell_root(), verbose=True) output = cmd.run([binary], print_output=True) if not "Prediction=0, 1, 2, 3, 4, 5, 6, 7, 8, 9" in output: print( "### FAILED: wrap_test cpp binary did not print the expected results, got the following:\n{}" .format(output)) return 1 else: print("### PASSED wrap_test: test_cpp") return 0
def test(): testing = Testing() dataset = ell.data.AutoSupervisedDataset() dataset.Load( os.path.join(find_ell.get_ell_root(), "examples/data/testData.txt")) num = dataset.NumExamples() print("Number of Examples:", num) testing.ProcessTest("Dataset NumExamples test", testing.IsEqual(int(num), 200)) features = dataset.NumFeatures() print("Number of Features:", features) testing.ProcessTest("Dataset NumFeatures test", testing.IsEqual(int(features), 21)) for i in range(num): exampleTest(dataset.GetExample(i)) testing.ProcessTest("Dataset eumeration test", True) return 0
################################################################################################### # # Project: Embedded Learning Library (ELL) # File: add_input_metadata.py # Authors: Mason Remy # # Requires: Python 3.x # ################################################################################################### import sys import os import argparse import find_ell ell_root = find_ell.get_ell_root() sys.path += [ os.path.join(ell_root, "build", "interfaces", "python", "package") ] import ell def get_input_node(model): """ return the model input node """ iter = model.GetNodes() while iter.IsValid(): node = iter.Get() if "InputNode" in node.GetRuntimeTypeName(): return node iter.Next()
def test(): testing = Testing() # -dd auto -sw 1 -sb 1 -sz 1 -pd 10 -l 2 -mp 5 -v --evaluationFrequency 1 -plf L2 args = ell.trainers.ProtoNNTrainerParameters() args.projectedDimension = 10 args.numPrototypesPerLabel = 5 args.numLabels = 2 args.sparsityW = 1 args.sparsityB = 1 args.sparsityZ = 1 args.gamma = -1 args.lossFunction = ell.trainers.ProtoNNLossFunction.L2 args.numInnerIterations = 1 args.numFeatures = 0 args.verbose = True trainer = ell.trainers.ProtoNNTrainer(args) dataset = ell.data.AutoSupervisedDataset() testFile = os.path.join(find_ell.get_ell_root(), "examples/data/protonnTestData.txt") print("Loading: " + testFile) dataset.Load(testFile) total = dataset.NumExamples() features = dataset.NumFeatures() testing.ProcessTest("ProtoNN dataset loaded", testing.IsEqual(int(total), 200)) trainer.SetDataset(dataset) numIterations = 20 print("Training...") for i in range(numIterations): trainer.Update() predictor = trainer.GetPredictor() accuracy = get_accuracy(predictor, dataset, features, predictor.Predict) print("Accuracy %f" % (accuracy)) testing.ProcessTest("ProtoNN accuracy test", testing.IsEqual(int(accuracy), 1)) map = predictor.GetMap() map.Save("protonnTestData.ell") testing.ProcessTest( "Saving protonnTestData.ell", testing.IsEqual(os.path.isfile("protonnTestData.ell"), True)) # make sure we can compile this map. try: compilerSettings = ell.model.MapCompilerOptions() compilerSettings.useBlas = False optimizerSettings = ell.model.ModelOptimizerOptions() compiledMap = map.CompileDouble("host", "protonn", "predict", compilerSettings, optimizerSettings) compiled = True except Exception as e: print("Compile ProtoNN model failed: {}", e) compiled = False testing.ProcessTest("Compiling protonnTestData.ell", testing.IsEqual(compiled, True)) if compiled: accuracy = get_accuracy(predictor, dataset, features, compiledMap.ComputeDouble) print("Compiled Accuracy %f" % (accuracy)) testing.ProcessTest("ProtoNN compiled accuracy test", testing.IsEqual(int(accuracy), 1)) if testing.DidTestFail(): raise Exception("protonn_trainer_test failed") return 0