def export_all(): ''' Export all sklearn pickle model to C code ''' ml_model_path = os.path.join(os.path.dirname(__file__), "../../ml-model") output_path = os.path.join(os.path.dirname(__file__), "output") if not os.path.exists(output_path): os.mkdir(output_path) with open( os.path.join(ml_model_path, "./saved-models/decision_tree_model.pickle"), "rb") as fd_model: model = pickle.load(fd_model) with open(os.path.join(output_path, "decision_tree.c"), "w") as fd: porter = Porter(model, language='C') fd.write(porter.export(embed_data=True)) with open( os.path.join(ml_model_path, "./saved-models/random_forest_model.pickle"), "rb") as fd_model: model = pickle.load(fd_model) with open(os.path.join(output_path, "random_forest.c"), "w") as fd: porter = Porter(model, language='C') fd.write(porter.export(embed_data=True))
def _port_estimator(self, export_data=False): self.estimator.fit(self.X, self.y) subp.call('rm -rf tmp'.split()) subp.call('mkdir tmp'.split()) with open(self.tmp_fn, 'w') as f: porter = Porter(self.estimator, language=self.LANGUAGE) if export_data: out = porter.export(class_name='Brain', method_name='foo', export_data=True, export_dir='tmp') else: out = porter.export(class_name='Brain', method_name='foo') f.write(out)
def test_python_command_execution(self): """Test command line execution.""" subp.call('rm -rf tmp'.split()) subp.call('mkdir tmp'.split()) filename = '{}.java'.format(self.tmp_fn) cp_src = os.path.join('tmp', filename) with open(cp_src, 'w') as f: porter = Porter(self.estimator) out = porter.export(method_name='predict', class_name=self.tmp_fn) f.write(out) # $ javac tmp/Tmp.java subp.call(['javac', cp_src]) # Rename estimator for comparison: filename = '{}_2.java'.format(self.tmp_fn) cp_dest = os.path.join('tmp', filename) # $ mv tmp/Brain.java tmp/Brain_2.java subp.call(['mv', cp_src, cp_dest]) # Dump estimator: filename = '{}.pkl'.format(self.tmp_fn) pkl_path = os.path.join('tmp', filename) joblib.dump(self.estimator, pkl_path) # Port estimator: cmd = 'python -m sklearn_porter -i {} --class_name Brain'.format( pkl_path).split() subp.call(cmd) # Compare file contents: equal = filecmp.cmp(cp_src, cp_dest) self.assertEqual(equal, True)
def test_python_command_execution(self): """Test command line execution.""" subp.call(['rm', '-rf', 'tmp']) subp.call(['mkdir', 'tmp']) filename = '{}.java'.format(self.tmp_fn) cp_src = os.path.join('tmp', filename) with open(cp_src, 'w') as f: porter = Porter(self.mdl) out = porter.export(method_name='predict', class_name=self.tmp_fn) f.write(out) # $ javac tmp/Tmp.java subp.call(['javac', cp_src]) # Rename model for comparison: filename = '{}_2.java'.format(self.tmp_fn) cp_dest = os.path.join('tmp', filename) # $ mv tmp/Brain.java tmp/Brain_2.java subp.call(['mv', cp_src, cp_dest]) # Dump model: filename = '{}.pkl'.format(self.tmp_fn) pkl_path = os.path.join('tmp', filename) joblib.dump(self.mdl, pkl_path) # Port model: cmd = ['python', '-m', 'sklearn_porter', '-i', pkl_path] subp.call(cmd) # Compare file contents: equal = filecmp.cmp(cp_src, cp_dest) self.assertEqual(equal, True)
def export_model_to_js(selector, filename): """Exports the trained model to JavaScript""" porter = Porter(selector.estimator_, language='js') output = porter.export(embed_data=True) with open(os.path.join(args.out_dir, filename), mode='w+', encoding='utf8') as f: f.write('export ' + output)
def train(request): data_dir = settings.BASE_DIR + "/phishing/app/ia/datasets/" models_dir = settings.BASE_DIR + "/phishing/app/ia/models/" # data, meta = arff.loadarff(data_dir + "dataset.arff" ) data, meta = arff.loadarff(data_dir + "dataset_modificado.arff") array_data = data.tolist() dataset_array = [i[:-1] for i in array_data] target_array = [i[-1] for i in array_data] dataset = np.asarray(dataset_array, dtype=np.float64) target = np.asarray(target_array, dtype=str) clf = tree.DecisionTreeClassifier() clf = clf.fit(dataset, target) joblib.dump(clf, models_dir + 'tree_model.pkl') # build model in PHP porter = Porter(clf, language='php') output = porter.export(embedded=True) php_file = open(models_dir + 'model_php.php', "w") php_file.write(output) php_file.close() return HttpResponse('Model builded')
def _port_estimator(self, export_data=False, embed_data=False): self.estimator.fit(self.X, self.y) Shell.call('rm -rf tmp') Shell.call('mkdir tmp') with open(self.tmp_fn, 'w') as f: porter = Porter(self.estimator, language=self.LANGUAGE) if export_data: out = porter.export(class_name='Brain', method_name='foo', export_data=True, export_dir='tmp') else: out = porter.export(class_name='Brain', method_name='foo', embed_data=embed_data) f.write(out)
def _port_model(self): self.mdl.fit(self.X, self.y) subp.call(['rm', '-rf', 'tmp']) subp.call(['mkdir', 'tmp']) with open(self.tmp_fn, 'w') as f: porter = Porter(self.mdl, language=self.LANGUAGE) out = porter.export(class_name='Brain', method_name='foo') f.write(out)
def _port_estimator(self): self.estimator.fit(self.X, self.y) Shell.call('rm -rf tmp') Shell.call('mkdir tmp') filename = self.tmp_fn + '.rb' path = os.path.join('tmp', filename) with open(path, 'w') as f: porter = Porter(self.estimator, language=self.LANGUAGE) out = porter.export(class_name='Brain', method_name='foo') f.write(out)
def _port_model(self): self.mdl.fit(self.X, self.y) subp.call('rm -rf tmp'.split()) subp.call('mkdir tmp'.split()) filename = self.tmp_fn + '.rb' path = os.path.join('tmp', filename) with open(path, 'w') as f: porter = Porter(self.mdl, language=self.LANGUAGE) out = porter.export(class_name='Brain', method_name='foo') f.write(out)
def main(): args = parse_args(sys.argv[1:]) # Check input data: pkl_file_path = str(args.get('input')) if not isfile(pkl_file_path): exit_msg = 'No valid estimator in pickle ' \ 'format was found at \'{}\'.'.format(pkl_file_path) sys.exit('Error: {}'.format(exit_msg)) # Load data: estimator = joblib.load(pkl_file_path) # Determine the target programming language: language = str(args.get('language')) # with default language languages = ['c', 'java', 'js', 'go', 'php', 'ruby'] for key in languages: if args.get(key): # found explicit assignment language = key break # Define destination path: dest_dir = str(args.get('to')) if dest_dir == '' or not isdir(dest_dir): dest_dir = pkl_file_path.split(sep) del dest_dir[-1] dest_dir = sep.join(dest_dir) # Port estimator: try: class_name = args.get('class_name') method_name = args.get('method_name') with_export = bool(args.get('export')) with_checksum = bool(args.get('checksum')) porter = Porter(estimator, language=language) output = porter.export(class_name=class_name, method_name=method_name, export_dir=dest_dir, export_data=with_export, export_append_checksum=with_checksum, details=True) except Exception as exception: # Catch any exception and exit the process: sys.exit('Error: {}'.format(str(exception))) else: # Print transpiled estimator to the console: if bool(args.get('pipe', False)): print(output.get('estimator')) sys.exit(0) only_data = bool(args.get('data')) if not only_data: filename = output.get('filename') dest_path = dest_dir + sep + filename # Save transpiled estimator: with open(dest_path, 'w') as file_: file_.write(output.get('estimator'))
def _port_model(self): self.mdl.fit(self.X, self.y) subp.call(['rm', '-rf', 'tmp']) subp.call(['mkdir', 'tmp']) filename = self.tmp_fn + '.java' path = os.path.join('tmp', filename) with open(path, 'w') as f: porter = Porter(self.mdl, language=self.LANGUAGE) out = porter.export(class_name='Brain', method_name='foo') f.write(out) # $ javac temp/Tmp.java subp.call(['javac', path])
def _port_estimator(self): self.estimator.fit(self.X, self.y) Shell.call('rm -rf tmp') Shell.call('mkdir tmp') path = os.path.join('.', 'tmp', self.tmp_fn + '.go') output = os.path.join('.', 'tmp', self.tmp_fn) with open(path, 'w') as f: porter = Porter(self.estimator, language=self.LANGUAGE) out = porter.export(class_name='Brain', method_name='foo') f.write(out) cmd = 'go build -o {} {}'.format(output, path) Shell.call(cmd)
def _port_model(self): self.mdl.fit(self.X, self.y) subp.call('rm -rf tmp'.split()) subp.call('mkdir tmp'.split()) path = os.path.join('.', 'tmp', self.tmp_fn + '.go') output = os.path.join('.', 'tmp', self.tmp_fn) with open(path, 'w') as f: porter = Porter(self.mdl, language=self.LANGUAGE) out = porter.export(class_name='Brain', method_name='foo') f.write(out) cmd = 'go build -o {} {}'.format(output, path) subp.call(cmd.split())
def _port_estimator(self, export_data=False): self.estimator.fit(self.X, self.y) subp.call('rm -rf tmp'.split()) subp.call('mkdir tmp'.split()) filename = self.tmp_fn + '.java' path = os.path.join('tmp', filename) with open(path, 'w') as f: porter = Porter(self.estimator, language=self.LANGUAGE) if export_data: out = porter.export(class_name='Brain', method_name='foo', export_data=True, export_dir='tmp') else: out = porter.export(class_name='Brain', method_name='foo') f.write(out) if export_data: cmd = 'javac -cp ./gson.jar {}'.format(path).split() subp.call(cmd) else: subp.call(['javac', path])
def _port_estimator(self): self.estimator.fit(self.X, self.y) Shell.call('rm -rf tmp') Shell.call('mkdir tmp') filename = self.tmp_fn + '.c' path = os.path.join('tmp', filename) with open(path, 'w') as f: porter = Porter(self.estimator, language=self.LANGUAGE) out = porter.export(class_name='Brain', method_name='foo') f.write(out) # $ gcc temp/tmp.c -o temp/tmp cmd = 'gcc {} -std=c99 -lm -o tmp/{}'.format(path, self.tmp_fn) Shell.call(cmd)
def _port_model(self): self.mdl.fit(self.X, self.y) subp.call(['rm', '-rf', 'tmp']) # $ mkdir tmp subp.call(['mkdir', 'tmp']) filename = self.tmp_fn + '.c' path = os.path.join('tmp', filename) with open(path, 'w') as f: porter = Porter(self.mdl, language='c') out = porter.export(class_name='Brain', method_name='foo') f.write(out) # $ gcc temp/tmp.c -o temp/tmp subp.call(['gcc', path, '-lm', '-o', 'tmp/' + self.tmp_fn])
def _port_model(self): self.mdl.fit(self.X, self.y) subp.call('rm -rf tmp'.split()) subp.call('mkdir tmp'.split()) filename = self.tmp_fn + '.c' path = os.path.join('tmp', filename) with open(path, 'w') as f: porter = Porter(self.mdl, language='c') out = porter.export(class_name='Brain', method_name='foo') f.write(out) # $ gcc temp/tmp.c -o temp/tmp cmd = 'gcc {} -std=c99 -lm -o tmp/{}'.format(path, self.tmp_fn) subp.call(cmd.split())
def main(argv): path = argv[0] data = pd.read_csv(path) mode = argv[1] if mode not in ['stats', 'code']: print('Invalid mode: {}'.format(mode)) sys.exit(1) e_vars = argv[2:] if mode == 'code' and len(e_vars) != 1: print('Can only export code for one model') sys.exit(2) if len(e_vars) == 0: e_vars = output_vars(data) for var in e_vars: xs, ys = input_data(data), output_var(data, var) accs = [] max_a = 0 for s in seeds(): train_xs, test_xs, train_ys, test_ys = train_test_split( xs, ys, test_size=0.33, random_state=s) mod = model(s).fit(train_xs, train_ys) acc = np.sum(mod.predict(test_xs) == test_ys) / float(len(test_ys)) if acc > max_a: best_mod = mod max_a = acc accs.append(acc) if mode == 'stats': print("{}: {:.2f}%".format(var, 100 * sum(accs) / len(accs))) if mode == 'code': func_name = 'predict_{}'.format(var) port = Porter(best_mod, language='c') code = port.export(embed_data=True) code = code.replace('int predict_', 'static int predict_') code = code.replace('predict', func_name) code = "\n".join(code.split('\n')[:-10]) print(code, file=sys.stdout) header_line = 'extern "C" int {} (float features[]);'.format( func_name) print(header_line, file=sys.stderr)
def port_model(model, name="rf_from_sklearn"): if isinstance(model, XGBClassifier): model._Booster.save_model(name) else: porter = Porter(model, language='c') output = porter.export(embed_data=True) name = name + '.cpp' with open(name, "w") as text_file: text_file.write(output) statinfo = os.stat(name) print(bcolors.OKBLUE + f'model {name} saved, with size {statinfo.st_size} in bytes: ' + bcolors.ENDC)
def _save_model_to_java(model): from sklearn_porter import Porter cpm_filename = "cp_xnn/MLPClassifier.java" print("preper new trained model {}".format(cpm_filename)) if os.path.isfile(cpm_filename): os.unlink(cpm_filename) porter = Porter(model, language='java') output = porter.export() with open(cpm_filename, 'w+') as file: n = file.write(output) print("traned model saved in c: {} len: {}, wlen: {} ".format( cpm_filename, len(output), n)) if not os.path.isfile(cpm_filename): print("Error: no training model saved") sys.exit(0)
def classify(X_test, X_train, base_dir, samples_test, y_test, y_train, model, model_alias): base_dir = os.path.join(base_dir, 'features') file_helper.guarantee_path_preconditions(base_dir) resnet50features_train = extract_features(X_train, base_dir, 'train', model, model_alias) resnet50features_test = extract_features(X_test, base_dir, 'test', model, model_alias) print('extract featuers done, reshaping') resnet50features_train = np.reshape(resnet50features_train, (resnet50features_train.shape[0], -1)) resnet50features_test = np.reshape(resnet50features_test, (resnet50features_test.shape[0], -1)) print('begin fit') # top_model = OneVsRestClassifier(BaggingClassifier(SVC(verbose=False), n_jobs=-1)) top_model = SVC(verbose=False, C=1., kernel='rbf', gamma=0.001) top_model.fit(resnet50features_train, y_train) joblib.dump(top_model, 'fitted_print.sav', protocol=2) porter = Porter(top_model) output = porter.export(export_data=True) file = open("java_svc.java", "w") file.write(output) file.close() print('begin predict') y_pred = top_model.predict(resnet50features_test) y_pred_proba = top_model.predict_proba(resnet50features_test) output_dir = os.path.join(base_dir, model_alias) file_helper.guarantee_path_preconditions(output_dir) np.save(os.path.join(output_dir, "y_test.npy"), y_test) np.save(os.path.join(output_dir, "y_pred.npy"), y_pred) np.save(os.path.join(output_dir, "y_pred_proba.npy"), y_pred_proba) np.save(os.path.join(output_dir, "y_train.npy"), y_train) file = open(os.path.join(output_dir, "names_test.txt"), "w") file.write(str(json.dumps(samples_test)) + "\n") file.close() #hter, apcer, bpcer = evaluate_hter.evaluate_predictions(output_dir) #print('HTER: ', hter) # Computing the average accuracy acc_score = accuracy_score(y_test, y_pred) print('acc_score: ', acc_score)
def export_learner(learner, folder="learners"): '''Export transpiled java and python dill dump for a learner.''' outfolder = folder + "/" + slugify(learner.codename) if not os.path.isdir(outfolder): os.makedirs(outfolder) with open("{}/MLPClassifier.java".format(outfolder), "w") as f: porter = Porter(learner.classifier, language='java') f.write(porter.export(export_data=True, export_dir=outfolder)) with open("{}/DataMapper.java".format(outfolder), "w") as f: _export_mapper(learner, out=f) with open("{}/learner.dill".format(outfolder), "wb") as f: dill.dump( { k: v for k, v in learner.__dict__.items() if not k.startswith('_Learner__') }, f)
def randomForest(X_train, Y_train, X_test, Y_test): forest = RandomForestClassifier(max_depth=10, max_features=X_train.shape[1] // 3, min_samples_leaf=X_train.shape[0] // 20, n_estimators=1000, n_jobs=7, oob_score=False) forest.fit(X_train, Y_train) porter = Porter(forest, language='js') output = porter.export(embed_data=True) with open('RandomForestClassifier.js', 'w+') as f: f.write(output) show_forest(forest, list(X_train.columns)) predictions = pd.DataFrame(forest.predict_proba(X_test), X_test.index, ["away", "home"]) predictions['fcst'] = predictions.apply( lambda df: 0 if df.idxmax(axis=1) == 'away' else 1, axis=1) return predictions
def generate(self, destination): if not os.path.exists(destination): os.mkdir(destination) if not os.path.isdir(destination): raise Exception("The destination has to be a directory") if len(os.listdir(destination)) > 0: raise Exception("The destination directory is not empty") context = { 'gem_name': self.name, 'class_name': get_class_name(self.name), 'n_features': self.clf.n_features_, 'version': self.version, } os.mkdir(f'{destination}/lib') os.mkdir(f'{destination}/ext') os.mkdir(f'{destination}/ext/{self.name}') with open(f'{destination}/{self.name}.gemspec', 'w') as f: f.write(GEMSPEC.render(**context)) with open(f'{destination}/Rakefile', 'w') as f: f.write(RAKEFILE.render(**context)) with open(f'{destination}/lib/{self.name}.rb', 'w') as f: f.write(INIT_RB.render(**context)) with open(f'{destination}/ext/{self.name}/{self.name}.c', 'w') as f: f.write(BINDING_C.render(**context)) with open(f'{destination}/ext/{self.name}/extconf.rb', 'w') as f: f.write(EXTCONF_RB.render(**context)) porter = Porter(self.clf, language='c', method='predict') with open(f'{destination}/ext/{self.name}/sklearn_model.c', 'w') as f: f.write(porter.export(embed_data=True))
def change_model_to_c(dir, name): clf = joblib.load(dir + name) porter = Porter(clf, language='c') output = porter.export(embed_data=True) print(output) cpp_file_name = dir + name.split('.')[0]+".cpp" f = open(cpp_file_name, 'w') f.truncate() f.write(output) #写入模型转换后的原始c文件 f.close() fp = open(cpp_file_name, "r") content = fp.read() fp.close() feature_add = [] feature_add.append("#include <classifier_forest.h>") feature_add.append("void gene_used_features_index(vector<int>& needed_feature){") str_feature_index = get_feature_index(clf) feature_add.append(" needed_feature = vector<int>{" + str_feature_index + "};") feature_add.append("}\n") pos = content.find("int predict_0(float features[]) {") if pos != -1: content = content[:pos] + "\n".join(e for e in feature_add) + content[pos:] #增加特征序号向量 file = open("a.txt", "w") file.write(content) file.close() pos = content.find("int predict (float features[]) {") if pos != -1: file_add = open("predict_function.cpp", "r") content_add = file_add.read() content = content[:pos] + content_add #修改predict函数 file_add.close() file = open(cpp_file_name, "w") file.write(content) file.close()
plt.imshow(image[:, :, ::-1]); plt.axis('off'); percent = 0.8 split = int(len(data) * percent) images = [tuple_value[0] for tuple_value in data.values()] bounding_boxes = [tuple_value[1] for tuple_value in data.values()] options = dlib.simple_object_detector_training_options() options.add_left_right_image_flips = False options.C = 8 st = time.time() #detector = dlib.train_simple_object_detector(images[:split], bounding_boxes[:split], options) detector = dlib.train_simple_object_detector(images, bounding_boxes, options) print('Training Completed, Total Time taken: {:.2f} seconds'.format(time.time() - st)) #file_name = 'models/Hand_Detector_v8_c8.svm' #detector.save(file_name) win_det = dlib.image_window() win_det.set_image(detector) print("Training Metrics: {}".format(dlib.test_simple_object_detector(images[:split], bounding_boxes[:split], detector))) print("Testing Metrics: {}".format(dlib.test_simple_object_detector(images[split:], bounding_boxes[split:], detector))) porter = Porter(detector, language='C') output = porter.export(embed_data=True) print(output)
# ### Train classifier # %% from sklearn import svm clf = svm.NuSVC(gamma=0.001, kernel='rbf', random_state=0) clf.fit(X, y) # %% [markdown] # ### Transpile classifier # %% from sklearn_porter import Porter porter = Porter(clf, language='js') output = porter.export() print(output) # %% [markdown] # ### Run classification in JavaScript # %% # Save classifier: # with open('NuSVC.js', 'w') as f: # f.write(output) # Run classification: # if hash node 2/dev/null; then # node NuSVC.js 1 2 3 4 # fi
# %% from sklearn.ensemble import RandomForestClassifier clf = RandomForestClassifier(n_estimators=15, max_depth=None, min_samples_split=2, random_state=0) clf.fit(X, y) # %% [markdown] # ### Transpile classifier # %% from sklearn_porter import Porter porter = Porter(clf, language='java') output = porter.export(embed_data=True) print(output) # %% [markdown] # ### Run classification in Java # %% # Save classifier: # with open('RandomForestClassifier.java', 'w') as f: # f.write(output) # Compile model: # $ javac -cp . RandomForestClassifier.java # Run classification:
# -*- coding: utf-8 -*- from sklearn.tree import tree from sklearn.datasets import load_iris from sklearn_porter import Porter iris_data = load_iris() X = iris_data.data y = iris_data.target clf = tree.DecisionTreeClassifier() clf.fit(X, y) porter = Porter(clf, language='php') output = porter.export(embedded=True) print(output) """ <?php class DecisionTreeClassifier { public static function predict($atts) { if (sizeof($atts) != 4) { return -1; } $classes = array_fill(0, 3, 0); if ($features[3] <= 0.800000011921) { $classes[0] = 50; $classes[1] = 0; $classes[2] = 0; } else {
test_size=0.4, random_state=5) clf = MLPClassifier(activation='relu', hidden_layer_sizes=50, max_iter=500, alpha=1e-4, solver='sgd', tol=1e-4, random_state=1, learning_rate_init=.1) clf.fit(X_train, y_train) porter = Porter(clf, language='js') output = porter.export(export_data=True) print(output) """ if (typeof XMLHttpRequest === 'undefined') { var XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest; } var MLPClassifier = function(jsonFile) { this.mdl = undefined; var promise = new Promise(function(resolve, reject) { var httpRequest = new XMLHttpRequest(); httpRequest.onreadystatechange = function() { if (httpRequest.readyState === 4) { if (httpRequest.status === 200) { resolve(JSON.parse(httpRequest.responseText));
# ### Train classifier # %% from sklearn.tree import tree clf = tree.DecisionTreeClassifier() clf.fit(X, y) # %% [markdown] # ### Transpile classifier # %% from sklearn_porter import Porter porter = Porter(clf, language='java') output = porter.export(export_data=True) print(output) # %% [markdown] # ### Run classification in Java # %% # Save classifier: # with open('DecisionTreeClassifier.java', 'w') as f: # f.write(output) # Check model data: # $ cat data.json # Download dependencies:
name: metric(y_test, y_pred, average='weighted') for name, metric in metrics.items() } model_scores['estimator'] = estimator_name scores = scores.append(model_scores, ignore_index=True) scores.set_index('estimator') # %% scores.plot.bar(x='estimator', rot=20) plt.show() # %% [markdown] # ## Test classifier estimator = RandomForestClassifier(n_estimators=40, random_state=42) estimator.fit(x_train, y_train) y_pred = estimator.predict(x_test) report = classification_report(y_test, y_pred) # , output_dict=True) print(report) # report = pd.DataFrame(report) # report['estimator'] = 'decision_tree' # %% [markdown] # ## Export model # Use sklearn porter to export trained model to java script. porter = Porter(estimator, language='js') output = porter.export(embed_data=False) with open('model.js', 'w') as file: file.write(output)