def output_stuff(model, history): #parameters params={ #dictionary 'filename' : filename, 'batch_size' : batch_size, 'classes' : classes, 'epochs' : epochs, 'learning_rate' : learning_rate, 'decay' : decay, 'momentum' : momentum, 'nesterov' : nesterov, 'train_datapoints' : train_datapoints, 'test_datapoints' : train_datapoints, } with open(filename+'_params.json', 'w') as fp: json.dump(params, fp) #architecture, for keras and nengo loading respectively json_string = model.to_json() open(filename+'_model.json', 'w').write(json_string) with open(filename+"_arch.json","w") as datafile: dump(arch_dict, datafile) # weights # model.save_weights(filename+'_weights.h5') #history history_file = open(filename+"_history.txt", "w") history_file.write(str(history.history)) history_file.close()
def test_file_numpy(): path = join(mkdtemp(), 'pytest-np.json') with open(path, 'wb+') as fh: dump(deepcopy(npdata), fh, compression=9) with open(path, 'rb') as fh: data2 = load(fh, decompression=True) _numpy_equality(data2)
def extract_caffe_model(model, weights, output_path): """extract caffe model's parameters to numpy array, and write them to files Args: model: path of '.prototxt' weights: path of '.caffemodel' output_path: output path of numpy params Returns: None """ net = caffe.Net(model, weights, caffe.TEST) if not os.path.exists(output_path): os.makedirs(output_path) with open(os.path.join(output_path, 'shapes.ahsf'), 'w') as shape_file: for item in net.params.items(): name, layer = item print('convert layer: ' + name) num = 0 for p in net.params[name]: name = name.replace('/', '\\') name = name if num == 0 else '{}_bias'.format(name) with open(os.path.join(output_path, '{}.ahsf'.format(name)), 'w') as outfile: dump(p.data.flatten(), outfile) shape_file.write("{} {} ".format(name, len(p.data.shape))) for dim in p.data.T.shape: shape_file.write(str(dim) + " ") shape_file.write('\n') num += 1
def test_file_numpy(): path = join(mkdtemp(), 'pytest-np.json') with open(path, 'wb+') as fh: dump(npdata, fh, compression=9) with open(path, 'rb') as fh: data2 = load(fh, decompression=True) _numpy_equality(data2)
def save_json(obj, outfile, allow_nan=True, compression=False): """Save an ssbio object as a JSON file using json_tricks""" if compression: with open(outfile, 'wb') as f: dump(obj, f, allow_nan=allow_nan, compression=compression) else: with open(outfile, 'w') as f: dump(obj, f, allow_nan=allow_nan, compression=compression) log.info('Saved {} (id: {}) to {}'.format(type(obj), obj.id, outfile))
def output_stuff(model, history): #parameters params={ #dictionary 'filename' : filename, 'batch_size' : batch_size, 'classes' : classes, 'epochs' : epochs, 'learning_rate' : learning_rate, 'decay' : decay, 'momentum' : momentum, 'nesterov' : nesterov, 'train_datapoints' : train_datapoints, 'test_datapoints' : train_datapoints, } with open(filename+'_params.json', 'w') as fp: json.dump(params, fp) #architecture, for keras and nengo loading respectively json_string = model.to_json() open(filename+'_model.json', 'w').write(json_string) #To-do:make this into a loop, or figure out how to properly extract weight information arch_dict=OrderedDict(( ('input',{'type':'input', 'input_name':'image', 'weights':None, 'biases':None, 'activation':None}), ('conv0', {'type':'Convolution2D', 'input_name':'input', 'weights':model.nodes['conv0'].get_weights()[0],'biases':model.nodes['conv0'].get_weights()[1],'activation':'relu', 'stride':1,'activities':None}), #get_outputs(model,'input','conv1',X_test[:test_datapoints]) ('maxpool0', {'type':'MaxPooling2D', 'input_name':'conv0', 'weights':None, 'activation':None, 'biases':None,'stride':None, 'pool_type':'max', 'pool_size': ps,'activities':None}), ('conv1', {'type':'Convolution2D', 'input_name':'maxpool0', 'weights':model.nodes['conv1'].get_weights()[0], 'biases':model.nodes['conv1'].get_weights()[1], 'activation':'relu', 'stride':1,'activities':None}), ('maxpool1', {'type':'MaxPooling2D', 'input_name':'conv1', 'weights':None,'activation':None, 'biases':None, 'stride':None, 'pool_type':'max', 'pool_size': ps,'activities':None}), ('flatten', {'type':'Flatten', 'input_name':'maxpool1', 'weights':None,'activation':None, 'biases':None, 'stride':1,'activities':None}), ('dense0', {'type':'Dense', 'input_name':'flatten', 'weights':model.nodes['dense0'].get_weights()[0], 'biases':model.nodes['dense0'].get_weights()[1],'activation':'relu','stride':1,'activities':None}), ('dense1', {'type':'Dense', 'input_name':'dense0', 'weights':model.nodes['dense1'].get_weights()[0],'biases':model.nodes['dense1'].get_weights()[1], 'activation':'softmax', 'stride':1,'activities':None}), ('output', {'type':'output', 'input_name':'dense1', 'weights':None, 'biases':None, 'activation':None}), )) # print ([get_outputs(model,'input',the_node,X_test[:test_datapoints]).shape for the_node in model.nodes]) # data=dumps(arch_dict) with open(filename+"_arch.json","w") as datafile: dump(arch_dict, datafile) #weights # model.save_weights(filename+'_weights.h5') #history history_file = open(filename+"_history.txt", "w") history_file.write(str(history.history)) history_file.close()
def test_compressed_to_disk(): arr = [array([[1.0, 2.0], [3.0, 4.0]])] path = join(mkdtemp(), 'pytest-np.json.gz') with open(path, 'wb+') as fh: dump(arr, fh, compression=True, properties={'ndarray_compact': True})
import argparse import os import numpy as np from PIL import Image from json_tricks.np import dump parser = argparse.ArgumentParser('Extract pixel and shape from an input image') parser.add_argument(metavar='image_path', dest='image', help='Path to model.prototxt') args = parser.parse_args() image_path = args.image output_path = './image' if not os.path.exists(output_path): os.makedirs(output_path) im = Image.open(image_path) im_data = np.asarray(im) with open(os.path.join(output_path, 'image.ahsf'), 'w') as mean_file: dump(im_data.flatten(), mean_file) with open(os.path.join(output_path, 'image_shape.ahsf'), 'w') as shape_file: shape = im_data[None] if len(im_data.shape) == 2 else im_data.shape for shape in im_data[None].T.shape: shape_file.write(str(shape) + ' ')
cv2.imshow('SURF Match + Inliers', img3) cv2.waitKey(0) cv2.destroyAllWindows() #Output CSV if arguments.o: outFolder = "surf_experiments/" util.initWrite() util.writeFile(kp2, d2, img1Path, img2Path, inliers, percent, len(kp2)) util.closeWrite(outFolder, img2Path, 'surf') print("\n#### Ranking ####") rList = np.sort(resList, order='inliers')[::-1] for bestPair in range(10): print '#%d: %s -> Inliers: %d' % (bestPair + 1, rList[bestPair][1], rList[bestPair][2]) print '{percent:.2%}'.format(percent=rList[bestPair][3]) ## # Results and Experimental Values Logging # ## if arguments.o: subprocess.check_output([ "sed -e '!d' surf_experiments/surf*.csv >> surf_experiments/surf_" + img1Path[8:-4] + "_merge.csv" ], shell=True) jList = rList.reshape((n, 1)) with open('results.json', 'w') as resultFile: dump({'Results': jList}, resultFile)
if rList[i][1] in imgClassList: rankedClassList[idx][0] = i + 1 rankedClassList[idx][1] = rList[i][1] rankedClassList[idx][2] = rList[i][2] rankedClassList[idx][3] = rList[i][3] print "%d->%s (%d inliers) " % (idx, rList[i][1], rList[i][2]) print '{percent:.2%}'.format(percent=rList[i][3]) idx += 1 classRank = rankedClassList.reshape(15, 1) ## # Results and Experimental Values Logging # ## if arguments.wdata: try: cmd = "sed -e '!d' sift_experiments/sift*.csv >> sift_experiments/" + img1Path[: -4] + "/data_merged.csv" rm = "rm -r sift_experiments/*.csv sift_experiments/*.xls" subprocess.check_output([cmd], shell=True) subprocess.check_output([rm], shell=True) jList = rList.reshape((n, 1)) with open(expDir + "/" + img1Path[:-4] + '/results.json', 'w') as resultFile: dump({'Results': jList}, resultFile) with open(expDir + "/" + img1Path[:-4] + '/classRank.json', 'w') as cRank: dump({'ClassRank': classRank}, cRank) except (RuntimeError, TypeError, NameError): print("Internal Structure Error")
def to_json(features, filepath=None): if filepath is None: return json.dumps(features, indent=4) else: json.dump(features, open(filepath, 'w'), indent=4)
import h5py import numpy as np from json_tricks.np import dump with h5py.File("model_weights.h5") as hf: dataset_names = ('/layer_0/param_0', '/layer_0/param_1') weights = {name: np.array(hf.get(name)) for name in dataset_names} fp = open("weights.json", 'wt') dump(weights, fp, indent=4) fp.close()
from json_tricks.np import dump parser = argparse.ArgumentParser( 'Extract pixel and shape from a mean mean.binaryproto file') parser.add_argument(metavar='mean_path', dest='mean', help='Path to model.prototxt') args = parser.parse_args() output_path = './mean' if not os.path.exists(output_path): os.makedirs(output_path) blob = caffe.proto.caffe_pb2.BlobProto() data = open(args.mean, 'rb').read() blob.ParseFromString(data) arr = np.array(caffe.io.blobproto_to_array(blob)) # Binaryproto has shape of (1, 1, 28, 28), arr[0] to get (1, 28, 28) out = arr[0] with open(os.path.join(output_path, 'mean.ahsf'), 'w') as mean_file: dump(out.flatten(), mean_file) with open(os.path.join(output_path, 'mean_shape.ahsf'), 'w') as shape_file: for shape in out.T.shape: shape_file.write(str(shape) + ' ')