def main(): args = parse_args() print('Audio socket classifier') print("Restoring model: ", args.model) mdl = model.restore(args.model) if mdl is None: print("Can't classify without an existing model") return endpoints.serverFromString(reactor, "tcp:80").listen(PubFactory(mdl, args)) reactor.run()
def main(): """ Loads an existing model, opens audio input stream, classifies input """ args = parse_args() print('Audio stream classifier') print("Restoring model: ", args.model) mdl = model.restore(args.model) if mdl is None: print("Can't classify data without an existing model.") return print("Opening audio input..") audio = pyaudio.PyAudio() stream = audio.open(format=pyaudio.paFloat32, channels=1, rate=args.sample_rate, input=True, frames_per_buffer=args.frame_size) label_a = label_b = "" if args.labels is not None: label_a = args.labels[0] label_b = args.labels[1] while True: # Peel off [frame_size] bytes from the audio stream stream_data = stream.read(args.frame_size) # Unpack the binary stream and expand data = struct.unpack("%df" % args.frame_size, stream_data) data = np.expand_dims([data], axis=2) avg = model.classify(mdl, data) steps = 20 a = int(math.ceil(avg * steps)) b = steps - a print(label_a + " [" + ("." * a) + "|" + ("." * b) + "] " + label_b + " - " + str(avg), end='\r')
import numpy as np import model from model import SET_HYPERPARAMETER SET_HYPERPARAMETER("diffLatentSpace", 6) data = np.load("./npz/diffs.npz")["arr_1"] model = model.emptyModel("DIFF_17jan_ls6_g", inputsShape=list(data.shape[1:]), use="diff", log=False) model.restore("DIFF_17jan_ls6_f") code = [[0, 0, 0, 0, 0, 0]] result = model.generate(code, data[0:1]) print(result) print(result.shape)
parser.add_argument('--connected-components-threshold', type=float, default=0.05) parser.add_argument('--width', type=int, default=768, help='input image width') parser.add_argument('--height', type=int, default=1024, help='input image height') opts = parser.parse_args() # feed data through an explicit placeholder to avoid using tf.data imgs = tf.placeholder(dtype=tf.float32, shape=(1, opts.height, opts.width, 3), name='input_imgs') # restore model model = model.Model(imgs, is_training=False, use_skip_connections=not opts.no_use_skip_connections, base_filter_size=opts.base_filter_size, use_batch_norm=not opts.no_use_batch_norm) sess = tf.Session() model.restore(sess, "ckpts/%s" % opts.run) if opts.output_label_db: db = LabelDB(label_db_file=opts.output_label_db) db.create_if_required() else: db = None if opts.export_pngs: export_dir = "predict_examples/%s" % opts.run print("exporting prediction samples to [%s]" % export_dir) if not os.path.exists(export_dir): os.makedirs(export_dir) # TODO: make this batched to speed it up for larger runs
import model import numpy as np from model import SET_HYPERPARAMETER SET_HYPERPARAMETER("contrast", 50.0) SET_HYPERPARAMETER("diffLatentSpace", 30) data = np.load("./npz/diffsWithNames.npz") goods = data["arr_0"] bads = data["arr_1"] model = model.emptyModel("generateSmoothDiff", inputsShape=list(goods[0].shape), use="diff", log=False) model.restore("28jan_ls30") trainDiffs = model.reproduce(goods) testDiffs = model.reproduce(bads) np.savez("./npz/smoothDiffsWithNames_ls30.npz", trainDiffs, testDiffs, data["arr_2"], data["arr_3"])
import numpy as np import os import jieba import config C = config.Config() jieba.load_userdict('data/coal_dict.txt') model = model.Model() labels = util.labelGenerator() compute_graph = tf.get_default_graph() with compute_graph.as_default(): model.lstm_crf_predict() #Define graph model.load_word2vec() #Initialize all variables model.restore() trainHelper = util.trainHelper( model.word2vector) #Train helper do the padding # for x,y in batchLoader: # x_raw = x.copy() # x,y,sequence_length,seq_max_len = trainHelper.generateBatch(x,y) # viterbi_sequence = model.predict(x,sequence_length,seq_max_len) # print(x_raw[0]) # print(labels.ID2label(viterbi_sequence[0][0])) while True: seg_X = input("Input your sentence :").replace(' ', '').replace('\n', '') x = list(jieba.cut(seg_X)) x_raw = x.copy() x, sequence_length, seq_max_len = trainHelper.generateData4Predict(x) x = np.reshape(x, (1, x.shape[0]))
from model import restore restore()
import model import numpy as np from model import SET_HYPERPARAMETER SET_HYPERPARAMETER("contrast", 300.0) SET_HYPERPARAMETER("learningRate", 0.0005) SET_HYPERPARAMETER("diffLatentSpace", 5) SET_HYPERPARAMETER("normalize", "individual") files = np.load("./npz/diffsWithNames.npz") goods = files["arr_0"] bads = files["arr_1"] data = np.concatenate([bads, goods]) model = model.emptyModel("5feb-ls5-inorm_f", inputsShape=list(data.shape[1:]), use="diff") model.restore("5feb-ls5-inorm_e") model.train(epoch=100, dataset=data) model.save()
import model import numpy as np from model import SET_HYPERPARAMETER SET_HYPERPARAMETER("contrast", 300.0) SET_HYPERPARAMETER("diffLatentSpace", 12) SET_HYPERPARAMETER("normalize", "individual") data = np.load("./npz/diffsWithNames.npz") goods = data["arr_0"] bads = data["arr_1"] model = model.emptyModel("generateEncode", use="diff", log=False, inputsShape=list(goods[0].shape)) model.restore("5feb-ls12-inorm_d") testEncoded = model.encode(bads) trainEncoded = model.encode(goods) np.savez("./npz/codesWithNames_inorm.npz", trainEncoded, testEncoded, data["arr_2"], data["arr_3"])
import model import numpy as np from model import SET_HYPERPARAMETER SET_HYPERPARAMETER("contrast", 50.0) SET_HYPERPARAMETER("diffLatentSpace", 8) data = np.load("./npz/diffsWithNames.npz") goods = data["arr_0"] bads = data["arr_1"] model = model.emptyModel("generateFeatures", use="diff", log=False, inputsShape=list(goods[0].shape)) model.restore("DIFF_23jan_ls8_e") testFeatures = model.getFeatures(bads) trainFeatures = model.getFeatures(goods) np.savez("./npz/featuresWithNames_ls8.npz", trainFeatures, testFeatures, data["arr_2"], data["arr_3"])
testdata.read(testpath, wikidata.char_id) webdata = data.Dataset(params) webdata.read(webpath, wikidata.char_id) print('data loaded') config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) model = model.Model(params) model.model_build(sess) print('model built') model.model_initialize(sess) print('model initialize') # model.lattice_train(wikidata, sess, 300000) # print('wiki_lattice finished') # model.train(traindata, sess, 40000) # print('all_train finished') # model.evaluate(testdata, sess) # model.train(traindata, sess, 40000) # print('train finished') # model.evaluate(testdata, sess) # print('lattice and text at the same time') model.restore('./calc_data/lattice2textlattice2divide_data/all_finished_save/model.ckpt', sess) wiki = [ x for x in open('hoge', 'r') ] for text in wiki: model.demo(text.strip(), wikidata.char_id, sess)
import model import numpy as np from model import SET_HYPERPARAMETER SET_HYPERPARAMETER("latentSpace", 1) data = np.load("./npz/dataWithNames.npz") goods = data["arr_0"] bads = data["arr_1"] model = model.emptyModel("generateDiff", inputsShape=list(goods[0].shape), log=False, use="jtekt") model.restore("4feb-ls1") trainDiffs = model.getDiff(goods) testDiffs = model.getDiff(bads) np.savez("./npz/diffsWithNames.npz", trainDiffs, testDiffs, data["arr_2"], data["arr_3"])
import os import time import tensorflow as tf import util import model import ujson as json if __name__ == "__main__": test_data = list() print('Start to process data...') with open('test.jsonlines', 'r') as f: for line in f: tmp_example = json.loads(line) test_data.append(tmp_example) print('finish processing data') os.environ["CUDA_VISIBLE_DEVICES"] = "1" config = util.initialize_from_env() model = model.KnowledgePronounCorefModel(config) with tf.Session() as session: model.restore(session) # print('we are working on NP-NP') model.evaluate(session, test_data, official_stdout=True) # model.evaluate(session) # model.evaluate_baseline_methods(session) print('end')
def main(): args = parse_args() print('Audio classifier') output_prefix = os.path.splitext(args.data_file)[0] if not os.path.exists(args.logdir): os.mkdir(args.logdir) # create / restore model mdl = None if args.model: print("Restoring model..") print(" input:", args.model) mdl = model.restore(args.model) elif model.model_exists(args.logdir) and not args.force: print("Restoring last created model..") file = model.most_recent(args.logdir) print(" input:", file) mdl = model.restore(file) if not args.predict: # training task print("Loading training data..") print(" input:", args.data_file) # load data from file X, Y = data.load_labeled_data(args.data_file) w, h = np.shape(X) print(" size: {}x{}".format(w, h)) # encode labels Y, Dict = data.encode_labels(Y) print(" labels: ", Dict) # save labels with data # model.save_labels(Dict, args.logdir + output_prefix + "_labels.txt") if mdl is None: print("Creating new model..") mdl = model.create(h) train(mdl, (X, Y), args.logdir + output_prefix, args.kfolds, args.epochs) else: # prediction task print("Loading classification data..") print(" input:", args.data_file) X = data.load_unlabeled_data(args.data_file) w, h = np.shape(X) print(" size: {}x{}".format(w, h)) if mdl is None: print("Can't classify without an existing model") return classify(mdl, X)