def setUp(self): with open(paths.TESTDATA_PATH + 'Dasher.fa') as f: self.b = batch.Batch('Gibson', 'Ctag', 'Lox-ZeoR', target_text='PAS_chr2-1_0666', tag_text=f.read())
def setUp(self): with open(paths.TESTDATA_PATH + 'pThi11.fa') as f: self.b = batch.Batch('Gibson', 'Overexpression', 'Lox-ZeoR', target_text='PAS_chr2-1_0666', tag_text=f.read())
def deepxi_args(args): ## DEPENDANT OPTIONS args.model_path = args.model_path + '/' + args.ver # model save path. args.train_s_path = args.set_path + '/train_clean_speech' # path to the clean speech training set. args.train_d_path = args.set_path + '/train_noise' # path to the noise training set. args.val_s_path = args.set_path + '/val_clean_speech' # path to the clean speech validation set. args.val_d_path = args.set_path + '/val_noise' # path to the noise validation set. args.out_path = args.out_path + '/' + args.ver + '/' + 'e' + str( args.epoch) # output path. args.Nw = int(args.fs * args.Tw * 0.001) # window length (samples). args.Ns = int(args.fs * args.Ts * 0.001) # window shift (samples). args.NFFT = int(pow(2, np.ceil(np.log2( args.Nw)))) # number of DFT components. ## A PRIORI SNR IN DB STATS mu_mat = spio.loadmat(args.stats_path + '/mu.mat') # mean of a priori SNR in dB from MATLAB. args.mu = tf.constant(mu_mat['mu'], dtype=tf.float32) sigma_mat = spio.loadmat( args.stats_path + '/sigma.mat') # standard deviation of a priori SNR in dB from MATLAB. args.sigma = tf.constant(sigma_mat['sigma'], dtype=tf.float32) ## DATASETS if args.train: ## TRAINING AND VALIDATION CLEAN SPEECH AND NOISE SET args.train_s_list = batch.Train_list( args.train_s_path, '*.wav', 'clean') # clean speech training list. args.train_d_list = batch.Train_list(args.train_d_path, '*.wav', 'noise') # noise training list. if not os.path.exists(args.model_path): os.makedirs(args.model_path) # make model path directory. args.val_s, args.val_s_len, args.val_snr, _ = batch.Batch( args.val_s_path, '*.wav', list(range(args.min_snr, args.max_snr + 1))) # clean validation waveforms and lengths. args.val_d, args.val_d_len, _, _ = batch.Batch( args.val_d_path, '*.wav', list(range(args.min_snr, args.max_snr + 1))) # noise validation waveforms and lengths. ## INFERENCE if args.infer: args.test_x, args.test_x_len, args.test_snr, args.test_fnames = batch.Batch( args.test_x_path, '*.wav', []) # noisy speech test waveforms and lengths. return args
def do_train(data, data_val, data_test, **classifier_parameters): ''' return ?? ''' batch_maker = batch.Batch(data, isShuffle=True, seg_window=15, seg_hop=5) return do_train_batch(batch_maker, data_val, data_test, **classifier_parameters)
def download(): if request.method == 'POST': batch_kw = {k:v for k,v in request.form.items() if not k.startswith('start')} b = batch.Batch(**batch_kw) shutil.rmtree(batch.paths.OUTPUT_PREFIX) os.mkdir(batch.paths.OUTPUT_PREFIX) b.write_primers_csv() b.assign_numbers(request.form['start_oligos']) b.write_idt_csv() b.write_plasmids_zip(request.form['start_plasmids']) b.write_loci_zip() rows = b.list_operations() return render_template('download.html', files=files, rows=rows) return render_template('index.html', **index_kw)
def calc_error(data_test, predict): ''' return error, cost on that set''' b = batch.Batch(data_test, max_batchsize=5000, seg_window=15, seg_hop=5) err = 0 cost_val = 0 eps = 1e-10 for (x, y_lab, _) in b: decision = predict(x.reshape((x.shape[0], -1)).astype('float32')) + eps pred_label = np.argmax(decision, axis=-1) y = onehot(y_lab) cost_val += -np.sum(y * np.log(decision)) err += np.sum(np.expand_dims(pred_label, axis=1) != y_lab) assert (not np.isnan(cost_val)) err = err / float(len(b.index_bkup)) cost_val = cost_val / len(b.index_bkup) return err, cost_val
def create_batch_operation(self, batch_flag=True, **kwargs): """ Operation that start a batch operation and returns a batch object. batch_flag(bool): Flag that denotes if a command is a batch command or not. Also used to indicate to the client whether to wait for a response or not for a START_BATCH, END_BATCH, or ABORT_BATCH. Returns: Batch object """ if not self.is_connected and self.auto_reconnect_count < 1: self._client_error_callback(kinetic_pb2.Command.Status.NOT_ATTEMPTED, "call connect() before the operation") else: try: next_batch_id = kwargs['batch_id'] except KeyError: next_batch_id = self._batch_id.next() kwargs['batch_id'] = next_batch_id finally: self.start_batch(batch_flag=batch_flag, **kwargs) return batch.Batch(self, next_batch_id)
def schedule_batch(batch_id, batch_run_id, batch_name, region, team_name, input_file, supplier_id, user_id, scheduled_date, scheduled_time): # Code below is responsible for batch scheduling --- Directly taken from Backbone module try: status.init( ) # This will initialize all the status variables and global variables main_log_file = status.current_path + '/logs/main_logs.log' mainlog = log_Writer(main_log_file) except Exception as e: print(e) sys.exit(1) try: print('-------------- Starting Batch --------------') # current_path = os.path.dirname(os.path.abspath(__file__)) properties_file = status.current_path + '/properties.pbf' property = essential.read_properties(properties_file) status.property = property mainlog.write('Batch Creating - ' + str(team_name) + ' - ' + str(batch_name) + ' - ' + str(datetime.datetime.now())) # Writing main logs except Exception as e: print(e) sys.exit(1) # base_address = status.current_path supplier_name, script_file_path, proxy_file_path, server_details, num_of_attempts, time_out = get_batch_data( supplier_id) batch_to_run = batch.Batch() batch_to_run.start(batch_id, batch_run_id, batch_name, region, team_name, user_id, input_file, script_file_path, proxy_file_path, server_details, num_of_attempts, time_out)
learning_rate = 1e-4 training_epochs = 200 batch_size = 100 num_classes = 10 total_batch = (int)(train_size / batch_size) keep_prob = 0.25 sess = tf.Session() m1 = model.Model(sess, "m1") m1.set_model(learning_rate) m1.set_dropout(keep_prob) m1.set_size(num_x_features, num_y_features, num_classes) m1.set_img(32, 32, 3) m1._build_net() bat = batch.Batch(trainX, trainY) sess.run(tf.global_variables_initializer()) avg_cost = 0 sum_cost = 0 avg_accuracy = 0 sum_accuracy = 0 print('Learning Started!') with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess: for epoch in range(training_epochs): sum_accuracy = 0 sum_cost = 0 for step in range(total_batch): batch_xs, batch_ys = bat.next_batch(batch_size)
import scipy import scipy.io import numpy as np import batch #print 'started' trainSet = scipy.io.loadmat('TrainBatch.mat')['arr'] testSet = scipy.io.loadmat('TestBatch.mat')['arr'] #print 'data loaded' train = batch.Batch(trainSet) #print 'train batched' test = batch.Batch(testSet) #print 'data batched' print trainSet.shape print trainSet[0][0].shape
model = "_tr" # or "" or "_nn" where nn is hybrid # TODO change model if __name__ == "__main__": max_freq = int(sys.argv[1]) lhs, rel, rhs = batch.load_labeled_entities(io.open("../data/prescription-sparse2-train.txt")) # sparse 2 is by different train,valid,test ratio chardict, charcount = batch.build_char_dictionary(lhs) n_char = len(chardict.keys()) + 1 lhs_dict, lhs_count = batch.build_entity_dictionary(lhs) n_lhs = len(lhs_dict.keys()) rel_dict, rel_count = batch.build_entity_dictionary(rel) n_rel = len(rel_dict.keys()) rhs_dict, rhs_count = batch.build_entity_dictionary(rhs) n_rhs = len(rhs_dict.keys()) lhs_s, rel_s, rhs_s = batch.load_labeled_entities(io.open("../data/prescription-sparse2-rare-{}-test.txt".format(max_freq))) test_iter = batch.Batch(lhs_s, rel_s, rhs_s, batch_size=N_BATCH) m = charLM(n_char, n_lhs + 1, n_rel, n_rhs) # emb_dim = WDIM by default m.param = load_params_shared("temp{}/best_model.npz".format(model)) ''' num_param = 0 for kk in m.param.keys(): print m.param[kk].get_value().shape print m.param[kk].get_value().size num_param += m.param[kk].get_value().size print "total number of param: {}".format(num_param) quit() ''' # compute example predictions
def setUp(self): self.b = batch.Batch('SOE-PCR', 'Knockout', 'Lox-ZeoR', 'PAS_chr2-1_0666') self.d = self.b.designs[0]
x_data = dataset[:, 2:-4] for i in range(len(y_data)): y_data[i] = 1 if y_data[ i] == -1.0 else 0 # if UNLOADED, 1 if y_data[i] == -1.0 else 0 # if LOADED, 1 if y_data == 1.0 else 0 y_data = np.reshape(y_data, [-1, 1]) print(x_data.shape) print(y_data.shape) X = tf.placeholder(tf.float32) Y = tf.placeholder(tf.float32) with tf.Session() as sess: modelmaker = model.NetworkModel() bat = batch.Batch(x_data[0:50000, :], y_data[0:50000, :]) Train, Hypothesis_prob, Cost, Saver = modelmaker.get_model( sess, MODEL_DIR, X, Y) Prediction = tf.floor(Hypothesis_prob + 0.5) Correct = tf.equal(Prediction, Y) Accuracy = tf.reduce_mean(tf.cast(Correct, tf.float32)) for step in range(15000): x_batch, y_batch = bat.next_batch(128) #y_batch = np.reshape(y_batch, [-1,1]) sess.run(Train, feed_dict={X: x_batch, Y: y_batch}) if step % 100 == 0: print( step,
def setup(self): # CONFIG PARAMETERS try: with open('config.ini') as self._cfg: self._config.read_file(self._cfg) self._hname = self._config.get("TestHost", "hostname") self._dd_dir = self._config.get("Dir", "dd_dir") self._src = self._config.get("Sources", "sources") self._a_filters = (self._config.get("Filters", "filters")).split(', ') self._n_batch = self._config.get("Batch", "n_batch") if self._n_batch: self._n_batch = int(self._n_batch) print("\n[NOTICE] Check your settings in 'config.ini' file. ") except (IOError, OSError): print("[ERROR] Unable to find config file (config.ini)") sys.exit() # NETWORK TEST try: print("\n>>Test network connection...") self._hostname = hostname.Test_hostname() self._hostname.set_hostname(self._hname) self._hostname.get_hostname() self._hostname.test_conn() except (IOError, OSError): print("[ERROR] Unable to get test_hostname.") # Logging... self._setup_log = utility.Logger() self._setup_log.set_file('log.txt') self._setup_log.get_file() # BEGIN SETUP ######################### try: # DIRECTORY self._dd_directory = directory.Download_dir() self._dd_directory.set_directory(self._dd_dir) self._dd_directory = self._dd_directory.get_directory() # BATCH FOR TASK POOL self._batch = batch.Batch() self._batch.set_batch(self._n_batch) self._batch = self._batch.get_batch() # FILTERS self._filters = filters.Filters() self._filters.set_filters(self._a_filters) self._filters = self._filters.get_filters() # SOURCES # There is no need to set a default action. If the source file doesn't exist or it's corrupted, # the script will terminate its operation instantly. print("\n>>Source File:") print(self._src) self._sources = sources.Sources() self._sources.set_urls(self._src) self._sources = self._sources.get_urls() print("\n>>Options") print("Download directory: ", self._dd_dir) # If filters is empty in config.ini, don't apply filters. if self._a_filters[0] == '' and len(self._a_filters) == 1: self._filtered_urls = self._sources print("No filter rules applied.\n") else: print("Filter rules: ", self._a_filters, "\n") self._filtered_urls = utility.Filtered_urls() self._filtered_urls = self._filtered_urls.apply_filters( self._a_filters, self._sources) # for logging... self._conf_log = "\n[CONFIG] [time: " + str( (datetime.now() ).strftime('%Y/%m/%d %H:%M:%S')) + "] Setup complete. [OK]" self._setup_log.logger(self._conf_log) except BaseException: print("[ERROR] Unable to complete script setup.") self._fail_conf_log = "\n[ERROR] [time: " + str( (datetime.now() ).strftime('%Y/%m/%d %H:%M:%S')) + "] Setup Failed!!!" self._setup_log.logger(self._fail_conf_log) sys.exit() return self._filtered_urls, self._dd_directory, self._n_batch
GL.glViewport(0, 0, 640, 480) animation = loadSheet( 'animation.png', 20, ) projectionMatrix = numpy.array( ((2 / 320, 0, 0, 0), (0, -2 / 240, 0, 0), (0, 0, 1, 0), (0, 0, 0, 1)), numpy.float32) viewMatrix = numpy.array( ((1, 0, 0, 0), (0, 1, 0, 0), (0, 0, 1, 0), (-160, -120, 0, 1)), numpy.float32) batch.setup_shaders(vpMatrix=numpy.dot(viewMatrix, projectionMatrix)) renderer = batch.Batch() running = True imageIndex = 0 clock = pg.time.Clock() while running: imageIndex = (imageIndex + 0.1) % len(animation) GL.glClearColor(0, 0, 0, 1) GL.glClear(GL.GL_COLOR_BUFFER_BIT) renderer.begin() for i in range(710): renderer.draw(animation[int(imageIndex)], 10 + (20 * i) % 300, 10 + 20 * int(20 * i / 300))
def setUp(self): self.b = batch.Batch('Gibson', 'Knockout', 'Lox-ZeoR', 'PAS_chr2-1_0666')
print(len(image_x)) num_data = settings["num_data"] ##訓練用データ数 num_test = settings["num_test"] train_x = image_x[:num_data] test_x = image_x[num_data:num_data + num_test] train_t = image_t[:num_data] test_t = image_t[num_data:num_data + num_test] #UNETを初期化しています。 unet = unet.UNET(settings["input_sizex"], settings["input_sizey"], settings["num_class"], depth=settings["depth"], layers_default=settings["layers_default"]) Batch_x = batch.Batch(train_x) Batch_t = batch.Batch(train_t) Batch_num = settings["Batch_num"] ##バッチ数 i = 0 for _ in range(settings["learning_times"]): ##学習回数 i += 1 batch_x = Batch_x.next_batch(Batch_num) batch_t = Batch_t.next_batch(Batch_num) unet.sess.run(unet.train_step, feed_dict={ unet.x: batch_x, unet.t: batch_t, unet.keep_prob: settings["keep_prob"] }) if i % 10 == 0:
n_lhs = len(lhs_dict.keys()) batch.save_dictionary(lhs_dict,lhs_count,'%s/lhs_dict.pkl' % save_path) # build dictionary for relations rel_dict, rel_count = batch.build_entity_dictionary(rel) batch.save_dictionary(rel_dict, rel_count, '%s/rel_dict.pkl' % save_path) n_rel = len(rel_dict.keys()) # this tells number of triples in different relations # build dictionary for right hand side entities rhs_dict, rhs_count = batch.build_entity_dictionary(rhs) batch.save_dictionary(rhs_dict, rhs_count, '%s/rhs_dict.pkl' % save_path) n_rhs = len(rhs_dict.keys()) # batches train_iter = batch.Batch(lhs, rel, rhs, batch_size=N_BATCH) valid_iter = batch.Batch(lhs_v, rel_v, rhs_v, batch_size=N_BATCH) test_iter = batch.Batch(lhs_s, rel_s, rhs_s, batch_size=N_BATCH) print("Building model...") m = charLM(n_char, n_lhs + 1, n_rel, n_rhs) # emb_dim = WDIM by default # Training print("Training...") uidx = 0 start = time.time() valcosts = [] min_mean_rank = float("inf") try: for epoch in range(N_EPOCH): n_samples = 0
def setUp(self): self.b = batch.Batch('Gibson', 'Knockout', 'Lox-ZeoR', 'PAS_chr1-4_0370\nPAS_chr3_1028')
def setUp(self): with open(paths.TESTDATA_PATH + 'ku70_precise.fa') as f: self.b = batch.Batch('Gibson', 'Knockout', 'Lox-ZeoR', target_text=f.read())
def setUp(self): self.b = batch.Batch('Gibson', 'Knockdown', 'Lox-ZeoR', 'PAS_chr1-4_0370')
def do_train(data, data_val, data_test, **classifier_parameters): ''' input ------- data: {label:np.array(features)} classifier_param: {n_layers:3,...} output ------ model_param = {structure: {n_layers: int, n_dense: int...}, params:lasagne.layers.get_all_params(l_out) } ''' import time batch_maker = batch.Batch(data, isShuffle=True, seg_window=classifier_parameters['max_length'], seg_hop=classifier_parameters['max_length'] / 2) b_v = batch.Batch(data_val, isShuffle=True, seg_window=classifier_parameters['max_length'], seg_hop=classifier_parameters['max_length'] / 2) b_t = batch.Batch(data_test, isShuffle=True, seg_window=classifier_parameters['max_length'], seg_hop=classifier_parameters['max_length'] / 2) input_var = T.tensor3('input') mask = T.matrix('mask') target_output = T.matrix('target_output') network, layers = build(input_var, mask, **classifier_parameters) eps = 1e-10 loss_train = cost( lasagne.layers.get_output(network, deterministic=False) + eps, target_output, mask) loss_eval = cost( lasagne.layers.get_output(network, deterministic=True) + eps, target_output, mask) all_params = lasagne.layers.get_all_params(network) updates = lasagne.updates.adadelta(loss_train, all_params, learning_rate=1.0) #updates = lasagne.updates.momentum(loss_train , all_params, #learning_rate, momentum) pred_fun = lasagne.layers.get_output(network, deterministic=True) train = theano.function([input_var, target_output, mask], loss_train, updates=updates) #compute_cost = theano.function([input_var, target_output, mask],loss_eval) predict = theano.function([input_var, mask], pred_fun) #theano.config.warn_float64='pdb' print "start training" #err, cost_test = calc_error(data_val,predict) epoch = 0 no_best = 10 best_cost = np.inf best_epoch = epoch model_params = [] # TO REMOVE #model_params.append(lasagne.layers.get_all_param_values(network)) while epoch < 100: start_time = time.time() cost_train = 0 for _, (x, y, m) in enumerate(batch_maker): x = x.astype('float32') m = m.astype('float32') y = onehot(y) y = y.astype('float32') assert (not np.any(np.isnan(x))) cost_train += train(x, y, m) * x.shape[0] #*x .shape[1] assert (not np.isnan(cost_train)) cost_train = cost_train / len(batch_maker.index_bkup) err_val, cost_val = calc_error(b_v, predict) err_test, cost_test = calc_error(b_t, predict) #cost_val, err_val = 0, 0 #pdb.set_trace() end_time = time.time() is_better = False if cost_val < best_cost: best_cost = cost_val best_epoch = epoch is_better = True if is_better: print "epoch: {} ({}s), training cost: {}, val cost: {}, val err: {}, test cost {}, test err: {}, New best.".format( epoch, end_time - start_time, cost_train, cost_val, err_val, cost_test, err_test) else: print "epoch: {} ({}s), training cost: {}, val cost: {}, val err: {}, test cost {}, test err: {}".format( epoch, end_time - start_time, cost_train, cost_val, err_val, cost_test, err_test) sys.stdout.flush() model_params.append(lasagne.layers.get_all_param_values(network)) #check_path('dnn') #save_data('dnn/epoch_{}.autosave'.format(epoch), (classifier_parameters, model_params[best_epoch])) #savename = os.path.join(modelDir,'epoch_{}.npz'.format(epoch)) #files.save_model(savename,structureDic,lasagne.layers.get_all_param_values(network)) if epoch - best_epoch >= no_best: ## Early stoping print "Training stops, best epoch is {}".format(best_epoch) break epoch += 1 return (classifier_parameters, model_params[best_epoch])
def do_train(data, data_val, data_test, **classifier_parameters): ''' return ?? ''' batch_maker = batch.Batch(data, isShuffle = True, seg_window=100, seg_hop=20 ,max_batchsize=50) num_epochs = 10000 #num_epochs = 3 # prepare theano variables for inputs and targets input_var = T.tensor4('inputs') target_var = T.imatrix('targets') # ?? network = build(input_var,**classifier_parameters) # create a loss expression for training prediction = lasagne.layers.get_output(network) loss = lasagne.objectives.categorical_crossentropy(prediction,target_var) loss = loss.mean() # create update expressions for training: SGD with momentum params = lasagne.layers.get_all_params(network, trainable=True) updates = lasagne.updates.adadelta( loss, params, learning_rate=1) train = theano.function([input_var, target_var],loss , updates=updates) # create a loss expression for validation/testing # 'deterministic' disable dropout layers test_prediction = lasagne.layers.get_output(network, deterministic=True) test_loss = lasagne.objectives.categorical_crossentropy(test_prediction, target_var) test_loss = test_loss.mean() predict = theano.function( [input_var], lasagne.layers.get_output(network, deterministic=True)) # training loop print("Starting training...") epoch = 0 #no_best = 1 no_best = 10 best_cost = np.inf best_epoch = epoch model_params = [] # TO REMOVE #model_params.append(lasagne.layers.get_all_param_values(nnet)) while epoch < num_epochs: try: start_time = time.time() cost_train = 0 for _, (x ,y ,_) in enumerate(batch_maker): x = reshape(x) y=onehot(y) assert(not np.any(np.isnan(x))) cost_train+= train(x, y) *x .shape[0]#*x .shape[1] assert(not np.isnan(cost_train)) cost_train = cost_train/ len(batch_maker.index_bkup) err_val, cost_val = calc_error(data_val,predict) err_test, cost_test = calc_error(data_test,predict) #import pdb; pdb.set_trace() #cost_val, err_val = 0, 0 #pdb.set_trace() end_time = time.time() is_better = False if cost_val < best_cost: best_cost =cost_val best_epoch = epoch is_better = True if is_better: print "epoch: {} ({}s), training cost: {}, val cost: {}, val err: {}, test cost {}, test err: {}, New best.".format(epoch, end_time-start_time, cost_train, cost_val, err_val, cost_test, err_test) else: print "epoch: {} ({}s), training cost: {}, val cost: {}, val err: {}, test cost {}, test err: {}".format(epoch, end_time-start_time, cost_train, cost_val, err_val, cost_test, err_test) sys.stdout.flush() model_params.append(lasagne.layers.get_all_param_values(network)) #check_path('dnn') #save_data('dnn/epoch_{}.autosave'.format(epoch), (classifier_parameters, model_params[best_epoch])) #savename = os.path.join(modelDir,'epoch_{}.npz'.format(epoch)) #files.save_model(savename,structureDic,lasagne.layers.get_all_param_values(nnet)) if epoch - best_epoch >= no_best: ## Early stoping break epoch += 1 except: if best_epoch == 0: return (classifier_parameters, model_params[-1]) else: return (classifier_parameters, model_params[best_epoch]) return (classifier_parameters, model_params[best_epoch])
predict_op = tf.argmax(hypo, 1) # In[ ]: with tf.Session() as sess: tf.initialize_all_variables().run() total_batch = int(num_train / batch_size) batch_num = 0 cur = 0 curIndex = 0 for epoch in range(training_epochs): cur = total_batch * (curIndex) batch_num = 0 avg_cost = 0. bat = batch.Batch(x_train, label_train) for i in range(cur, total_batch * (curIndex + 1)): fetch_batch, label_batch = bat.next_batch(batch_size) fetch_batch = fetch_batch.reshape(batch_size, 28, 28, 1) label_onehot = make_onehot(label_batch, 10) _, c = sess.run( [train_op, cost], feed_dict={ X: fetch_batch, Y: label_onehot, p_keep_conv: 0.8, p_keep_hidden: 0.5 }) avg_cost = (avg_cost * i + c) / (i + 1) pg.progress(total_batch * training_epochs, i, avg_cost) curIndex += 1
import batch import numpy as np vec = np.array(range(10)) bv = batch.Batch(vec) for i in range(10): print (bv.next_batch(3))