def textChanger(pdfText, mostAuthor="", mostPaper="",extractOptions=["nltk",5,5,5],devMode=False): """"Takes the semi-cleaned text of a pdf and extracts the desired portions. Output in markdown suitable for display on the website.""" pdfText = pre_clean.pre_clean(pdfText) if mostAuthor: mostAuthor = evaluator(authorCounter(pdfText)) if mostPaper: mostPaper = evaluator(paperCounter(pdfText)) ex = extract(pdfText,extractOptions) return ex
def test(self,sess): start_time = datetime.datetime.now() img_r,img_i,lab = image_test(self.raw_path,self.label_path) img_r_raw = image_add_border(img_r[0],[img_r[0].shape[0]+6,img_r[0].shape[1]+6]) img_i_raw = image_add_border(img_i[0],[img_i[0].shape[0]+6,img_i[0].shape[1]+6]) feed_dict={self.input_real:[img_r_raw],self.input_imag:[img_i_raw],self.input_raw_r:img_r,self.input_raw_i:img_i,self.label:lab,self.decay_steps:1,self.keep_prob:1.,self.keep_prob2:1.} pred_label = sess.run(self.out, feed_dict=feed_dict) print("image_size: "+str(pred_label.shape[1:])) print("classification time: "+(str(datetime.datetime.now()-start_time)[5:])+' (s)') print('classification image saving...') pred_file = image_save(pred_label[0,...],pred_label.shape[1],pred_label.shape[2],"CRPM_Net",self.label_path,True) evaluator(pred_file,self.label_path)
def textChanger(pdfText, mostAuthor="", mostPaper="", extractOptions=["nltk", 5, 5, 5], devMode=False): """"Takes the semi-cleaned text of a pdf and extracts the desired portions. Output in markdown suitable for display on the website.""" pdfText = pre_clean.pre_clean(pdfText) if mostAuthor: mostAuthor = evaluator(authorCounter(pdfText)) if mostPaper: mostPaper = evaluator(paperCounter(pdfText)) ex = extract(pdfText, extractOptions) return ex
def __init__(self, work_path, tool_path, train_fname, test_fname, gt_fname, rl_path, rl_names, sel_uni_fname): # path to folder stores trained models and generated results self.work_path = work_path # path to the executable file of RankSVM or RankNet self.tool_path = tool_path # filename of training data (different format for different tool) self.train_fname = train_fname # filename of testing data self.test_fname = test_fname # filename of ground truth self.gt_fname = gt_fname # historical ranking lists from traditional methods (for Evaluator) self.rl_path = rl_path self.rl_names = rl_names os.chdir(self.work_path) # read selected universities self.sel_unis = [] data = np.genfromtxt(sel_uni_fname, dtype=str, delimiter=',') if len(data.shape) == 1: self.sel_unis = copy.copy(data) else: for uni_kv in data: self.sel_unis.append(uni_kv[0]) print '#universities selected:', len(self.sel_unis) # initialize evaluator self.evaluator = evaluator.evaluator(self.rl_path, self.rl_names, self.gt_fname)
def process(self, connection, source, target, args): if len(args) < 2: return None f_name = args[0] expr = u' '.join(args[1:]) if 'lambda' not in expr: return None try: lmbda = evaluator(expr) except SyntaxError: return {'action': self.Action.PRIVMSG, 'target': target, 'message': (u'Syntax Error',) } if len(inspect.getargspec(lmbda)[0]) != 1: return None class DefinedPlugin(bot.CommandPlugin): name = f_name def process(self, c, s, t, a): try: result = lmbda(u' '.join(a)) except Exception, e: ename = e.__class__.__name__ msg = '%s: %s' % (ename, e) return {'action': self.Action.PRIVMSG, 'target': target, 'message': (msg,) } else: return {'action': self.Action.PRIVMSG,
def process(self, connection, source, target, args): expr = u' '.join(args) try: result = unicode(evaluator(expr)) msg = result[:100] if result != msg: msg += '...' return {'action': self.Action.PRIVMSG, 'target': target, 'message': (msg,) } except SyntaxError: return {'action': self.Action.PRIVMSG, 'target': target, 'message': (u'Syntax Error',) } except Exception, e: ename = e.__class__.__name__ msg = '%s: %s' % (ename, e) return {'action': self.Action.PRIVMSG, 'target': target, 'message': (msg,) }
def do(self): i = 0 while (i < self.lencora): if (self.marks[i] == 0): self._bfs(i) self.setnum += 1 i += 1 s = 0 res = [] fp = open(self.resultfile, 'w+') for i in xrange(len(self.clusters)): s += len(self.clusters[i]) tmp = list(self.clusters[i]) res.append(tmp) for j in xrange(len(tmp)): if (j == len(tmp) - 1): fp.write(str(tmp[j]) + '\n') else: fp.write(str(tmp[j]) + ' ') fp.close() fp = open(self.answerfile) lines = [line.strip().split() for line in fp] ans = [] for line in lines: tmp = [] for j in xrange(len(line)): tmp.append(int(line[j])) ans.append(tmp) ev = evaluator(self.lencora) ev.load_answer_clusters(ans) ev.evaluate_clusters(res)
def initForDriving(self, *args, **kwargs): self.numsteps = 0 self.last_action = None self.repeated_action_for = self.action_repeat self.use_evaluator = kwargs["use_evaluator"] if "use_evaluator" in kwargs else True self.show_plots = kwargs["show_plots"] if "show_plots" in kwargs else True if self.isSupervised and self.use_evaluator: #nur bei supervised-agents wird diese Variable explizit auf true gesetzt. Dann nutzen sie KLEINEN evalutor, sonst den von AbstractRLAgent self.evaluator = evaluator(self.containers, self, self.show_plots, self.conf.save_xml, ["progress", "laptime"], [100, 60] )
def run_code(usr_grp, mov_grp, alg, k): t0 = time.time() # Checks to see if the data file that algorithms will use exists. filename = "data.csv" data_file = Path(filename) if not data_file.is_file(): rearrange_data(filename) t1 = time.time() #print("Processing raw data took: " + str(t1 - t0)) # Reads the needed data array from file. df = pd.read_csv(filename) df_red = pp.reduce_cluster(df, usr_grp) df_red = df_red.iloc[:, 1:] df_red.columns = df_red.columns.astype(int) df_red = pp.reduce_cluster(df_red.transpose(), mov_grp) df_red = df_red.transpose() t2 = time.time() #print("Loading and preprocess data took: " + str(t2 - t1)) # Splitting data to training and test sets. train, test = ev.split_data(df_red, 0.6) t3 = time.time() #print("Splitting data took: " + str(t3 - t2)) # Removing some values from test set. newtest, ans_col, ans_rate = ev.corruptor(test, 1) t4 = time.time() #print("Processing test set took: " + str(t4 - t3)) # Evaluate performance of both svd and collaborative filtering methods. if alg == "both": col_err = ev.evaluator(train, newtest, ans_col, ans_rate, "collab", round(usr_grp * 0.625)) t5 = time.time() print("Evaluating with collab took: " + str(t5 - t4)) col_time = t5 - t4 svd_err = ev.evaluator(train, newtest, ans_col, ans_rate, "svd", k) t5 = time.time() svd_time = t5 - t4 print("Evaluating with svd took: " + str(t5 - t4)) return col_err, col_time, svd_err, svd_time else: ev.evaluator(train, newtest, ans_col, ans_rate, alg, k) t5 = time.time()
def __init__(self, playerNumber = 1, rows = 6, cols = 7, maxDepth = 4): #, initialState): #self.initialState = initialState self.evaluator = ev.evaluator() if playerNumber == 1: self.root = Node(GameState(rows=rows,cols=cols), True, self.evaluator) elif playerNumber == 2: self.root = Node(GameState(rows=rows,cols=cols), False, self.evaluator) else: raise Exception('playerNumber must be 1 or 2') self.maxDepth = maxDepth
def __init__(self, state, isMaximizer, evaluator=None): self.state = state if evaluator is not None: self.evaluator = evaluator else: self.evaluator = ev.evaluator() self.updateEvalScore() # self.depth = depth self.isMaximizer = isMaximizer self.isTerminal = (self.evalScore == numpy.inf) or \ (self.evalScore == -numpy.inf) or \ (numpy.count_nonzero(self.state.data) == numpy.size(self.state.data)) self.children = [] self.orderToSearch = []
def create_plugin(self, f_name, expr, target=None): try: lmbda = evaluator(expr) except SyntaxError: if target is None: return False return self.privmsg(target, 'Syntax Error') if target is not None: if len(inspect.getargspec(lmbda)[0]) != 1: return self.privmsg(target, 'Lambda must have one argument') class DefinedPlugin(bot.CommandPlugin): name = f_name def process(self, c, s, t, a): try: result = lmbda(u' '.join(a)) except Exception as e: ename = e.__class__.__name__ msg = '%s: %s' % (ename, e) return self.privmsg(t, msg) else: return self.privmsg(t, result) return DefinedPlugin
def initForDriving(self, *args, **kwargs): assert not self.isPretrain, "You need to load the agent as Not-pretrain for a run!" assert self.containers is not None, "if you init the net for a run, the containers must not be None!" if not self.start_fresh: assert os.path.exists(self.folder(self.conf.pretrain_checkpoint_dir) or self.folder(self.conf.checkpoint_dir)), "I need any kind of pre-trained model" if not hasattr(self, "memory"): #einige agents haben bereits eine andere memory-implementation, die sollste nicht überschreiben self.memory = Memory(self.conf.memorysize, self.conf, self, load=(not self.nomemoryload)) super().initForDriving(*args, **kwargs) self.keep_memory = kwargs["keep_memory"] if "keep_memory" in kwargs else self.conf.keep_memory self.freezeInfReasons = [] self.freezeLearnReasons = [] self.numInferencesAfterLearn = 0 self.numLearnAfterInference = 0 self.stepsAfterStart = -1 #für headstart self.epsilon = self.startepsilon self.episode_statevals = [] #für evaluator self.episodes = 0 #für evaluator, wird bei jedem neustart auf null gesetzt aber das ist ok dafür if self.use_evaluator: self.evaluator = evaluator(self.containers, self, self.show_plots, self.conf.save_xml, \ ["average rewards", "average Q-vals", "progress", "laptime" ], \ [(-0.1,1.3), (-1,100), 100, self.time_ends_episode ] )
def train(self, epochs, batch_size=10): #create adversarial ground truths out_shape = (batch_size,) + (self.target_shape[0], self.target_shape[1], 1) valid_D = np.ones(out_shape) fake_D = np.zeros(out_shape) #define an evaluator object to monitor the progress of the training dynamic_evaluator = evaluator(img_res=self.img_shape, SRscale = self.SRscale) for epoch in range(epochs): for batch, (_, img_y, img_Y) in enumerate(self.data_loader.load_batch(batch_size)): #translate domain y to domain Y fake_img_Y = self.SR.predict(img_y) #train Discriminator D_loss_real = self.D.train_on_batch(img_Y, valid_D) D_loss_fake = self.D.train_on_batch(fake_img_Y, fake_D) D_loss = 0.5 * np.add(D_loss_real, D_loss_fake) #train Generator G_loss = self.Generator.train_on_batch([img_y], [valid_D, img_Y]) print("[Epoch %d/%d] [Batch %d/%d]--[D: %.3f] -- [G_adv: %.3f] [G_rec: %.3f]" % (epoch, epochs, batch, self.data_loader.n_batches, D_loss, G_loss[1], G_loss[2])) if batch % 25 == 0 and batch!=0: """save the model""" model_name="{}_{}.h5".format(epoch, batch) self.SR.save("pretrained models/"+model_name) print("Epoch: {} --- Batch: {} ---- saved".format(epoch, batch)) dynamic_evaluator.model = self.SR dynamic_evaluator.epoch = epoch dynamic_evaluator.batch = batch dynamic_evaluator.perceptual_test(5) sample_mean_ssim = dynamic_evaluator.objective_test(batch_size=250) print("Sample mean SSIM: ------------------- %05f -------------------" % (sample_mean_ssim))
def __init__(self, work_path, tool_path, train_fname, test_fname, gt_fname, rl_path, rl_names, sel_uni_fname): self.work_path = work_path self.tool_path = tool_path self.train_fname = train_fname self.test_fname = test_fname self.gt_fname = gt_fname self.rl_path = rl_path self.rl_names = rl_names os.chdir(self.work_path) # read selected universities self.sel_unis = [] data = np.genfromtxt(sel_uni_fname, dtype=str, delimiter=',') if len(data.shape) == 1: self.sel_unis = copy.copy(data) else: for uni_kv in data: self.sel_unis.append(uni_kv[0]) print '#universities selected:', len(self.sel_unis) # initialize evaluator self.evaluator = evaluator.evaluator(self.rl_path, self.rl_names, self.gt_fname)
#!/usr/bin/env python3 import sys import parser, evaluator fn = sys.argv[1] program = parser.parse(open(fn, "rb")) evaluator.evaluator(program)
import serial import time import sys from evaluator import evaluator myeval = evaluator() s = serial.Serial('/dev/ttyACM1', 115200, timeout=1) # opens a serial port (resets the device!) time.sleep(2) # give the device some time to startup (2 seconds) # write to the device’s serial port s.write(b"a[AB]\n") # set the device address to AB time.sleep(0.1) # wait for settings to be applied # print(s.readline()) s.write(b"c[1,0,5]\n") # set number of retransmissions to 5 time.sleep(0.1) # wait for settings to be applied # print(s.readline().decode('unicode_escape').strip()) s.write(b"c[0,1,30]\n" ) # set FEC threshold to 30 (apply FEC to packets with payload >= 30) time.sleep(0.1) # wait for settings to be applied message = "\0" payload = 1 s.write(("m[" + message + ",CD]\n").encode("ascii")) """ while True: print(s.readline().decode('unicode_escape').strip()) """
import datagate import learner import evaluator import sys NumFeat = 46 NumberOfGames = int(sys.argv[1]) pastdata = datagate.getdata(NumberOfGames) # this part uses learner.py , which applies linear regression and get a vector of coefficients print("Begin learning...") coe = learner.learner(pastdata, NumFeat) print(coe) # this part is to test whether learner works reasonably by # allowing manual input while (True): inp = input().split(" ") for i in range(0, len(inp)): inp[i] = int(inp[i]) if (inp[0] == -1): break if (len(inp) != NumFeat): print("Not a legal vector\n") else: print(evaluator.evaluator(coe, inp, NumFeat))
if(input_var[0] == 'exit'): break elif(input_var[0] == 'help'): if len(input_var) == 1: print("Commands: analyze, generate, evaluate, subset, exit") print("Type help _command_ for more information") else: if(input_var[1] == 'analyze'): print("Mandatory args:input, output") if(input_var[1] == 'generate'): print("Mandatory args: input, output") print("Optional args: size, randomUsers, randomOrders") if(input_var[1] == 'evaluate'): print("Mandatory args: file1, file2") if(input_var[1] == 'subset'): print("Mandatory args:input, output, size") elif(input_var[0] == 'analyze'): analyzer.analyze('../Data/'+args['input'],'../Data/'+args['output']) elif(input_var[0] == 'generate'): randomUsers = 'randomUsers' in args randomOrders = 'randomOrders' in args generator.generate('../Data/'+args['input'],'../Data/'+args['output'], int(args.get('size', -1)), randomUsers, randomOrders) elif(input_var[0] == 'evaluate'): e = evaluator.evaluator('../Data/'+args['file1'],'../Data/'+args['file2']) elif(input_var[0] == 'subset'): df = dataReader.readOrders('../Data/'+args['input']) subset_df = dataWriter.makeOrderSubset(df, int(args['size'])) dataWriter.writeOrderFileRandom(subset_df, '../Data/'+args['output']) else: print("Commands: analyze, generate, evaluate, subset, exit") print("Type help _command_ for more information")
__author__ = "rogersjeffrey" """ use the heuristics to identify if the given sentence has a possible interaction or not and print the stats """ import identify_interaction as id import evaluator as evalu import pprint import utils import cPickle as pickle id_instance = id.indentify_interaction() eval_instance = evalu.evaluator() files_list = [ "models/bigram_diff.p", "models/bigram_diff_total.p", "models/trigram_diff.p", "models/trigram_diff_total.p", ] id_instance.load_model(files_list) # paths=["/Users/rogersjeffrey/Downloads/DDICorpus/DDICorpus/Train/DrugBank","/Users/rogersjeffrey/Downloads/DDICorpus/DDICorpus/Train/Medline"] paths = ["./Test/Test for DDI Extraction task/DrugBank", "./Test/Test for DDI Extraction task/Medline"] files = utils.get_files_list(paths) total_sentence_count = 0 for file in files: result = id_instance.identify_interaction(file) for sentence_id in result: total_sentence_count += 1 prediction = result[sentence_id]
def _abNegamax(self, board, maxDepth, depth, alpha, beta): alphaOriginal = alpha zhash = zobrist_hash(board) entry = self._transTable.table.get(zhash) if entry and entry.depth >= maxDepth - depth: if entry.scoreType == self._transTable.EXACT_SCORE: self._transTable.hits = self._transTable.hits + 1 return (entry.move, entry.score, entry.finalBoard) elif entry.scoreType == self._transTable.LOWER_BOUND_SCORE: alpha = max(alpha, entry.score) else: beta = min(beta, entry.score) if alpha >= beta: return (entry.move, entry.score, entry.finalBoard) newEntry = False if not entry: entry = transTable.transTableEntry() entry.zobristHash = zhash newEntry = True entry.result = board.result() entry.depth = maxDepth - depth entry.move = None #result = board.result() if (depth == maxDepth or entry.result != "*"): entry.score = evaluator(board, entry.result) entry.finalBoard = board if (self._transTable.size == self._transTable.maxSize and newEntry): self._transTable.table.popitem() self._transTable.size = self._transTable.size - 1 self._transTable.table[entry.zobristHash] = entry self._transTable.size = self._transTable.size + 1 return ('', entry.score, board) maxScore = -(1<<64) score = maxScore bestMove = None finalBoard = None for move in board.legal_moves: board.push(move) _, score, finalBoard = self._abNegamax(board, maxDepth, depth + 1, -beta, -alpha) score = -score board.pop() if score > maxScore: maxScore = score bestMove = move alpha = max(alpha, score) if alpha >= beta: break entry.score = maxScore entry.move = bestMove entry.finalBoard = finalBoard if maxScore <= alphaOriginal: entry.scoreType = self._transTable.UPPER_BOUND_SCORE elif maxScore >= beta: entry.scoreType = self._transTable.LOWER_BOUND_SCORE else: entry.scoreType = self._transTable.EXACT_SCORE if (self._transTable.size == self._transTable.maxSize and newEntry): self._transTable.table.popitem() self._transTable.size = self._transTable.size - 1 self._transTable.table[entry.zobristHash] = entry self._transTable.size = self._transTable.size + 1 return (bestMove, maxScore, finalBoard)
random.shuffle(data) delta = [0 for ii in xrange(self.feature_num)] for j in xrange(len(data)): for k in xrange(self.feature_num): z = self.sigmoid( sum(map(lambda x, y: x * y, self.w, data[j]))) delta[k] += (z - self.train_label[j]) * data[j][k] if j % self.batch_size == 0 and j != 0: for k in xrange(self.feature_num): self.w[k] -= delta[k] * self.learning_rate delta = [0 for ii in xrange(self.feature_num)] if i % 100 == 0: print 'Round %d, loss:%.5f' % (i, self.get_loss()) def get_pred(self): pred = [] for i in xrange(len(self.test_data)): temp = 0 for j in xrange(self.feature_num): temp += self.w[j] * self.test_data[i][j] pred.append(self.sigmoid(temp)) return pred if __name__ == '__main__': ml = logistic_regressor('car', 0.2) ml.train_gd() pred = ml.get_pred() lr_eval = evaluator(ml.test_label, pred, 'logistic regression') lr_eval.plot_roc()
from interpretor import interpretor from configurer import configurer from evaluator import evaluator from admin import admin itp=interpretor('localhost',27017) config=configurer(itp) eva=evaluator(itp) ad=admin(eva,config) """This script performs a complete test of our Python+MongoDB OrBAC single Tenant implementation""" """Test Scenario 1. we start from zero having a tenant called "apple" 2. we insert all administrative views including: srole,activity,view,role_assignment,activity_assignment,licence,cross_licence 3. we initialize it with assigning "John" to subject, "admin" to role, insert delete to action, insertActivity, deleteActivity and manage to activity and the first licence " John is permitted to manage licence in apple, also "nominal" to context 4. we then use John to create licences for himself for all administrative views, then use John to create different users,actions, resources and assign them to different abstract roles,activities,views 5. use John to assign users privileges 6. use John to assign admin privileges to someone """ """1. we start from zero having a tenant called apple""" #create tenant config.CreateTenant('null','apple') """2. we insert all administrative views including: subject,action,object,role,activity,view,role_assignment,activity_assignment,licence """ #create administrative views config.AssignView('null','apple',{'_id':'context','attr':{}}) config.AssignView('null','apple',{'_id':'role','attr':{}})
def _abNegamax(self, board, maxDepth, depth, alpha, beta): alphaOriginal = alpha zhash = zobrist_hash(board) entry = self._transTable.table.get(zhash) if entry and entry.depth >= maxDepth - depth: if entry.scoreType == self._transTable.EXACT_SCORE: self._transTable.hits = self._transTable.hits + 1 return (entry.move, entry.score, entry.finalBoard) elif entry.scoreType == self._transTable.LOWER_BOUND_SCORE: alpha = max(alpha, entry.score) else: beta = min(beta, entry.score) if alpha >= beta: return (entry.move, entry.score, entry.finalBoard) newEntry = False if not entry: entry = transTable.transTableEntry() entry.zobristHash = zhash newEntry = True entry.result = board.result() entry.depth = maxDepth - depth entry.move = None #result = board.result() if (depth == maxDepth or entry.result != "*"): entry.score = evaluator(board, entry.result) entry.finalBoard = board if (self._transTable.size == self._transTable.maxSize and newEntry): self._transTable.table.popitem() self._transTable.size = self._transTable.size - 1 self._transTable.table[entry.zobristHash] = entry self._transTable.size = self._transTable.size + 1 return ('', entry.score, board) maxScore = -(1 << 64) score = maxScore bestMove = None finalBoard = None for move in board.legal_moves: board.push(move) _, score, finalBoard = self._abNegamax(board, maxDepth, depth + 1, -beta, -alpha) score = -score board.pop() if score > maxScore: maxScore = score bestMove = move alpha = max(alpha, score) if alpha >= beta: break entry.score = maxScore entry.move = bestMove entry.finalBoard = finalBoard if maxScore <= alphaOriginal: entry.scoreType = self._transTable.UPPER_BOUND_SCORE elif maxScore >= beta: entry.scoreType = self._transTable.LOWER_BOUND_SCORE else: entry.scoreType = self._transTable.EXACT_SCORE if (self._transTable.size == self._transTable.maxSize and newEntry): self._transTable.table.popitem() self._transTable.size = self._transTable.size - 1 self._transTable.table[entry.zobristHash] = entry self._transTable.size = self._transTable.size + 1 return (bestMove, maxScore, finalBoard)
filejson = open("../topo/q_fattree_k_8_MINEDGE_SPT.json") #filejson = open("../topo/q_fattree_k_10_SIMPLE_SPT.json") topojson = json.load(filejson) entries = topojson['entries'] entries = sorted(entries.items(), key=lambda x: int(x[0])) #for k in range(len(entries)): # print entries[k][0] , ":" , entries[k][1] src_file = entries[0][1]['src_file'] load_json_topology("../topo/" + src_file) indexer = 0 shortest_paths_all() for k in range(len(entries)): #print entries[k][0] if int(entries[k][0]) == 501: break entry = entries[k][1] source = entry['src_host'] destination = entry['dest_hosts'] algorithm1 = entry['algorithm'] algorithm = algorithm1 flow_rate = entry['flow_rate'] #print "** Generating port sets for trees **" tree_no = int(entries[k][0]) count = tree_ports_all(tree_no, source, destination, algorithm, flow_rate) #for entry in final_trees: # print entry evaluator_obj = evaluator(switches, hosts, dpids) evaluator_obj.save_final_trees(final_trees, final_trees_temp) evaluator_obj.evaluate(final_trees, "avalanche") show_final_trees()
def main(): # chainer.config.autotune = True # chainer.config.cudnn_fast_batch_normalization = True print("dataset", CONFIG.dataset) print("output_dir:", output_dir) if CONFIG.dataset == "tennis_serve": dataset = load_penn_action(dataset_dir=CONFIG.dataset_path, stride=CONFIG.penn_action.stride, dict_ok=False) dataset_train = dataset[:115] dataset_test = dataset[115:] elif CONFIG.dataset == "pouring": dataset_train, dataset_test = load_pouring( dataset_dir=CONFIG.dataset_path, stride=CONFIG.pouring.stride, dict_ok=False) elif CONFIG.dataset == "multiview_pouring": dataset_train, dataset_test = load_multiview_pouring( dataset_dir=CONFIG.dataset_path, stride=CONFIG.multiview_pouring.stride, dict_ok=False) else: print("dataset error.") exit() dataset_train = load_dataset(dataset_train, augment=None, img_size=CONFIG.img_size, k=CONFIG.k) dataset_test = load_dataset(dataset_test, augment=None, img_size=CONFIG.img_size, k=CONFIG.k) train_iter = MultiprocessIterator(dataset_train, batch_size=CONFIG.batchsize, n_processes=6) test_iter = MultiprocessIterator(dataset_test, batch_size=1, n_processes=6, repeat=False, shuffle=None) model = tcc(use_bn=True, k=CONFIG.k) device = chainer.get_device(OPTION.device) device.use() model.to_device(device) optimizer = make_optimizer(model) if CONFIG.weight_decay_rate != 0: for param in model.params(): param.update_rule.add_hook(WeightDecay(CONFIG.weight_decay_rate)) updater = tcc_updater({"main": train_iter}, optimizer, device) trainer = Trainer(updater, (CONFIG.iteration, 'iteration'), out=output_dir) display_interval = (100, 'iteration') plot_interval = (100, 'iteration') trainer.extend(extensions.ProgressBar(update_interval=5)) trainer.extend( extensions.LogReport(trigger=display_interval, filename='log.txt')) trainer.extend(extensions.PrintReport( ["iteration", "main/loss", "test/loss", "test/tau", "elapsed_time"]), trigger=display_interval) trainer.extend(extensions.PlotReport(["main/loss", "test/loss"], "iteration", file_name="loss.png"), trigger=plot_interval) trainer.extend(evaluator(test_iter, model, device, epoch=plot_interval[0], out=output_dir), trigger=plot_interval) trainer.extend(extensions.PlotReport(["test/tau"], "iteration", file_name="tau.png"), trigger=plot_interval) trainer.extend(extensions.snapshot_object(model, "{.updater.iteration}" + ".npz"), trigger=plot_interval) trainer.run()
def train(self, epochs, batch_size=10, sample_interval=50): #every sample_interval batches, the model is saved and sample images are generated and saved start_time = datetime.datetime.now() def chop_microseconds(delta): #utility to help avoid printing the microseconds return delta - datetime.timedelta(microseconds=delta.microseconds) """ Adversarial loss ground truths for patchGAN discriminators""" # Calculate output shape of the patchGAN discriminators based on the target shape len_x = self.img_shape[0] len_y = self.img_shape[1] self.D1_out_shape = (len_x, len_y, 1) #define the adversarial ground truths for D1 valid_D1 = np.ones((batch_size,) + self.D1_out_shape) fake_D1 = np.zeros((batch_size,) + self.D1_out_shape) print("valid D1: ", valid_D1.shape) #similarly for D2 len_x = self.target_res[0] len_y = self.target_res[1] self.D2_out_shape = (len_x, len_y, 1) valid_D2 = np.ones((batch_size,) + self.D2_out_shape) fake_D2 = np.zeros((batch_size,) + self.D2_out_shape) #define an evaluator object to monitor the progress of the training dynamic_evaluator = evaluator(img_res=self.img_shape, SRscale = self.SRscale) for epoch in range(epochs): for batch, (img_x, img_y, img_Y) in enumerate(self.data_loader.load_batch(batch_size)): # Update the discriminators # Make the appropriate generator translations for discriminator training fake_y = self.G1.predict(img_x) #translate x to denoised x fake_Y = self.SR.predict(fake_y) #translate denoised x to super-resolved x # Train the discriminators (original images = real / translated = fake) #we will need different adversarial ground truths for the discriminators D1_loss_real = self.D1.train_on_batch(img_y, valid_D1) D1_loss_fake = self.D1.train_on_batch(fake_y, fake_D1) D1_loss = 0.5 * np.add(D1_loss_real, D1_loss_fake) D2_loss_real = self.D2.train_on_batch(img_Y, valid_D2) D2_loss_fake = self.D2.train_on_batch(fake_Y, fake_D2) D2_loss = 0.5 * np.add(D2_loss_real, D2_loss_fake) # ------------------ # Train Generators # ------------------ blur_img_x = self.blur.predict(img_x) #passes img_x through the blurring kernel to provide GT. # Train the combined models (all generators basically) cyclic_loss_1 = self.cyclic1.train_on_batch([img_x], [valid_D1, img_x, blur_img_x, fake_y]) cyclic_loss_2 = self.cyclic2.train_on_batch([fake_y], [valid_D2, img_x, fake_Y]) """update log values""" #save the training point (measured in epochs) self.training.append(round(epoch+batch/self.data_loader.n_batches, 3)) #adversarial losses self.D1_loss.append(D1_loss[0]) self.D2_loss.append(D2_loss[0]) self.G1_adv.append(cyclic_loss_1[1]) self.SR_adv.append(cyclic_loss_2[1]) #1cycleGAN losses self.cyc1.append(cyclic_loss_1[2]) self.blur1.append(cyclic_loss_1[3]) self.tv1.append(cyclic_loss_1[4]) #2nd cycleGan losses self.cyc2.append(cyclic_loss_2[2]) self.tv2.append(cyclic_loss_2[3]) elapsed_time = datetime.datetime.now() - start_time elapsed_time = chop_microseconds(elapsed_time) print("[elapsed time: %s][Epoch %d/%d] [Batch %d/%d] -- [D1_adv: %.3f D2_adv: %.3f] -- [G1_adv: %.3f - SR_adv: %.3f - cyc1: %.4f - cyc2: %.4f]" % (elapsed_time, epoch, epochs, batch, self.data_loader.n_batches, D1_loss[0], D2_loss[0], cyclic_loss_1[1], cyclic_loss_2[1], cyclic_loss_1[2], cyclic_loss_2[2])) if batch % 20 == 0 and not(batch == 0 and epoch == 0): """save the model""" model_name="{}_{}.h5".format(epoch, batch) self.SR.save("models/"+model_name) print("Epoch: {} --- Batch: {} ---- saved".format(epoch, batch)) dynamic_evaluator.model = [self.G1, self.SR] dynamic_evaluator.epoch = epoch dynamic_evaluator.batch = batch dynamic_evaluator.perceptual_test(5) sample_mean_ssim = dynamic_evaluator.objective_test(batch_size=250) print("Sample mean SSIM: ------------------- %05f -------------------" % (sample_mean_ssim)) self.ssim_eval_time.append(round(epoch+batch/self.data_loader.n_batches, 3)) self.ssim.append(sample_mean_ssim) self.log()
flowgraph.start() try: raw_input('\nPress Enter after the USRP has finished processing ' '(red light goes out). Unfortunately, we currently have ' 'no way to have the flowgraph automatically finish after all ' 'bits have been transmitted (see https://lists.gnu.org/' 'archive/html/discuss-gnuradio/2016-07/msg00319.html):\n ') except EOFError: pass flowgraph.stop() flowgraph.wait() elif args.evaluator: print("Running evaluation flowgraph") evaluator_params = config['evaluator_params'].copy() evaluator_params.update(config['shared_params']) flowgraph = evaluator(**evaluator_params) flowgraph.start() try: raw_input('Press Enter after all files have been read by the folder source.\n') except EOFError: pass flowgraph.stop() flowgraph.wait() if 'report_output_file' in config and config['report_output_file']: print("Saving statistics to {}".format(config['report_output_file'])) generate_report(config['report_output_file'], config, flowgraph.starcoder_utils_prbs_sink_pdu_0.statistics) if 'plot_output_file' in config and config['plot_output_file']: print("Saving output plot to {}".format(config['plot_output_file'])) plot_collected_packets(flowgraph.starcoder_utils_prbs_sink_pdu_0.collected_packets, config['plot_output_file']) elif args.filegen: print("Running IQ file")
def make_evaluator(self): return evaluator(self.__names,self.__weights)
def train(self, epochs, batch_size=1, sample_interval=50): #every sample_interval batches, the model is saved and sample images are generated and saved start_time = datetime.datetime.now() # Adversarial loss ground truths valid = np.ones((batch_size, ) + self.disc_patch) print("valid shape:{}".format(valid.shape)) fake = np.zeros((batch_size, ) + self.disc_patch) dynamic_evaluator = evaluator(img_res=self.img_shape, SRscale=self.SRscale) for epoch in range(epochs): for batch_i, (imgs_A, imgs_B) in enumerate( self.data_loader.load_batch(batch_size)): # ---------------------- # Train Discriminators # ---------------------- # Translate images to opposite domain fake_B = self.G.predict(imgs_A) imgs_A_vgg = self.block2_conv1_LR.predict(imgs_A) imgs_B_vgg = self.block2_conv1_SR.predict(imgs_B) # Train the discriminators (original images = real / translated = Fake) d_loss_real = self.D2.train_on_batch(imgs_B, valid) d_loss_fake = self.D2.train_on_batch(fake_B, fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) self.log_D_loss.append(d_loss[0]) # ------------------ # Train Generators # ------------------ # Train the generators g_loss = self.combined.train_on_batch( [imgs_A, imgs_B], [valid, imgs_A_vgg, imgs_B_vgg, imgs_B_vgg, imgs_B]) elapsed_time = datetime.datetime.now() - start_time training_time_point = epoch + batch_i / self.data_loader.n_batches self.log_TrainingPoints.append( np.around(training_time_point, 3)) self.log_G_loss.append(g_loss[1]) self.log_ReconstructionLoss.append(np.mean(g_loss[2:4])) self.log_ID_loss.append(g_loss[4]) self.log_TotalVariation.append(g_loss[5]) # Plot the progress print ("[Epoch %d/%d] [Batch %d/%d] [D loss: %f, acc: %3d%%] [G loss: %05f, adv: %05f, recon: %05f, ID: %05f, TV: %05f] time: %s " \ % ( epoch, epochs, batch_i, self.data_loader.n_batches, d_loss[0], 100*d_loss[1], g_loss[0], g_loss[1], np.mean(g_loss[2:4]), g_loss[4], g_loss[5], elapsed_time)) # If at save interval => save generated image samples if batch_i % sample_interval == 0: print("Epoch: {} --- Batch: {} ---- saved".format( epoch, batch_i)) dynamic_evaluator.model = self.G dynamic_evaluator.epoch = epoch dynamic_evaluator.batch = batch_i dynamic_evaluator.perceptual_test(5) sample_mean_ssim = dynamic_evaluator.objective_test( batch_size=800) print( "Sample mean SSIM: ------------------- %05f -------------------" % (sample_mean_ssim)) self.log_sample_ssim_time_point.append( np.around(training_time_point, 3)) self.log_sample_ssim.append(sample_mean_ssim) #self.sample_images(epoch, batch_i) self.G.save("models/{}_{}.h5".format(epoch, batch_i)) self.logger()
preprocessor.transform_test_sent_fsoftmax_v2(config) print("pairing data...") preprocessor.pair_train_sent_fsoftmax(config) preprocessor.pair_valid_sent_fsoftmax(config) preprocessor.pair_test_sent_fsoftmax(config) print("training model...") import trainer_hier_fsoftmax_v2 trainer_hier_fsoftmax_v2.trainer(config) import decoder_hier_fsoftmax_v2 decoder_hier_fsoftmax_v2.decoder(config) ''' import evaluator evaluator.evaluator(config) import read_kp_kp20k read_kp_kp20k.reader(config)
dataset, batch_size=cfg.TRAIN.BATCH_SIZE, drop_last=True, shuffle=bshuffle, num_workers=int(cfg.WORKERS)) # Define models and go to training algo = trainer(output_dir, dataloader, dataset) algo.train() else: dataset = TestDataset(cfg.DATA_DIR, split_dir, base_size=cfg.TREE.BASE_SIZE) assert dataset dataloader = torch.utils.data.DataLoader( dataset, batch_size=cfg.TRAIN.BATCH_SIZE, drop_last=True, shuffle=bshuffle, num_workers=int(cfg.WORKERS)) # Define models and go to evaluating algo = evaluator(output_dir, dataloader, dataset) if dataset.acts_dict is None: algo.dump_fid_acts(cfg.DATA_DIR, split_dir) dataset.acts_dict = load_acts_data(cfg.DATA_DIR, split_dir) algo.evaluate(split_dir, dataset.imsize) end_t = time.time() print('Total time for {0}:'.format(split_dir), end_t - start_t)