def test(): # Reading test set test_data = preprocess(FLAGS.test_files.split(','), FLAGS.test_batch_size, Config.n_input, Config.n_context, Config.alphabet, hdf5_cache_path=FLAGS.test_cached_features_path) graph = create_inference_graph(batch_size=FLAGS.test_batch_size, n_steps=-1) evaluate.evaluate(test_data, graph)
def evaluate_mean(corpus, lexicon, mark): valence_pred = [] valence_true = [] arousal_pred = [] arousal_true = [] def VA_mean(text): sum_valence = 0 sum_arousal = 0 count = 0 for word in text: for l in lexicon: if word == l[0]: if l[1] > 9: l[1] = 9 if l[1] < 1: l[1] = 1 if l[2] > 9: l[2] = 9 if l[2] < 1: l[2] = 1 count = count + 1 sum_valence = sum_valence + l[1] sum_arousal = sum_arousal + l[2] return [5., 5.] if count == 0 else [sum_valence / count, sum_arousal / count] num = len(corpus) for (i, text) in enumerate(corpus): V, A = VA_mean(text) valence_pred.append(V) arousal_pred.append(A) try: ind = [item[0] for item in mark].index(i + 1) except ValueError: raise Exception('File not found. NO. %i' % (i + 1)) valence_true.append(mark[ind][1]) arousal_true.append(mark[ind][2]) # for item in mark: # if (i + 1) == item[0]: # valence_true.append(item[1]) # arousal_true.append(item[2]) # break # else: # raise Exception('File not found. NO. %i' % (i + 1)) if i % 10 == 0: logger.info("evaluate for text : %i/%i..." % (i, num)) evaluate(valence_true, valence_pred, 'valence') evaluate(arousal_true, arousal_pred, 'arousal')
def evaluate_tfidf_geo(corpus, lexicon, mark): valence_pred = [] valence_true = [] arousal_pred = [] arousal_true = [] num = len(corpus) for (i, text) in enumerate(corpus): sum_valence = 1. sum_arousal = 1. count = 0. for word in text: for l in lexicon: if word == l[0]: word_tfidf = tfidf(word, corpus[i], corpus) # logger.info("tfidf of word %s is %f" % (word, word_tfidf)) if l[1] > 9: l[1] = 9 if l[1] < 1: l[1] = 1 if l[2] > 9: l[2] = 9 if l[2] < 1: l[2] = 1 count = count + word_tfidf sum_valence = sum_valence * (l[1] ** word_tfidf) sum_arousal = sum_arousal * (l[2] ** word_tfidf) if count == 0.: valence_pred.append(5.) arousal_pred.append(5.) else: # logger.info("%f %f" % (sum_valence ** (1. / count), sum_arousal ** (1. / count))) valence_pred.append(sum_valence ** (1. / count)) arousal_pred.append(sum_arousal ** (1. / count)) for item in mark: if (i + 1) == item[0]: valence_true.append(item[1]) arousal_true.append(item[2]) break if (i + 1) % 10 == 0: logger.info("evaluate for text : %i/%i..." % ((i + 1), num)) evaluate(valence_true, valence_pred, 'valence') evaluate(arousal_true, arousal_pred, 'arousal')
def linear_regression(X_train, X_test, Y_train, Y_test, plot=False): # Create linear regression object # The training data should be column vectors X_train, X_test = np.array(X_train).reshape((len(X_train), 1)), np.array(X_test).reshape((len(X_test), 1)) regr = linear_model.LinearRegression() # Train the model using the training sets regr.fit(X_train, Y_train) predict = regr.predict(X_test) # record the experiment performance, Explained variance score: 1 is perfect prediction np.seterr(invalid='ignore') evaluate(list(predict), np.array(Y_test), 'linear regression ' + 'Explained variance score: %.2f' % regr.score(X_test, Y_test)) if plot is True: draw_linear_regression(X_train, Y_train, regr.predict(X_train)) draw_linear_regression(X_test, Y_test, predict)
def evaluate_tfidf_mean(corpus, lexicon, mark): valence_pred = [] valence_true = [] arousal_pred = [] arousal_true = [] num = len(corpus) for (i, text) in enumerate(corpus): sum_valence = 0. sum_arousal = 0. count = 0. for word in text: for l in lexicon: if word == l[0]: word_tfidf = tfidf(word, corpus[i], corpus) # logger.info("tfidf of word %s is %f" % (word, word_tfidf)) if l[1] > 9: l[1] = 9 if l[1] < 1: l[1] = 1 if l[2] > 9: l[2] = 9 if l[2] < 1: l[2] = 1 count = count + word_tfidf sum_valence = sum_valence + word_tfidf * l[1] sum_arousal = sum_arousal + word_tfidf * l[2] if count == 0: valence_pred.append(5.) arousal_pred.append(5.) else: valence_pred.append(sum_valence / count) arousal_pred.append(sum_arousal / count) for item in mark: if (i + 1) == item[0]: valence_true.append(item[1]) arousal_true.append(item[2]) break if i % 10 == 0: logger.info("evaluate for text : %i/%i..." % (i, num)) evaluate(valence_true, valence_pred, 'valence') evaluate(arousal_true, arousal_pred, 'arousal')
def update_json(inp_data, mode): updated_data = [] ctr = 0 for example in inp_data: bmpmaker.clear() if mode == "relative": true_seq_relative = RelativeEventSequence.from_eval_str(example["y_ref"]) true_events_absolute = AbsoluteEventSequence.from_relative(true_seq_relative, width, width).events pred_seq_relative = RelativeEventSequence.from_eval_str(example["y_pred"]) pred_events_absolute = AbsoluteEventSequence.from_relative(pred_seq_relative, width, width).events elif mode == "cursor": true_seq_cursor = CursorEventSequence.from_eval_str(example["y_ref"]) true_events_absolute = AbsoluteEventSequence.from_cursor(true_seq_cursor, width, width).events pred_seq_cursor = CursorEventSequence.from_eval_str(example["y_pred"]) pred_events_absolute = AbsoluteEventSequence.from_cursor(pred_seq_cursor, width, width).events else: raise Exception("Unknown or unsupported mode: {}".format(mode)) bmpmaker.process_commands(true_events_absolute) true_bitmap = np.array(bmpmaker.bitmap) bmpmaker.clear() bmpmaker.process_commands(pred_events_absolute) pred_bitmap = np.array(bmpmaker.bitmap) hamming = evaluate(true_bitmap, pred_bitmap) example["hamming_distance"] = hamming updated_data.append(example) ctr += 1 if ctr % 500 == 0: print "Progress: %d" % ctr return updated_data
def tabu_search(initSol, tabuListSz): MAX_ITER = 10000 MAX_NO_IMPROVEMENTS = int(MAX_ITER * 0.25) noImporvementsCount = 0 tabuList = deque(maxlen=tabuListSz) bestSol = initSol (solPath,bestSolVal) = evaluate(initSol) sol = initSol solVal = bestSolVal i = 0 #TO DO: better condition while i < MAX_ITER and noImporvementsCount < MAX_NO_IMPROVEMENTS: i += 1 sol, move, solVal, solPath = get_new_solution(sol,solPath,tabuList) if tabuList.__len__() == tabuListSz: tabuList.popleft() tabuList.append(move) if solVal < bestSolVal: noImporvementsCount = 0 bestSol = sol bestSolVal = solVal else: noImporvementsCount += 1 #print(move) #print(solVal) #print(bestSolVal) #print() return bestSolVal
def main(): clean() ratings_train_text = sc.textFile(config.ML_RATINGS_TRAIN) ratings_train = ( ratings_train_text .map(ml_parse.parse_line) .map(ml_parse.rating_convert)) ratings_validation_text = sc.textFile(config.ML_RATINGS_VALIDATION) ratings_validation = ( ratings_validation_text .map(ml_parse.parse_line) .map(ml_parse.rating_convert)) ratings_train.cache() ratings_validation.cache() best_result = evaluate.evaluate(ratings_train, ratings_validation, config.ML_RESULTS_FILE) with open(config.ML_BEST_PARAMS_FILE, "w") as outfile: outfile.write("%s,%s\n" % ("rank", "lambda")) outfile.write("%s,%s" % ( best_result.get("rank"), best_result.get("lambda"))) best_model = best_result.get("model") best_model.save(sc, config.ML_MODEL) sc.stop()
def _min_max(board, alpha, beta, depth=DEPTH-1): hsh = board.zobrist_hash() score = search_hash(depth, alpha, beta, hsh) best_move = None if score: return score if board.is_game_over() or depth == 0: score = evaluate(board) input_hash(hsh, depth, score, exact_flag) return score else: if board.turn == chess.WHITE: for move in check_best_move(hsh, board.generate_legal_moves()): board.push(move) # make move score = _min_max(board, alpha, beta, depth-1) board.pop() # unmake move if score > alpha: alpha = score best_move = move if alpha >= beta: break input_hash(hsh, depth, alpha, alpha_flag, best_move) return alpha else: for move in check_best_move(hsh, board.generate_legal_moves()): board.push(move) # make move score = _min_max(board, alpha, beta, depth-1) board.pop() # unmake move if score < beta: beta = score best_move = move if alpha >= beta: break input_hash(hsh, depth, beta, beta_flag, best_move) return beta
def p_expression(p): '''expression : expression list | terminal | quote | empty''' from evaluate import evaluate p[0] = evaluate(p[2] if len(p) == 3 else p[1], p.parser.toplevel)
def train_and_evaluate(model, train_dataloader, val_dataloader, optimizer, loss_fn, metrics, params, model_dir, restore_file=None): """Train the model and evaluate every epoch. Args: model: (torch.nn.Module) the neural network params: (Params) hyperparameters model_dir: (string) directory containing config, weights and log restore_file: (string) - name of file to restore from (without its extension .pth.tar) """ # reload weights from restore_file if specified if restore_file is not None: restore_path = os.path.join(args.model_dir, args.restore_file + '.pth.tar') logging.info("Restoring parameters from {}".format(restore_path)) utils.load_checkpoint(restore_path, model, optimizer) best_val_acc = 0.0 # learning rate schedulers for different models: if params.model_version == "resnet18": scheduler = StepLR(optimizer, step_size=150, gamma=0.1) # for cnn models, num_epoch is always < 100, so it's intentionally not using scheduler here elif params.model_version == "cnn": scheduler = StepLR(optimizer, step_size=100, gamma=0.2) for epoch in range(params.num_epochs): scheduler.step() # Run one epoch logging.info("Epoch {}/{}".format(epoch + 1, params.num_epochs)) # compute number of batches in one epoch (one full pass over the training set) train(model, optimizer, loss_fn, train_dataloader, metrics, params) # Evaluate for one epoch on validation set val_metrics = evaluate(model, loss_fn, val_dataloader, metrics, params) val_acc = val_metrics['accuracy'] is_best = val_acc>=best_val_acc # Save weights utils.save_checkpoint({'epoch': epoch + 1, 'state_dict': model.state_dict(), 'optim_dict' : optimizer.state_dict()}, is_best=is_best, checkpoint=model_dir) # If best_eval, best_save_path if is_best: logging.info("- Found new best accuracy") best_val_acc = val_acc # Save best val metrics in a json file in the model directory best_json_path = os.path.join(model_dir, "metrics_val_best_weights.json") utils.save_dict_to_json(val_metrics, best_json_path) # Save latest val metrics in a json file in the model directory last_json_path = os.path.join(model_dir, "metrics_val_last_weights.json") utils.save_dict_to_json(val_metrics, last_json_path)
def main(): parser = argparse.ArgumentParser() parser.add_argument('-i', '--input', dest='input', help='Training data file, e.g. data/1_train.txt') parser.add_argument('-r', '--oround', dest='oround', default=20, help='Number of output rounds, e.g. 20 [default=20]') parser.add_argument('-m', '--model', dest='model', help='Model python library, e.g. modelxxx, should located in \'rgmodels\'') parser.add_argument('-o', '--ofile', dest='ofile', help='Output data file, e.g. result/1_modelxxx.txt') parser.add_argument('-gt', '--gt', dest='groundtruth', help='Groundtruth, use to test the model if provided') args = parser.parse_args() assert args.input is not None assert args.model is not None assert args.groundtruth is not None gt_data = io.read_database(args.groundtruth) in_data = io.read_database(args.input) nr_rows = len(in_data) nr_cols = len(in_data[0]) nr_types = numpy.array(in_data).max()+1 out_data = numpy.zeros((nr_rows, args.oround), dtype=numpy.int32) model = load_model(args.model)() model.train(in_data, nr_rows, nr_cols, nr_types) model.predict(out_data, nr_rows, args.oround, nr_types, gt_data) if args.ofile is not None: io.write_database(args.ofile, out_data) score, count = evaluate(out_data, gt_data) print("model={} score={}/{}".format(args.model, score, count))
def gen_random_evo(monograms, bigrams, line_types, a, A, B, C, D): # generates random skeleton for each type list_skeleton_1 = [x for x in line_types if line_types[x].typenum == 1] list_weight_1 = [line_types[x].occurrences for x in list_skeleton_1] random_line_type_1 = line_types[random_weighted_occurrence(list_skeleton_1, list_weight_1)] my_line_1 = populate_words(random_line_type_1, monograms) if my_line_1 == None: # failure when populating with words return None list_skeleton_2 = [x for x in line_types if line_types[x].typenum == 2] list_weight_2 = [line_types[x].occurrences for x in list_skeleton_2] random_line_type_2 = line_types[random_weighted_occurrence(list_skeleton_2, list_weight_2)] my_line_2 = populate_words(random_line_type_2, monograms) if my_line_2 == None: # failure when populating with words return None list_skeleton_3 = [x for x in line_types if line_types[x].typenum == 3] list_weight_3 = [line_types[x].occurrences for x in list_skeleton_3] random_line_type_3 = line_types[random_weighted_occurrence(list_skeleton_3, list_weight_3)] my_line_3 = populate_words(random_line_type_3, monograms) if my_line_3 == None: # failur when populating with words return None my_random_haiku = Evo_object(my_line_1, my_line_2, my_line_3) my_random_haiku.update_score( evaluate.evaluate([my_line_1, my_line_2, my_line_3], monograms, bigrams, a, A, B, C, D) ) # print ([l.wordarray for l in my_random_haiku.triple]) return my_random_haiku
def run_baseline(test_data, train_data, baseline_type="random_train", aligned=False, verbose=False): total_hamming_distance = 0.0 source_dataset = train_data if baseline_type == "random_test" else test_data ctr = 0 for example in test_data: selected = pick_random(source_dataset) if aligned: true_sequence_absolute = events.AbsoluteEventSequence.from_aligned_string(example[1]) pred_sequence_absolute = events.AbsoluteEventSequence.from_aligned_string(selected[1]) else: true_sequence_absolute = events.AbsoluteEventSequence.from_string(example[1]) pred_sequence_absolute = events.AbsoluteEventSequence.from_string(selected[1]) bmpmaker.clear() bmpmaker.process_commands(true_sequence_absolute.events) true_bitmap = np.array(bmpmaker.bitmap) bmpmaker.clear() bmpmaker.process_commands(pred_sequence_absolute.events) pred_bitmap = np.array(bmpmaker.bitmap) hamming = evaluate.evaluate(true_bitmap, pred_bitmap) total_hamming_distance += hamming ctr += 1 if verbose and ctr % 500 == 0: print "Progress: %d" % ctr avg_hamming = total_hamming_distance / len(test_data) print "Average Hamming distance: %2.2f" % avg_hamming
def workflow() : metric = [] adset = set([line.strip().split()[1] for line in file(TMP_DATA_DIR_PATH+'topAdClickCnt.dict.final')]) adidlist = [] wholeResult = None total_advertisments = 0 for adid in adset : if adid in blacklist : continue adidlist.append(adid) res, finalres = evaluate(adid, 20, reCal=False, testing=True) if wholeResult == None : if finalres[-1][0] < 0.2 : wholeResult = numpy.array(finalres) total_advertisments += 1 else : if finalres[-1][0] < 0.2 : wholeResult += numpy.array(finalres) total_advertisments += 1 metric.append(res) linenum = 1 for adid, res in zip(adidlist, metric) : print '|'.join([str(linenum), adid, str(res[0]), str(res[1])]) linenum += 1 wholeResult = wholeResult / total_advertisments print total_advertisments print wholeResult print numpy.mean(metric, axis=0) plotTopDistribution(wholeResult)
def mutate(evo_object1, monograms, bigrams, a, A, B, C, D): line_index_to_mutate = random.randint(0, 2) mutated_object = deepcopy(evo_object1) mutated_object.triple[line_index_to_mutate] = mutate_line(mutated_object.triple[line_index_to_mutate], monograms) mutated_object.update_score(evaluate.evaluate(mutated_object.triple, monograms, bigrams, a, A, B, C, D)) return mutated_object
def eval(self, symbol_table): logger.debug("self=%s", self) t = self.token_list[0] s = evaluate(self.token_list, symbol_table) print("{}:{}: *** {}. Stop.".format(t.string[0].filename, t.string[0].linenumber, s), file=self.fh) sys.exit(1)
def get_new_solution(startSol,startPath,tabuList): neighbours = startSol.neighbours(startPath) (sol,move) = neighbours[0] (solPath,solVal) = evaluate(sol) if move in tabuList: solVal = 999999999 for n in neighbours: (newSol, newMove) = n (v1, v2) = newMove if (newMove and (v2, v1)) not in tabuList: (newSolPath,newSolVal) = evaluate(newSol) if newSolVal < solVal: sol = newSol move = newMove solVal= newSolVal solPath = newSolPath return sol, move, solVal, solPath
def main(files): #clears files with open ('prvalues.txt', 'w'): pass #results with open ('ranks.txt', 'w'): pass #ranks with open ('iprec.txt', 'w'): pass #interpolation #goes through the topics file and makes a dict topics = {} with open (files[1], 'r') as f: for lines in f: info = lines.split(' ') topics[info[0]] = ' '.join(info[1:]) # goes through a qrels file # makes a dict {query: list of rel docs} qrel = findQrels([files[2]]) # gets the files in the directory tim = getFiles([], files[3]) found = findTerms(tim) terms = found[0] l = found [1] N = len(l) output = '' ap = [] binOptions = input('1 - binary, 2 - not? ') if binOptions != 1: wOptions = input('enter: 1 - idf, 2 - length normalization: ') for q in topics: #q is the topic number rank = getRanks(topics[q], terms, l, binOptions, wOptions) sortRank = sorted(rank.items(), key = operator.itemgetter(1), reverse=True) ret = list(x[0] for x in sortRank) # for the export count = 0 for doc in sortRank: count+=1 output += (q+' Q0 '+doc[0]+' '+str(count)+' '+str(doc[1])+' x\n') rel = qrel[q] print (q, topics[q].replace('\n', '')) ap.append(evaluate(q, ret, rel, N)) print ('\n') maps = sum(ap)/len(ap) print (ap) print (maps) # exports precision and recall values with open ('ranks.txt', 'w') as f: f.write(output)
def run_evaluation(self, ref, crank_flag, single_rank_flag, expand_rank_flag): file_set = set(self.text_dict.keys()) ref_file_list = ref.keys() relscore = dict() eval_score = self.init_eval_score() for file_name in deepcopy(file_set): if file_name in ref_file_list: relscore[file_name] = read_relscore(self.relscore_path, file_name) else: print 'ref file<%s> does not exist in the given gold_standard data' % (file_name) file_set.remove(file_name) self.excluded_files.add(file_name) if expand_rank_flag: graph_dict = dict() for winsize in self.winsize_list: graph_dict[winsize] = dict() for file_name in file_set: graph_dict[winsize][file_name] = Graph(file_name, self.text_dict[file_name], relscore[file_name], winsize, self.lamb_list, self.cutoff_list, return_graph_flag=True, crank_flag=crank_flag, single_rank_flag=single_rank_flag, input_graph=None).graph self.update_neighbor_weight(graph_dict[winsize]) for winsize in self.winsize_list: new_gs = dict() for file_name in file_set: input_graph = None if expand_rank_flag: input_graph = graph_dict[winsize][file_name] graph = Graph(file_name, self.text_dict[file_name], relscore[file_name], winsize, self.lamb_list, self.cutoff_list, return_graph_flag=False, crank_flag=crank_flag, single_rank_flag=single_rank_flag, input_graph=input_graph, lda_dict=self.lda_dict) checked_gs_list = self.check_gold_standard(graph, ref[file_name]) new_gs[file_name] = checked_gs_list for cutoff in self.cutoff_list: for lamb in self.lamb_list: candidates = graph.score_candidates(cutoff, lamb) file_path = '../checked_inspec.ref' save_candidates(file_path, winsize, cutoff, lamb, candidates); eval_score[winsize][cutoff][lamb][file_name] = ( evaluate(self.stem_list(candidates), self.stem_list(checked_gs_list))) write_gs('./checked_inspec.ref', new_gs) return eval_score
def cross_pollinate(evo_object1, evo_object2, monograms, bigrams, a, A, B, C, D): new_triple = [] for y in range(3): if random.randint(0, 1) == 0: new_triple.append(evo_object1.get_triple()[y]) else: new_triple.append(evo_object2.get_triple()[y]) new_kid = Evo_object(new_triple[0], new_triple[1], new_triple[2]) new_kid.update_score(evaluate.evaluate(new_kid.triple, monograms, bigrams, a, A, B, C, D)) return new_kid
def evaluator(predictfile='../output/latest/result.txt',realfile='../data/test.csv'): r=open(predictfile) predictdata={} for line in r: userid,brandids=line.split() predictdata[userid]=brandids.split(',') import csv reader=csv.reader(open(realfile)) reader.next() testdata={} for userid,brandid,dtype,date in reader: if dtype=='1': brandids=testdata[userid] if testdata.has_key(userid) else [] if brandid not in brandids: brandids.append(brandid) testdata[userid]=brandids evaluate.evaluate(predictdata,testdata)
def refevaluate(environment, expression): if isTestCommand(expression): launchAllTests(environment) return if expression == NIL: return NIL if isNumber(expression): return expression elif isSymbol(expression): return environment[getSymbolValue(expression)] elif isString(expression): return expression elif isFunction(environment, expression): functionName = getSymbolValue(list_get(expression, 0)) function = environment["functions"][functionName]["name"] argumentsEvaluated = [] for expressionIndex in range(1, list_getLength(expression)): nextArgument = list_get(expression, expressionIndex) argumentsEvaluated.append(evaluate(environment, nextArgument)) metadata = environment["functions"][functionName] return function(environment, metadata, argumentsEvaluated) elif isMacro(environment, expression): macroName = getSymbolValue(list_get(expression, 0)) macro = environment["macros"][macroName]["name"] arguments = [] for expressionIndex in range(1, list_getLength(expression)): arguments.append(list_get(expression, expressionIndex)) metadata = environment["macros"][macroName] return evaluate(environment, macro(environment, metadata, arguments)) elif isSpecial(environment, expression): assert(isSpecial(environment, expression)) operatorName = getSymbolValue(list_get(expression, 0)) operator = environment["special"][operatorName]["name"] arguments = [] for expressionIndex in range(1, list_getLength(expression)): arguments.append(list_get(expression, expressionIndex)) metadata = environment["special"][operatorName] return operator(environment, metadata, arguments) else: print("failed on " + expressionToString(expression)) assert(False)
def main(): parser = argparse.ArgumentParser() parser.add_argument('-i', '--input', dest='input', help='Training data file, e.g. data/1_train.txt') parser.add_argument('-r', '--oround', dest='oround', default=20, help='Number of output rounds, e.g. 20 [default=20]') parser.add_argument('-m', '--model', dest='model', nargs='+', help='Model python library, e.g. modelxxx, should located in \'rgmodels\'') parser.add_argument('-val', '--val', dest='val_split', type=int, default=15, help='Split input data to train, val.') parser.add_argument('-o', '--ofile', dest='ofile', help='Output data file, e.g. result/1_modelxxx.txt') parser.add_argument('-gt', '--gt', dest='groundtruth', help='Groundtruth, use to test the model if provided') args = parser.parse_args() assert args.input is not None assert args.model is not None in_data = io.read_database(args.input) in_data = numpy.array(in_data) assert args.groundtruth is not None gt_data = io.read_database(args.groundtruth) train_data, val_data = in_data[:, 0:args.val_split], in_data[:, args.val_split:] nr_rows = len(train_data) nr_cols = len(in_data[0]) nr_train_cols = len(train_data[0]) nr_val_cols = len(val_data[0]) nr_types = numpy.array(train_data).max()+1 all_outs = list() for current_model in args.model: current_outs = list() for _ in range(10): out_data = numpy.zeros((nr_rows, nr_val_cols), dtype=numpy.int32) model = load_model(current_model)() model.train(train_data, nr_rows, nr_train_cols, nr_types) model.predict(out_data, nr_rows, nr_val_cols, nr_types, val_data) current_outs.append(out_data) all_outs.append(current_outs) final_map = ensemble(all_outs, gt=val_data) # print(final_map) all_outs = list() for current_model in args.model: out_data = numpy.zeros((nr_rows, args.oround), dtype=numpy.int32) model = load_model(current_model)() model.train(in_data, nr_rows, nr_cols, nr_types) model.predict(out_data, nr_rows, args.oround, nr_types, gt_data) all_outs.append(out_data) final = ensemble(all_outs, final_map=final_map) # final = cheat_ensemble(all_outs, gt_data) score, count = evaluate(final, gt_data) print("model={} score={}/{}".format(args.model, score, count))
def main(): # sentences=None, size=100, alpha=0.025, window=5, min_count=5, # sample=0, seed=1, workers=1, min_alpha=0.0001, sg=1, hs=1, negative=0, # cbow_mean=0, hashfxn=hash, iter=1 corp=sys.argv[1] goldfile=sys.argv[2] kop=int(sys.argv[3]) # GORDELEKU="/gscratch/users/maguirrezaba008/vectorFiles/" GORDELEKU="/home/magirrezaba008/semantika/bestEmbeddings/results/" filename = corp.split("/")[-1] from configuration import confs sentences=word2vec.LineSentence(corp) resu=[] for conf in confs: strconf=[str(i) for i in conf] modelfilename='vectors_'+'---'.join(strconf)+'.bin' print "Let's start working with the file "+modelfilename if os.path.isfile(GORDELEKU+modelfilename): print modelfilename+" exists, so we don't need to train it again!" model=Word2Vec.load_word2vec_format(GORDELEKU+modelfilename,None, True) print "Loaded!" else: print "Training with "+filename+" file..." model = Word2Vec(sentences, conf[0], conf[1], conf[2], conf[3], conf[4], conf[5], conf[6], conf[7], conf[8], conf[9], conf[10], conf[11], conf[12], conf[13]) print "Trained!" # model.save_word2vec_format(modelfilename, binary=True) # print "Saved "+modelfilename+" file..." (pearsoncoef, significance) = evaluate.evaluate(model, goldfile) print "File name: "+modelfilename print "Pearson coefficient and significance: "+str(pearsoncoef)+" "+str(significance) print this={} this['name']=modelfilename this['coef']=pearsoncoef this['significance']=significance if len(resu)<kop and this['significance'] < 0.05: resu.append(this) model.save_word2vec_format(GORDELEKU+modelfilename, binary=True) print "Saved "+modelfilename+" file..." elif this['significance'] < 0.05 and min(resu, key=lambda s: s['coef'])['coef']<this['coef']: resu.append(this) model.save_word2vec_format(GORDELEKU+modelfilename, binary=True) print "Saved "+modelfilename+" file..." mini=min(resu, key=lambda s: s['coef']) ignore(GORDELEKU+mini['name']) resu.remove(mini) else: ignore(GORDELEKU+this['name']) del(model) print "resu has "+str(len(resu))+" elements!"
def main(files): with open ('prvalues.txt', 'w'): pass #results with open ('ranks.txt', 'w'): pass #ranks with open ('iprec.txt', 'w'): pass #interpolation #goes through the topics file and makes a dict topics = {} with open (sys.argv[1], 'r') as f: for lines in f: info = lines.split(' ') topics[info[0]] = ' '.join(info[1:]) qrels = findQrels([files[2]]) # gets the files in the directory #tim = getFiles([], files[3]) terms, l = findTerms(files[3:]) N = len(l) terms = get(terms, l) output = '' ap = [] #query = input("Enter: ") #topics = [query] for q in topics: #q is the topic number rank = getRanks(topics[q], terms) sortRank = sorted(rank.items(), key = operator.itemgetter(1), reverse=True) ret = list(x[0] for x in sortRank) # for the export count = 0 for doc in sortRank: count+=1 output += (q+' Q0 '+doc[0]+' '+str(count)+' '+str(doc[1])+' x\n') rel = qrel[q] print (q, topics[q].replace('\n', '')) ap.append(evaluate(q, ret, rel, N)) print ('\n') maps = sum(ap)/len(ap) print (ap) print (maps) # exports precision and recall values with open ('ranks.txt', 'w') as f: f.write(output)
def main(): parser = argparse.ArgumentParser() parser.add_argument('-i', '--input', dest='input', help='Training data file, e.g. data/1_train.txt') parser.add_argument('-r', '--oround', dest='oround', default=20, help='Number of output rounds, e.g. 20 [default=20]') parser.add_argument('-m', '--model', dest='model', nargs='+', help='Model python library, e.g. modelxxx, should located in \'rgmodels\'') parser.add_argument('-o', '--ofile', dest='ofile', help='Output data file, e.g. result/1_modelxxx.txt') parser.add_argument('-gt', '--gt', dest='groundtruth', help='Groundtruth, use to test the model if provided') args = parser.parse_args() assert args.input is not None assert args.model is not None assert args.groundtruth is not None in_data = io.read_database(args.input) in_data = numpy.array(in_data) gt_data = io.read_database(args.groundtruth) gt_data = numpy.array(gt_data) all_data = numpy.concatenate([in_data, gt_data], axis=1) nr_rows = len(all_data) nr_types = numpy.array(all_data).max()+1 final_data = numpy.zeros_like(gt_data) for i in range(30, 50): for j1 in range(all_data.shape[0]): best_score, best_model = -1, None for current_model in args.model: sum_score = 0 for _ in range(1): train_data = all_data[:, 0:25] val_data = all_data[:, 25:i] out_data = numpy.zeros_like(val_data) model = load_model(current_model)() model.train(train_data, nr_rows, train_data.shape[1], nr_types) model.predict(out_data, nr_rows, val_data.shape[1], nr_types, val_data) for j2 in range(val_data.shape[1]): sum_score += 1 if val_data[j1, j2] == out_data[j1, j2] else 0 if sum_score > best_score: best_score = sum_score best_model = current_model current_final_out = numpy.zeros_like(all_data[:, i:i+1]) print(i, j1, best_model) model = load_model(best_model)() model.train(all_data[:, 0:i], nr_rows, i, nr_types) model.predict(current_final_out, nr_rows, 1, nr_types, all_data[:, i:i+1]) final_data[j1, i-30] = current_final_out[j1, 0] score, count = evaluate(final_data, gt_data) print("model={} score={}/{}".format(args.model, score, count))
def train(opts): train_file = opts.train model_file = opts.model beam_size = opts.beam_size assert (train_file != "") assert (model_file != "") assert (beam_size >= 1) # if not os.path.exits(model_file): # params = {} # else: # params = pickle.load(open(model_file)) params = {} data = [] with codecs.open(train_file, encoding='utf-8') as infile: for line in infile: words = line.strip().split() data.append(words) num_instances = len(data) if opts.dev: dev_data = [] dev_file = opts.dev with codecs.open(dev_file, encoding='utf-8') as infile: for line in infile: words = line.strip().split() dev_data.append(words) for nr_iter in xrange(opts.iteration): for idx, sentence in enumerate(data): now = nr_iter * num_instances + 1 gold_actions = get_gold_actions(sentence) raw_sentence = ''.join(sentence) beam_search(True, raw_sentence, beam_size, params, gold_actions, now) temp_model_file = "{0}.{1}".format(model_file, nr_iter+1) assert(len(params)>0) flush_parameters(params, now) pickle.dump(params, open(temp_model_file, 'w')) if opts.dev: temp_file = "temp/temp%s" % nr_iter with open(temp_file, 'w') as outfile: for sentence in dev_data: raw_sentence = ''.join(sentence) words = beam_search(False, raw_sentence, beam_size, params) line = ' '.join(words) line += '\n' # print raw_sentence # print line.strip() outfile.write(line) p, r, f = evaluate(temp_file, dev_file) print "Precision:{0}, Recall:{1}, Fscore:{2}".format(p, r, f)
def _ai_min_max(board, alpha, beta, competence, depth=DEPTH-1): hsh = board.zobrist_hash() score = search_hash(depth, alpha, beta, hsh) if score: return score if board.is_game_over() or depth == 0: score = evaluate(board) input_hash(hsh, depth, score, exact_flag) return score else: if board.turn == chess.WHITE: best_moves_and_alphas = [] for move in check_best_move(hsh, board.generate_legal_moves()): board.push(move) # make move score = _ai_min_max(board, alpha, beta, competence, depth-1) board.pop() # unmake move if score > alpha: alpha = score best_moves_and_alphas.append((move, alpha)) if alpha >= beta: break if not best_moves_and_alphas: input_hash(hsh, depth, alpha, alpha_flag, None) return alpha index = int(round((len(best_moves_and_alphas)-1) * competence)) best_move_and_alpha = best_moves_and_alphas[index] # attempt at cutting out very bad moves that the player would # obviously counter 50 is arbitrary my thinking was the player # would notice missing half a pawn their is probably a better way # to do this so that the weight isn't constant and is based off of # the players competence level. while abs(best_move_and_alpha[1] - best_moves_and_alphas[-1][1]) >= 50: index += 1 best_move_and_alpha = best_moves_and_alphas[index] input_hash(hsh, depth, best_move_and_alpha[1], alpha_flag, best_move_and_alpha[0]) return best_move_and_alpha[1] else: best_move = None for move in check_best_move(hsh, board.generate_legal_moves()): board.push(move) # make move score = _ai_min_max(board, alpha, beta, competence, depth-1) board.pop() # unmake move if score < beta: beta = score if alpha >= beta: break input_hash(hsh, depth, beta, beta_flag, best_move) return beta
def linear_regression_multivariant(X_train, X_test, Y_train, Y_test, cost_fun='ordinary_least_squares'): if cost_fun == 'ordinary_least_squares': regr = linear_model.LinearRegression() elif cost_fun == 'Ridge_Regression': regr = linear_model.Ridge(alpha=1) elif cost_fun == 'Bayesian_Regression': regr = linear_model.BayesianRidge() elif cost_fun == 'SVR': regr = SVR(C=1.0, epsilon=0.2, kernel='linear') elif cost_fun == 'KNN_Reg': regr = neighbors.KNeighborsRegressor(5, weights='distance') else: raise Exception('The type of cost function is not specified.') # Train the model using the training sets regr.fit(X_train, Y_train) predict = regr.predict(X_test) # record the experiment performance, Explained variance score: 1 is perfect prediction np.seterr(invalid='ignore') print(list(predict)[:100]) print(Y_test[:100]) evaluate(list(predict), np.array(Y_test), 'linear regression ' + 'Explained variance score: %.2f' % regr.score(X_test, Y_test))
def main(args): torch.manual_seed(0) # Get device device = torch.device('cuda'if torch.cuda.is_available()else 'cpu') # Get dataset dataset = Dataset("train.txt") loader = DataLoader(dataset, batch_size=hp.batch_size**2, shuffle=True, collate_fn=dataset.collate_fn, drop_last=True, num_workers=0) # Define model model = nn.DataParallel(FastSpeech2()).to(device) print("Model Has Been Defined") num_param = utils.get_param_num(model) print('Number of FastSpeech2 Parameters:', num_param) # Optimizer and loss optimizer = torch.optim.Adam(model.parameters(), betas=hp.betas, eps=hp.eps, weight_decay = hp.weight_decay) scheduled_optim = ScheduledOptim(optimizer, hp.decoder_hidden, hp.n_warm_up_step, args.restore_step) Loss = FastSpeech2Loss().to(device) print("Optimizer and Loss Function Defined.") # Load checkpoint if exists checkpoint_path = os.path.join(hp.checkpoint_path) try: checkpoint = torch.load(os.path.join( checkpoint_path, 'checkpoint_{}.pth.tar'.format(args.restore_step))) model.load_state_dict(checkpoint['model']) optimizer.load_state_dict(checkpoint['optimizer']) print("\n---Model Restored at Step {}---\n".format(args.restore_step)) except: print("\n---Start New Training---\n") if not os.path.exists(checkpoint_path): os.makedirs(checkpoint_path) # Load vocoder if hp.vocoder == 'melgan': melgan = utils.get_melgan() melgan.to(device) elif hp.vocoder == 'waveglow': waveglow = utils.get_waveglow() waveglow.to(device) # Init logger log_path = hp.log_path if not os.path.exists(log_path): os.makedirs(log_path) logger = SummaryWriter(log_path) # Init synthesis directory synth_path = hp.synth_path if not os.path.exists(synth_path): os.makedirs(synth_path) # Define Some Information Time = np.array([]) Start = time.perf_counter() # Training model = model.train() for epoch in range(hp.epochs): # Get Training Loader total_step = hp.epochs * len(loader) * hp.batch_size for i, batchs in enumerate(loader): for j, data_of_batch in enumerate(batchs): start_time = time.perf_counter() current_step = i*hp.batch_size + j + args.restore_step + epoch*len(loader)*hp.batch_size + 1 # Init scheduled_optim.zero_grad() # Get Data text = torch.from_numpy(data_of_batch["text"]).long().to(device) mel_target = torch.from_numpy(data_of_batch["mel_target"]).float().to(device) D = torch.from_numpy(data_of_batch["D"]).int().to(device) f0 = torch.from_numpy(data_of_batch["f0"]).float().to(device) energy = torch.from_numpy(data_of_batch["energy"]).float().to(device) mel_pos = torch.from_numpy(data_of_batch["mel_pos"]).long().to(device) src_pos = torch.from_numpy(data_of_batch["src_pos"]).long().to(device) src_len = torch.from_numpy(data_of_batch["src_len"]).long().to(device) mel_len = torch.from_numpy(data_of_batch["mel_len"]).long().to(device) max_len = max(data_of_batch["mel_len"]).astype(np.int16) # Forward mel_output, mel_postnet_output, duration_output, f0_output, energy_output = model( text, src_pos, mel_pos, max_len, D, f0, energy) # Cal Loss mel_loss, mel_postnet_loss, d_loss, f_loss, e_loss = Loss( duration_output, D, f0_output, f0, energy_output, energy, mel_output, mel_postnet_output, mel_target, src_len, mel_len) total_loss = mel_loss + mel_postnet_loss + d_loss + f_loss + e_loss # Logger t_l = total_loss.item() m_l = mel_loss.item() m_p_l = mel_postnet_loss.item() d_l = d_loss.item() f_l = f_loss.item() e_l = e_loss.item() with open(os.path.join(log_path, "total_loss.txt"), "a") as f_total_loss: f_total_loss.write(str(t_l)+"\n") with open(os.path.join(log_path, "mel_loss.txt"), "a") as f_mel_loss: f_mel_loss.write(str(m_l)+"\n") with open(os.path.join(log_path, "mel_postnet_loss.txt"), "a") as f_mel_postnet_loss: f_mel_postnet_loss.write(str(m_p_l)+"\n") with open(os.path.join(log_path, "duration_loss.txt"), "a") as f_d_loss: f_d_loss.write(str(d_l)+"\n") with open(os.path.join(log_path, "f0_loss.txt"), "a") as f_f_loss: f_f_loss.write(str(f_l)+"\n") with open(os.path.join(log_path, "energy_loss.txt"), "a") as f_e_loss: f_e_loss.write(str(e_l)+"\n") # Backward total_loss.backward() # Clipping gradients to avoid gradient explosion nn.utils.clip_grad_norm_(model.parameters(), hp.grad_clip_thresh) # Update weights scheduled_optim.step_and_update_lr() # Print if current_step % hp.log_step == 0: Now = time.perf_counter() str1 = "Epoch [{}/{}], Step [{}/{}]:".format( epoch+1, hp.epochs, current_step, total_step) str2 = "Total Loss: {:.4f}, Mel Loss: {:.4f}, Mel PostNet Loss: {:.4f}, Duration Loss: {:.4f}, F0 Loss: {:.4f}, Energy Loss: {:.4f};".format( t_l, m_l, m_p_l, d_l, f_l, e_l) str3 = "Time Used: {:.3f}s, Estimated Time Remaining: {:.3f}s.".format( (Now-Start), (total_step-current_step)*np.mean(Time)) print("\n" + str1) print(str2) print(str3) with open(os.path.join(log_path, "log.txt"), "a") as f_log: f_log.write(str1 + "\n") f_log.write(str2 + "\n") f_log.write(str3 + "\n") f_log.write("\n") logger.add_scalars('Loss/total_loss', {'training': t_l}, current_step) logger.add_scalars('Loss/mel_loss', {'training': m_l}, current_step) logger.add_scalars('Loss/mel_postnet_loss', {'training': m_p_l}, current_step) logger.add_scalars('Loss/duration_loss', {'training': d_l}, current_step) logger.add_scalars('Loss/F0_loss', {'training': f_l}, current_step) logger.add_scalars('Loss/energy_loss', {'training': e_l}, current_step) if current_step % hp.save_step == 0: torch.save({'model': model.state_dict(), 'optimizer': optimizer.state_dict( )}, os.path.join(checkpoint_path, 'checkpoint_{}.pth.tar'.format(current_step))) print("save model at step {} ...".format(current_step)) if current_step % hp.synth_step == 0: length = mel_len[0].item() mel_target_torch = mel_target[0, :length].detach().unsqueeze(0).transpose(1, 2) mel_target = mel_target[0, :length].detach().cpu().transpose(0, 1) mel_torch = mel_output[0, :length].detach().unsqueeze(0).transpose(1, 2) mel = mel_output[0, :length].detach().cpu().transpose(0, 1) mel_postnet_torch = mel_postnet_output[0, :length].detach().unsqueeze(0).transpose(1, 2) mel_postnet = mel_postnet_output[0, :length].detach().cpu().transpose(0, 1) Audio.tools.inv_mel_spec(mel, os.path.join(synth_path, "step_{}_griffin_lim.wav".format(current_step))) Audio.tools.inv_mel_spec(mel_postnet, os.path.join(synth_path, "step_{}_postnet_griffin_lim.wav".format(current_step))) if hp.vocoder == 'melgan': utils.melgan_infer(mel_torch, melgan, os.path.join(hp.test_path, 'step_{}_{}.wav'.format(current_step, hp.vocoder))) utils.melgan_infer(mel_postnet_torch, melgan, os.path.join(hp.test_path, 'step_{}_postnet_{}.wav'.format(current_step, hp.vocoder))) utils.melgan_infer(mel_target_torch, melgan, os.path.join(hp.test_path, 'step_{}_ground-truch_{}.wav'.format(current_step, hp.vocoder))) elif hp.vocoder == 'waveglow': utils.waveglow_infer(mel_torch, waveglow, os.path.join(hp.test_path, 'step_{}_{}.wav'.format(current_step, hp.vocoder))) utils.waveglow_infer(mel_postnet_torch, waveglow, os.path.join(hp.test_path, 'step_{}_postnet_{}.wav'.format(current_step, hp.vocoder))) utils.waveglow_infer(mel_target_torch, waveglow, os.path.join(hp.test_path, 'step_{}_ground-truch_{}.wav'.format(current_step, hp.vocoder))) f0 = f0[0, :length].detach().cpu().numpy() energy = energy[0, :length].detach().cpu().numpy() f0_output = f0_output[0, :length].detach().cpu().numpy() energy_output = energy_output[0, :length].detach().cpu().numpy() utils.plot_data([(mel_postnet.numpy(), f0_output, energy_output), (mel_target.numpy(), f0, energy)], ['Synthetized Spectrogram', 'Ground-Truth Spectrogram'], filename=os.path.join(synth_path, 'step_{}.png'.format(current_step))) if current_step % hp.eval_step == 0: model.eval() with torch.no_grad(): d_l, f_l, e_l, m_l, m_p_l = evaluate(model, current_step) t_l = d_l + f_l + e_l + m_l + m_p_l logger.add_scalars('Loss/total_loss', {'validation': t_l}, current_step) logger.add_scalars('Loss/mel_loss', {'validation': m_l}, current_step) logger.add_scalars('Loss/mel_postnet_loss', {'validation': m_p_l}, current_step) logger.add_scalars('Loss/duration_loss', {'validation': d_l}, current_step) logger.add_scalars('Loss/F0_loss', {'validation': f_l}, current_step) logger.add_scalars('Loss/energy_loss', {'validation': e_l}, current_step) model.train() end_time = time.perf_counter() Time = np.append(Time, end_time - start_time) if len(Time) == hp.clear_Time: temp_value = np.mean(Time) Time = np.delete( Time, [i for i in range(len(Time))], axis=None) Time = np.append(Time, temp_value)
def set_test(model, test_iter): if not test_iter.is_test: test_iter.is_test = True model.eval() with torch.no_grad(): pred_score_dict = {} pred_answer_dict = {} true_answer_list = [] pred_answer_list = [] for input_ids, input_mask, segment_ids, start_list, end_list, uid_list, answer_list, text_list, querylen_list, maping_list, _, seq_length in tqdm( test_iter): input_ids = list2ts2device(input_ids) input_mask = list2ts2device(input_mask) segment_ids = list2ts2device(segment_ids) y_preds = model(input_ids=input_ids, token_type_ids=segment_ids, attention_mask=input_mask) start_preds, end_preds = (p.detach().cpu() for p in y_preds) start_probs = softmax(start_preds.numpy()) end_probs = softmax(end_preds.numpy()) # cls_probs = softmax(cls_pred.numpy()) max_a_len = 64 answer_pred = [] epsilon = 1e-3 w1, w2 = 0.9, 0.1 for i in range(len(start_probs)): start_ = start_probs[i][querylen_list[i]:-1] end_ = end_probs[i][querylen_list[i]:-1] start_end, score = None, -100 for start, p_start in enumerate(start_): for end, p_end in enumerate(end_): if end >= start and end < start + max_a_len: """一定有答案""" if p_start * p_end > score: start_end = (start, end) score = p_start * p_end # score_ = np.exp((0.5 * np.log(p_start + epsilon) + 0.5 * np.log(p_end + epsilon)) / (0.5 + 0.5)) start, end = start_end """""" # out_class_=cls_probs[i][1] start_cls = start_probs[i][0] end_cls = end_probs[i][0] pos_cls = -(start_cls + end_cls) if config.addneg: score += pos_cls # score = np.exp((w1 * np.log(out_class_ + epsilon) + w2 * np.log(score_ + epsilon)) / (w1 + w2)) try: start_pos = maping_list[i][start][0] end_pos = maping_list[i][end][-1] answer = text_list[i][start_pos:end_pos + 1] except: answer = '' if uid_list[i] in pred_answer_dict: if pred_score_dict[uid_list[i]] < score: pred_score_dict[uid_list[i]] = score pred_answer_dict[uid_list[i]] = answer else: pred_answer_dict[uid_list[i]] = answer pred_score_dict[uid_list[i]] = score answer_pred.append(answer) true_answer_list.extend(answer_list) pred_answer_list.extend(answer_pred) assert len(true_answer_list) == len(pred_answer_list) print(true_answer_list) print(pred_answer_list) c = 0 for i in range(len(true_answer_list)): if true_answer_list[i] == pred_answer_list[i]: c += 1 print('em分数为', c / len(true_answer_list)) # pred_dict=dict(zip(qid_list,pred_answer_list)) true_dict = json.load( open(config.data + 'dureader_robust-data/dev.json')) F1, EM, TOTAL, SKIP = evaluate(true_dict, pred_answer_dict) print('F1: {}, EM {}, TOTAL {}'.format(F1, EM, TOTAL)) return F1, EM
def main(args): torch.manual_seed(0) # Get device device = torch.device('cuda'if torch.cuda.is_available()else 'cpu') # Get dataset dataset = Dataset("train.txt") loader = DataLoader(dataset, batch_size=hp.batch_size**2, shuffle=True, collate_fn=dataset.collate_fn, drop_last=True, num_workers=0) # Define model model = nn.DataParallel(FastSpeech2()).to(device) print("Model Has Been Defined") num_param = utils.get_param_num(model) print('Number of FastSpeech2 Parameters:', num_param) # Optimizer and loss optimizer = torch.optim.Adam(model.parameters(), betas=hp.betas, eps=hp.eps, weight_decay = hp.weight_decay) scheduled_optim = ScheduledOptim(optimizer, hp.decoder_hidden, hp.n_warm_up_step, args.restore_step) Loss = FastSpeech2Loss().to(device) print("Optimizer and Loss Function Defined.") # Load checkpoint if exists checkpoint_path = os.path.join(hp.checkpoint_path) try: checkpoint = torch.load(os.path.join( checkpoint_path, 'checkpoint_{}.pth.tar'.format(args.restore_step))) model.load_state_dict(checkpoint['model']) optimizer.load_state_dict(checkpoint['optimizer']) print("\n---Model Restored at Step {}---\n".format(args.restore_step)) except: print("\n---Start New Training---\n") if not os.path.exists(checkpoint_path): os.makedirs(checkpoint_path) # read params mean_mel, std_mel = torch.tensor(np.load(os.path.join(hp.preprocessed_path, "mel_stat.npy")), dtype=torch.float).to(device) mean_f0, std_f0 = torch.tensor(np.load(os.path.join(hp.preprocessed_path, "f0_stat.npy")), dtype=torch.float).to(device) mean_energy, std_energy = torch.tensor(np.load(os.path.join(hp.preprocessed_path, "energy_stat.npy")), dtype=torch.float).to(device) mean_mel, std_mel = mean_mel.reshape(1, -1), std_mel.reshape(1, -1) mean_f0, std_f0 = mean_f0.reshape(1, -1), std_f0.reshape(1, -1) mean_energy, std_energy = mean_energy.reshape(1, -1), std_energy.reshape(1, -1) # Load vocoder if hp.vocoder == 'vocgan': vocoder = utils.get_vocgan(ckpt_path = hp.vocoder_pretrained_model_path) vocoder.to(device) else: vocoder = None # Init logger log_path = hp.log_path if not os.path.exists(log_path): os.makedirs(log_path) os.makedirs(os.path.join(log_path, 'train')) os.makedirs(os.path.join(log_path, 'validation')) train_logger = SummaryWriter(os.path.join(log_path, 'train')) val_logger = SummaryWriter(os.path.join(log_path, 'validation')) # Define Some Information Time = np.array([]) Start = time.perf_counter() # Training model = model.train() for epoch in range(hp.epochs): # Get Training Loader total_step = hp.epochs * len(loader) * hp.batch_size for i, batchs in enumerate(loader): for j, data_of_batch in enumerate(batchs): start_time = time.perf_counter() current_step = i*hp.batch_size + j + args.restore_step + epoch*len(loader)*hp.batch_size + 1 # Get Data text = torch.from_numpy(data_of_batch["text"]).long().to(device) mel_target = torch.from_numpy(data_of_batch["mel_target"]).float().to(device) D = torch.from_numpy(data_of_batch["D"]).long().to(device) log_D = torch.from_numpy(data_of_batch["log_D"]).float().to(device) f0 = torch.from_numpy(data_of_batch["f0"]).float().to(device) energy = torch.from_numpy(data_of_batch["energy"]).float().to(device) src_len = torch.from_numpy(data_of_batch["src_len"]).long().to(device) mel_len = torch.from_numpy(data_of_batch["mel_len"]).long().to(device) max_src_len = np.max(data_of_batch["src_len"]).astype(np.int32) max_mel_len = np.max(data_of_batch["mel_len"]).astype(np.int32) # Forward mel_output, mel_postnet_output, log_duration_output, f0_output, energy_output, src_mask, mel_mask, _ = model( text, src_len, mel_len, D, f0, energy, max_src_len, max_mel_len) # Cal Loss mel_loss, mel_postnet_loss, d_loss, f_loss, e_loss = Loss( log_duration_output, log_D, f0_output, f0, energy_output, energy, mel_output, mel_postnet_output, mel_target, ~src_mask, ~mel_mask) total_loss = mel_loss + mel_postnet_loss + d_loss + f_loss + e_loss # Logger t_l = total_loss.item() m_l = mel_loss.item() m_p_l = mel_postnet_loss.item() d_l = d_loss.item() f_l = f_loss.item() e_l = e_loss.item() with open(os.path.join(log_path, "total_loss.txt"), "a") as f_total_loss: f_total_loss.write(str(t_l)+"\n") with open(os.path.join(log_path, "mel_loss.txt"), "a") as f_mel_loss: f_mel_loss.write(str(m_l)+"\n") with open(os.path.join(log_path, "mel_postnet_loss.txt"), "a") as f_mel_postnet_loss: f_mel_postnet_loss.write(str(m_p_l)+"\n") with open(os.path.join(log_path, "duration_loss.txt"), "a") as f_d_loss: f_d_loss.write(str(d_l)+"\n") with open(os.path.join(log_path, "f0_loss.txt"), "a") as f_f_loss: f_f_loss.write(str(f_l)+"\n") with open(os.path.join(log_path, "energy_loss.txt"), "a") as f_e_loss: f_e_loss.write(str(e_l)+"\n") # Backward total_loss = total_loss / hp.acc_steps total_loss.backward() if current_step % hp.acc_steps != 0: continue # Clipping gradients to avoid gradient explosion nn.utils.clip_grad_norm_(model.parameters(), hp.grad_clip_thresh) # Update weights scheduled_optim.step_and_update_lr() scheduled_optim.zero_grad() # Print if current_step % hp.log_step == 0: Now = time.perf_counter() str1 = "Epoch [{}/{}], Step [{}/{}]:".format( epoch+1, hp.epochs, current_step, total_step) str2 = "Total Loss: {:.4f}, Mel Loss: {:.4f}, Mel PostNet Loss: {:.4f}, Duration Loss: {:.4f}, F0 Loss: {:.4f}, Energy Loss: {:.4f};".format( t_l, m_l, m_p_l, d_l, f_l, e_l) str3 = "Time Used: {:.3f}s, Estimated Time Remaining: {:.3f}s.".format( (Now-Start), (total_step-current_step)*np.mean(Time)) print("\n" + str1) print(str2) print(str3) with open(os.path.join(log_path, "log.txt"), "a") as f_log: f_log.write(str1 + "\n") f_log.write(str2 + "\n") f_log.write(str3 + "\n") f_log.write("\n") train_logger.add_scalar('Loss/total_loss', t_l, current_step) train_logger.add_scalar('Loss/mel_loss', m_l, current_step) train_logger.add_scalar('Loss/mel_postnet_loss', m_p_l, current_step) train_logger.add_scalar('Loss/duration_loss', d_l, current_step) train_logger.add_scalar('Loss/F0_loss', f_l, current_step) train_logger.add_scalar('Loss/energy_loss', e_l, current_step) if current_step % hp.save_step == 0: torch.save({'model': model.state_dict(), 'optimizer': optimizer.state_dict( )}, os.path.join(checkpoint_path, 'checkpoint_{}.pth.tar'.format(current_step))) print("save model at step {} ...".format(current_step)) if current_step % hp.eval_step == 0: model.eval() with torch.no_grad(): d_l, f_l, e_l, m_l, m_p_l = evaluate(model, current_step, vocoder) t_l = d_l + f_l + e_l + m_l + m_p_l val_logger.add_scalar('Loss/total_loss', t_l, current_step) val_logger.add_scalar('Loss/mel_loss', m_l, current_step) val_logger.add_scalar('Loss/mel_postnet_loss', m_p_l, current_step) val_logger.add_scalar('Loss/duration_loss', d_l, current_step) val_logger.add_scalar('Loss/F0_loss', f_l, current_step) val_logger.add_scalar('Loss/energy_loss', e_l, current_step) model.train() end_time = time.perf_counter() Time = np.append(Time, end_time - start_time) if len(Time) == hp.clear_Time: temp_value = np.mean(Time) Time = np.delete( Time, [i for i in range(len(Time))], axis=None) Time = np.append(Time, temp_value)
def train_and_evaluate(model, train_dataloader, val_dataloader, optimizer, loss_fn, metrics, params, setting, args, writer=None, logdir=None, restore_file=None): """Train the model and evaluate every epoch. Args: model: (torch.nn.Module) the neural network train_dataloader: (DataLoader) a torch.utils.data.DataLoader object that fetches training data val_dataloader: (DataLoader) a torch.utils.data.DataLoader object that fetches validation data optimizer: (torch.optim) optimizer for parameters of model loss_fn: a function that takes batch_output and batch_labels and computes the loss for the batch metrics: (dict) a dictionary of functions that compute a metric using mnisthe output and labels of each batch params: (Params) hyperparameters model_dir: (string) directory containing config, weights and log restore_file: (string) optional- name of file to restore from (withoutmnistits extension .pth.tar) covar_mode: (bool) does the data-loader give back covariates / additional data """ # setup directories for data setting_home = setting.home if not args.fase == "feature": data_dir = os.path.join(setting_home, "data") else: if setting.mode3d: data_dir = "data" else: data_dir = "slices" covar_mode = setting.covar_mode x_frozen = False best_val_metric = 0.0 if "loss" in setting.metrics[0]: best_val_metric = 1.0e6 val_preds = np.zeros((len(val_dataloader.dataset), params.num_epochs)) for epoch in range(params.num_epochs): # Run one epoch logging.info(f"Epoch {epoch+1}/{params.num_epochs}; setting: {args.setting}, fase {args.fase}, experiment: {args.experiment}") # compute number of batches in one epoch (one full pass over the training set) train_metrics = train(model, optimizer, loss_fn, train_dataloader, metrics, params, setting, writer, epoch) print(train_metrics) for metric_name in train_metrics.keys(): metric_vals = {'train': train_metrics[metric_name]} writer.add_scalars(metric_name, metric_vals, epoch+1) # for name, param in model.named_parameters(): # writer.add_histogram(name, param.clone().cpu().data.numpy(), epoch+1) if epoch % params.save_summary_steps == 0: # Evaluate for one epoch on validation set valid_metrics, outtensors = evaluate(model, loss_fn, val_dataloader, metrics, params, setting, epoch, writer) valid_metrics["intercept"] = model.regressor.fc.bias.detach().cpu().numpy() print(valid_metrics) for name, module in model.regressor.named_children(): if name == "t": valid_metrics["b_t"] = module.weight.detach().cpu().numpy() elif name == "zt": weights = module.weight.detach().cpu().squeeze().numpy().reshape(-1) for i, weight in enumerate(weights): valid_metrics["b_zt"+str(i)] = weight else: pass for metric_name in valid_metrics.keys(): metric_vals = {'valid': valid_metrics[metric_name]} writer.add_scalars(metric_name, metric_vals, epoch+1) # create plots val_df = val_dataloader.dataset.df xx_scatter = net.make_scatter_plot(val_df.x.values, outtensors['xhat'], xlabel='x', ylabel='xhat') xtruex_scatter= net.make_scatter_plot(val_df.x_true.values, outtensors['xhat'], xlabel='x', ylabel='xhat') xyhat_scatter = net.make_scatter_plot(val_df.x.values, outtensors['predictions'], c=val_df.t, xlabel='x', ylabel='yhat') zyhat_scatter = net.make_scatter_plot(val_df.z.values, outtensors['predictions'], c=val_df.t, xlabel='z', ylabel='yhat') yy_scatter = net.make_scatter_plot(val_df.y.values, outtensors['predictions'], c=val_df.t, xlabel='yhat', ylabel='y') writer.add_figure('x-xhat/valid', xx_scatter, epoch+1) writer.add_figure('xtrue-xhat/valid', xtruex_scatter, epoch+1) writer.add_figure('x-yhat/valid', xyhat_scatter, epoch+1) writer.add_figure('z-yhat/valid', zyhat_scatter, epoch+1) writer.add_figure('y-yhat/valid', yy_scatter, epoch+1) if params.save_preds: # writer.add_histogram("predictions", preds) if setting.num_classes == 1: val_preds[:, epoch] = np.squeeze(outtensors['predictions']) # write preds to file pred_fname = os.path.join(setting.home, setting.fase+"-fase", "preds_val.csv") with open(pred_fname, 'ab') as f: np.savetxt(f, preds.T, newline="") np.save(os.path.join(setting.home, setting.fase+"-fase", "preds.npy"), preds) else: val_metric = valid_metrics[setting.metrics[0]] if "loss" in str(setting.metrics[0]): is_best = val_metric<=best_val_metric else: is_best = val_metric>=best_val_metric # Save weights state_dict = model.state_dict() optim_dict = optimizer.state_dict() state = { 'epoch': epoch+1, 'state_dict': state_dict, 'optim_dict': optim_dict } utils.save_checkpoint(state, is_best=is_best, checkpoint=logdir) # If best_eval, best_save_path valid_metrics["epoch"] = epoch if is_best: logging.info("- Found new best {}: {:.3f}".format(setting.metrics[0], val_metric)) best_val_metric = val_metric # Save best val metrics in a json file in the model directory best_json_path = os.path.join(logdir, "metrics_val_best_weights.json") utils.save_dict_to_json(valid_metrics, best_json_path) # Save latest val metrics in a json file in the model directory last_json_path = os.path.join(logdir, "metrics_val_last_weights.json") utils.save_dict_to_json(valid_metrics, last_json_path) # final evaluation writer.export_scalars_to_json(os.path.join(logdir, "all_scalars.json")) if args.save_preds: np.save(os.path.join(setting.home, setting.fase + "-fase", "val_preds.npy"), val_preds)
def train_and_evaluate(model, train_data, val_data, optimizer, scheduler, params, model_dir, restore_file=None): """Train the model and evaluate every epoch.""" # reload weights from restore_file if specified if restore_file is not None: restore_path = os.path.join(args.model_dir, args.restore_file + '.pth.tar') logging.info("Restoring parameters from {}".format(restore_path)) utils.load_checkpoint(restore_path, model, optimizer) best_val_loss = 0.0 patience_counter = 0 for epoch in range(1, params.epoch_num + 1): # Run one epoch logging.info("Epoch {}/{}".format(epoch, params.epoch_num)) # Compute number of batches in one epoch params.train_steps = params.train_size // params.batch_size params.val_steps = params.val_size // params.batch_size # data iterator for training train_data_iterator = data_loader.data_iterator(train_data, shuffle=True) # Train for one epoch on training set train(model, train_data_iterator, optimizer, scheduler, params) # data iterator for evaluation train_data_iterator = data_loader.data_iterator(train_data, shuffle=False) val_data_iterator = data_loader.data_iterator(val_data, shuffle=False) # Evaluate for one epoch on training set and validation set params.eval_steps = params.train_steps train_metrics = evaluate(model, train_data_iterator, params, mark='Train') params.eval_steps = params.val_steps val_metrics = evaluate(model, val_data_iterator, params, mark='Val') print("val metrics :", val_metrics) val_loss = val_metrics['loss'] improve_loss = val_loss - best_val_loss # Save weights of the network model_to_save = model.module if hasattr( model, 'module') else model # Only save the model it-self optimizer_to_save = optimizer.optimizer if args.fp16 else optimizer utils.save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model_to_save.state_dict(), 'optim_dict': optimizer_to_save.state_dict() }, is_best=improve_loss > 0, checkpoint=model_dir) if improve_loss > 0: logging.info("- Found new best loss") best_val_loss = val_loss if improve_loss < params.patience: patience_counter += 1 else: patience_counter = 0 else: patience_counter += 1 # Early stopping and logging best f1 if (patience_counter >= params.patience_num and epoch > params.min_epoch_num) or epoch == params.epoch_num: logging.info("Best val loss: {:05.2f}".format(best_val_loss)) break
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Wed Sep 16 13:01:34 2020 @author: ziyi """ from evaluate import evaluate agentName = 'worse' evaluate(agentName)
def evaluate_wrapper(pred, lab): # pred = np.argmax(pred.detach().cpu().numpy(), 1) pred = pred.detach().cpu().numpy() lab = lab.cpu().numpy() return evaluate(pred, lab)
trainWriter = tf.summary.FileWriter('../report/tf-log/train', graph=sess.graph) testWriter = tf.summary.FileWriter('../report/tf-log/test', graph=sess.graph) tf.logging.info((green(SimpleNet.print_total_params()))) while iters.eval() <= iter_limit: current_iter = iters.eval() if current_iter%5==0: tf.logging.info(f'Iter:{iters.eval()}...') batch_xs, batch_ys = dh.get_next_batch(iter_based=True, split_channels=True) sess.run(update, feed_dict={x:batch_xs, y:batch_ys, keep_prob:dropout_keep}) if current_iter % 10 == 0: evals = evaluate.evaluate(sess, infer, x, y, keep_prob, batch_xs, batch_ys, dropout_keep, '../report/train_progress.csv') s = sess.run(summaries, feed_dict={x:batch_xs, y:batch_ys, keep_prob:-1.0}) trainWriter.add_summary(s, current_iter) if current_iter % 50 == 0: tf.logging.info(yellow('Testing...')) batch_xs, batch_ys = dh.get_next_batch(iter_based=True, force_test=True, split_channels=True) evals = evaluate.evaluate(sess, infer, x, y, keep_prob, batch_xs, batch_ys, dropout_keep, '../report/test_progress.csv') # run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) # run_metadata = tf.RunMetadata() # s = sess.run(summaries, # feed_dict={x:batch_xs, y:batch_ys}, # options=run_options, # run_metadata=run_metadata) # testWriter.add_run_metadata(run_metadata, f'train{current_iter}')
def train_and_evaluate(model, train_data, val_data, optimizer_1, optimizer_2, loss_fn, metrics, params, model_dir, restore_file=None): """Train the model and evaluate every epoch. Args: model: (torch.nn.Module) the neural network train_data: (dict) training data with keys 'data' and 'labels' val_data: (dict) validaion data with keys 'data' and 'labels' optimizer: (torch.optim) optimizer for parameters of model loss_fn: a function that takes batch_output and batch_labels and computes the loss for the batch metrics: (dict) a dictionary of functions that compute a metric using the output and labels of each batch params: (Params) hyperparameters model_dir: (string) directory containing config, weights and log restore_file: (string) optional- name of file to restore from (without its extension .pth.tar) """ # print(train_data) # reload weights from restore_file if specified if restore_file is not None: restore_path = os.path.join(args.model_dir, args.restore_file + '.pth.tar') logging.info("Restoring parameters from {}".format(restore_path)) utils.load_checkpoint(restore_path, model, optimizer) best_val_acc = 0.0 savecoeff = 0 for epoch in range(params.num_epochs): logging.info("Epoch {}/{}".format(epoch + 1, params.num_epochs)) train(model, optimizer_1, optimizer_2, loss_fn, train_data, metrics, params) val_metrics = evaluate(model, loss_fn, val_data, metrics, params) val_acc = val_metrics['accuracy'] is_best = val_acc >= best_val_acc # Save weights savecoeff = savecoeff + 1 utils.save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'optimw_1_dict': optimizer_1.state_dict(), 'optimw_2_dict': optimizer_2.state_dict() }, is_best=is_best, checkpoint=model_dir) if savecoeff % 5 == 0: fpath = osp.join(model_dir, 'model' + str(epoch) + '.pth.tar-' + str(epoch)) torch.save( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'optim_1_dict': optimizer_1.state_dict(), 'optim_2_dict': optimizer_2.state_dict() }, fpath) # If best_eval, best_save_path if is_best: logging.info("- Found new best accuracy") best_val_acc = val_acc # Save best val metrics in a json file in the model directory best_json_path = os.path.join(model_dir, "metrics_val_best_weights.json") utils.save_dict_to_json(val_metrics, best_json_path) # Save latest val metrics in a json file in the model directory last_json_path = os.path.join(model_dir, "metrics_val_last_weights.json") utils.save_dict_to_json(val_metrics, last_json_path)
def train_evaluate(model, optimizer, scheduler, dataloader_train, dataloader_val, loss_fn, metric_fn, model_dir, param): best_SILog_val = float('inf') epoch_start = param['current_epoch'] for epoch in range(epoch_start, param['epochs']): print() print('------ Epoch %d, Learning rate = %.2e ------' % (epoch, optimizer.param_groups[0]['lr'])) train(model, optimizer, dataloader_train, loss_fn, metric_fn, param, epoch) # metrics_test = evaluate(model, dataloader_val, loss_fn, metric_fn, param, epoch, writer) if isinstance(dataloader_val, dict): metrics_val_current = evaluate(model, dataloader_val['current'], loss_fn, metric_fn, param, epoch, writer) metrics_val_current = {k + '_val_current': v for k, v in metrics_val_current.items()} print( '\n------ After training %d epochs, current snowfall\'s validation set metrics mean: ------ \n%s\n' % (epoch, metrics_val_current)) for metric, value in metrics_val_current.items(): writer.add_scalar(metric, value, epoch) metrics_val_all = evaluate(model, dataloader_val['all'], loss_fn, metric_fn, param, epoch, writer) metrics_val_all = {k + '_val_all': v for k, v in metrics_val_all.items()} print( '\n------ After training %d epochs, all CADC validation set metrics mean: ------ \n%s\n' % (epoch, metrics_val_all)) for metric, value in metrics_val_all.items(): writer.add_scalar(metric, value, epoch) metrics_val = {**metrics_val_current, **metrics_val_all} SILog_val = metrics_val_current['SILog_val_current'] else: metrics_val = evaluate(model, dataloader_val, loss_fn, metric_fn, param, epoch, writer) print( '\n------ After training %d epochs, validation set metrics mean: ------ \n%s\n' % (epoch, metrics_val)) for metric, value in metrics_val.items(): writer.add_scalar(metric, value, epoch) SILog_val = metrics_val['SILog'] if scheduler is not None: scheduler.step(SILog_val) is_best = SILog_val <= best_SILog_val # Save weights save_dict = param.copy() save_dict['current_epoch'] = epoch save_dict['state_dict'] = model.state_dict() save_dict['optim_dict'] = optimizer.state_dict() if scheduler is not None: save_dict['sched_dict'] = scheduler.state_dict() experiments_dir = 'experiments/' + model_dir utils.save_checkpoint(save_dict, is_best=is_best, folder_path=experiments_dir) # If best_eval, best_save_path if is_best: best_SILog_val = SILog_val # Save best val metrics in a json file in the model directory best_json_path = os.path.join(experiments_dir, "metrics_test_best_weights.json") utils.save_dict_to_json(metrics_val, best_json_path) # Save latest val metrics in a json file in the model directory last_json_path = os.path.join(experiments_dir, "metrics_test_last_weights.json") utils.save_dict_to_json(metrics_val, last_json_path)
def main(): torch.set_default_tensor_type(torch.FloatTensor) torch.set_num_threads(3) if not torch.cuda.is_available(): logging.info('no gpu device available') sys.exit(1) np.random.seed(args.seed) torch.cuda.set_device(args.gpu) cudnn.benchmark = True torch.manual_seed(args.seed) cudnn.enabled = True torch.cuda.manual_seed(args.seed) logging.info('gpu device = %d' % args.gpu) logging.info("args = %s", args) data_start = time.time() if args.dataset == 'ml-100k': num_users = 943 num_items = 1682 dim = 2 elif args.dataset == 'ml-1m': num_users = 6040 num_items = 3952 dim = 2 elif args.dataset == 'ml-10m': num_users = 71567 num_items = 65133 dim = 2 elif args.dataset == 'youtube-small': num_ps = 600 num_qs = 14340 num_rs = 5 dim = 3 train_queue, valid_queue, test_queue = utils.get_data_queue(args) logging.info('prepare data finish! [%f]' % (time.time() - data_start)) if args.mode == 'libfm': start = time.time() from tffm import TFFMRegressor import tensorflow as tf os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu) model = TFFMRegressor( order=dim, rank=args.embedding_dim, optimizer=tf.train.AdagradOptimizer(learning_rate=args.lr), n_epochs=args.train_epochs, batch_size=args.batch_size, init_std=0.001, reg=args.weight_decay, input_type='sparse', log_dir=os.path.join(save_name, 'libfm-log')) model.fit(train_queue[0], train_queue[1], show_progress=True) inferences = model.predict(test_queue[0]) mse = mean_squared_error(test_queue[1], inferences) rmse = np.sqrt(mse) logging.info('rmse: %.4f[%.4f]' % (rmse, time.time() - start)) else: start = time.time() if args.mode == 'ncf': if dim == 2: model = NCF(num_users, num_items, args.embedding_dim, args.weight_decay).cuda() elif dim == 3: model = NCF_Triple(num_ps, num_qs, num_rs, args.embedding_dim, args.weight_decay).cuda() elif args.mode == 'deepwide': if dim == 2: model = DeepWide(num_users, num_items, args.embedding_dim, args.weight_decay).cuda() elif dim == 3: model = DeepWide_Triple(num_ps, num_qs, num_rs, args.embedding_dim, args.weight_decay).cuda() elif args.mode == 'altgrad': model = AltGrad(num_users, num_items, args.embedding_dim, args.weight_decay).cuda() elif args.mode == 'convncf': model = ConvNCF(num_users, num_items, args.embedding_dim, args.weight_decay).cuda() elif args.mode == 'outer': model = Outer(num_users, num_items, args.embedding_dim, args.weight_decay).cuda() elif args.mode == 'conv': model = Conv(num_users, num_items, args.embedding_dim, args.weight_decay).cuda() elif args.mode == 'plus': model = Plus(num_users, num_items, args.embedding_dim, args.weight_decay).cuda() elif args.mode == 'max': model = Max(num_users, num_items, args.embedding_dim, args.weight_decay).cuda() elif args.mode == 'min': model = Min(num_users, num_items, args.embedding_dim, args.weight_decay).cuda() elif args.mode == 'cp': model = CP(num_ps, num_qs, num_rs, args.embedding_dim, args.weight_decay).cuda() elif args.mode == 'tucker': model = TuckER(num_ps, num_qs, num_rs, args.embedding_dim, args.weight_decay).cuda() elif args.mode == 'sif': if dim == 2: arch = utils.load_arch(num_users, num_items, args) print(next(arch['mlp']['p'].parameters())) model = Network(num_users, num_items, args.embedding_dim, arch, args.weight_decay).cuda() elif dim == 3: arch = utils.load_arch_triple(num_ps, num_qs, num_rs, args) model = Network_Triple(num_ps, num_qs, num_rs, args.embedding_dim, arch, args.weight_decay).cuda() logging.info('build model finish! [%f]' % (time.time() - start)) optimizer = torch.optim.Adagrad(model.parameters(), args.lr) if dim == 2: train(model, train_queue, test_queue, optimizer, args) rmse = evaluate(model, test_queue) elif dim == 3: train_triple(model, train_queue, test_queue, optimizer, args) rmse = evaluate_triple(model, test_queue) logging.info('rmse: %.4f' % rmse)
def train(self): self.build() analyze_vars(tf.trainable_variables(), os.path.join(self.output_dir, 'model_vars.txt')) with open(os.path.join(self.output_dir, 'regularizers.txt'), 'w') as f: for v in tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES): f.write(v.name + '\n') # exit(-1) tf_config = tf.ConfigProto(allow_soft_placement=True) tf_config.gpu_options.allow_growth = True with tf.Session(config=tf_config) as sess: tf.global_variables_initializer().run() saver_ckpt = tf.train.Saver() saver_best = tf.train.Saver() summary_writer = tf.summary.FileWriter(self.log_dir, sess.graph) start_time = time.time() best_acc = 0 counter = 0 if config['pretrained_model'] != '': saver_ckpt.restore(sess, config['pretrained_model']) step = int( os.path.basename(config['pretrained_model']).split('.') [0].split('-')[-1]) sess.run(tf.assign(self.global_step, step)) counter = self.global_step.eval(sess) print('start step: %d' % counter) debug = True for i in range(self.epoch_num): for j in range(self.step_per_epoch): _, l, l_wd, l_inf, acc, s, _ = sess.run( [ self.train_op, self.train_loss, self.wd_loss, self.inference_loss, self.train_acc, self.train_summary, self.inc_op ], feed_dict={ self.train_phase_dropout: True, self.train_phase_bn: True }) counter += 1 # debug # self.save_image_label(train_img, train_lbl, counter) # if(debug): # if(len(train_imgs) < 100): # train_imgs.append(train_img[0]) # else: # np.save(os.path.join(self.debug_dir, 'train_imgs.npy'), np.array(train_imgs)) # debug=False print( "Epoch: [%2d/%2d] [%6d/%6d] time: %.2f, loss: %.3f (inference: %.3f, wd: %.3f), acc: %.3f" % (i, self.epoch_num, j, self.step_per_epoch, time.time() - start_time, l, l_inf, l_wd, acc)) start_time = time.time() if counter % self.val_freq == 0: saver_ckpt.save(sess, os.path.join(self.checkpoint_dir, 'ckpt-m'), global_step=counter) acc = [] with open(self.val_log, 'a') as f: f.write('step: %d\n' % counter) for k, v in self.val_data.items(): imgs, imgs_f, issame = load_bin( v, self.image_size) embds = self.run_embds(sess, imgs) embds_f = self.run_embds(sess, imgs_f) embds = embds / np.linalg.norm( embds, axis=1, keepdims=True) + embds_f / np.linalg.norm( embds_f, axis=1, keepdims=True) tpr, fpr, acc_mean, acc_std, tar, tar_std, far = evaluate( embds, issame, far_target=1e-3, distance_metric=0) f.write( 'eval on %s: acc--%1.5f+-%1.5f, tar--%1.5f+-%1.5f@far=%1.5f\n' % (k, acc_mean, acc_std, tar, tar_std, far)) acc.append(acc_mean) acc = np.mean(np.array(acc)) if acc > best_acc: saver_best.save(sess, os.path.join( self.model_dir, 'best-m'), global_step=counter) best_acc = acc
#score.extend(sc) reference.extend(ref) pred_all.extend(pre) print(str(i) + '_json readed.') length = len(target_words) assert len(reference) == length map_freq = [[], [], [], []] map_dn = [[], [], [], [], [], []] map = [] target_words0 = [] for i in range(10): target_words0.extend((open(str(i) + 'target_words.txt', 'r').readlines()[0]).split()) for i in range(length): m = evaluate(reference[i], pred_all[i]) map.append(m) map_freq[wf_ind[target_words0[i]]].append(m) if dn_ind[target_words0[i]] > 5: map_dn[5].append(m) else: map_dn[dn_ind[target_words0[i]] - 1].append(m) print('MAP in each word frequences:') for m in map_freq: print(np.mean(np.array(m))) print('MAP in each defi numbers:') for m in map_dn: print(np.mean(np.array(m))) print('-----MAP: ', np.mean(np.array(map)))
def train_and_evaluate(model, params, restore_file=None): """Train the model and evaluate every epoch.""" # load args args = parser.parse_args() # Load training data and val data dataloader = NERDataLoader(params) train_loader = dataloader.get_dataloader(data_sign='train') val_loader = dataloader.get_dataloader(data_sign='val') # 一个epoch的步数 params.train_steps = len(train_loader) # Prepare optimizer # fine-tuning # 取模型权重 param_optimizer = list(model.named_parameters()) # pretrain model param param_pre = [(n, p) for n, p in param_optimizer if 'bert' in n] # middle model param param_middle = [(n, p) for n, p in param_optimizer if 'bilstm' in n or 'dym_weight' in n] # crf param param_crf = [p for n, p in param_optimizer if 'crf' in n] # 不进行衰减的权重 no_decay = ['bias', 'LayerNorm', 'dym_weight', 'layer_norm'] # 将权重分组 optimizer_grouped_parameters = [ # pretrain model param # 衰减 {'params': [p for n, p in param_pre if not any(nd in n for nd in no_decay)], 'weight_decay': params.weight_decay_rate, 'lr': params.fin_tuning_lr }, # 不衰减 {'params': [p for n, p in param_pre if any(nd in n for nd in no_decay)], 'weight_decay': 0.0, 'lr': params.fin_tuning_lr }, # middle model # 衰减 {'params': [p for n, p in param_middle if not any(nd in n for nd in no_decay)], 'weight_decay': params.weight_decay_rate, 'lr': params.downs_en_lr }, # 不衰减 {'params': [p for n, p in param_middle if any(nd in n for nd in no_decay)], 'weight_decay': 0.0, 'lr': params.downs_en_lr }, # crf,单独设置学习率 {'params': param_crf, 'weight_decay': 0.0, 'lr': params.crf_lr} ] num_train_optimization_steps = len(train_loader) // params.gradient_accumulation_steps * args.epoch_num optimizer = BertAdam(optimizer_grouped_parameters, warmup=params.warmup_prop, schedule="warmup_cosine", t_total=num_train_optimization_steps, max_grad_norm=params.clip_grad) # TODO: reload weights from restore_file if specified(DataParallel) if restore_file is not None: restore_path = os.path.join(params.model_dir, args.restore_file + '.pth.tar') logging.info("Restoring parameters from {}".format(restore_path)) # 读取checkpoint utils.load_checkpoint(restore_path, model, optimizer) # patience stage best_val_f1 = 0.0 patience_counter = 0 for epoch in range(1, args.epoch_num + 1): # Run one epoch logging.info("Epoch {}/{}".format(epoch, args.epoch_num)) # Train for one epoch on training set train(model, train_loader, optimizer, params) # Evaluate for one epoch on training set and validation set # train_metrics = evaluate(model, train_loader, params, mark='Train', # verbose=True) # Dict['loss', 'f1'] val_metrics = evaluate(args, model, val_loader, params, mark='Val', verbose=True) # Dict['loss', 'f1'] # 验证集f1-score val_f1 = val_metrics['f1'] # 提升的f1-score improve_f1 = val_f1 - best_val_f1 # Save weights of the network model_to_save = model.module if hasattr(model, 'module') else model # Only save the model it-self optimizer_to_save = optimizer utils.save_checkpoint({'epoch': epoch + 1, 'state_dict': model_to_save.state_dict(), 'optim_dict': optimizer_to_save.state_dict()}, is_best=improve_f1 > 0, checkpoint=params.model_dir) params.save(params.params_path / 'params.json') # stop training based params.patience if improve_f1 > 0: logging.info("- Found new best F1") best_val_f1 = val_f1 if improve_f1 < params.patience: patience_counter += 1 else: patience_counter = 0 else: patience_counter += 1 # Early stopping and logging best f1 if (patience_counter > params.patience_num and epoch > params.min_epoch_num) or epoch == args.epoch_num: logging.info("Best val f1: {:05.2f}".format(best_val_f1)) break
criterion = nn.CrossEntropyLoss(ignore_index=TRG_PAD_IDX) if ARGS.train: print(f'The model has {count_parameters(model):,} trainable parameters') optimizer = optim.Adam(model.parameters()) N_EPOCHS = 10 CLIP = 0.1 best_valid_loss = float('inf') for epoch in range(N_EPOCHS): start_time = time.time() train_loss = train(model, train_iterator, optimizer, criterion, CLIP) valid_loss = evaluate(model, valid_iterator, criterion) end_time = time.time() epoch_mins, epoch_secs = epoch_time(start_time, end_time) if valid_loss < best_valid_loss: best_valid_loss = valid_loss torch.save(model.state_dict(), 'tut5-model.pt') print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s') print( f'\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}' ) print( f'\t Val. Loss: {valid_loss:.3f} | Val. PPL: {math.exp(valid_loss):7.3f}'
def train_and_evaluate(model, train_dataloader, val_dataloader, optimizer, critierion, metrics, params, model_dir, restore_file=None): # reload weights from restore_file if specified if restore_file is not None: restore_path = os.path.join(args.model_dir, args.restore_file + '.pth.tar') logging.info("Restoring parameters from {}".format(restore_path)) utils.load_checkpoint(restore_path, model, optimizer) best_val_acc = 0.0 best_val_metrics = [] learning_rate_0 = params.learning_rate train_acc_series = [] val_acc_series = [] train_loss_series = [] for epoch in range(params.num_epochs): logging.info("Epoch {}/{}".format(epoch + 1, params.num_epochs)) # train model train_metrics = train(model, train_dataloader, optimizer, critierion, metrics, params) # learning rate exponential decay params.learning_rate = learning_rate_0 * np.exp(-params.exp_decay_k * epoch) # evaluate val_metrics = evaluate(model, critierion, val_dataloader, metrics, params) # find accuracy from validation dataset val_acc = val_metrics['accuracy'] is_best = val_acc >= best_val_acc # save weights utils.save_checkpoint({'epoch': epoch + 1, 'state_dict': model.state_dict(), 'optim_dict': optimizer.state_dict()}, is_best=is_best, checkpoint=model_dir) # save accuracy / loss to array for plot train_acc_series.append(train_metrics['accuracy']) val_acc_series.append(val_metrics['accuracy']) train_loss_series.append(train_metrics['loss']) # If best_eval, best_save_path if is_best: logging.info("- Found new best accuracy") best_val_acc = val_acc best_val_metrics = val_metrics # Save best val metrics in a json file in the model directory best_json_path = os.path.join( model_dir, "metrics_val_best_weights.json") utils.save_dict_to_json(val_metrics, best_json_path) # Save latest val metrics in a json file in the model directory last_json_path = os.path.join( model_dir, "metrics_val_last_weights.json") utils.save_dict_to_json(val_metrics, last_json_path) print('******************************************') # plot visualized performance visualize.plot_train_val_accuracy(train_acc_series, val_acc_series) visualize.plot_loss(train_loss_series) # save best validation F1 score plot visualize.plot_individual_label_f1score(best_val_metrics)
def run(model, dataloaders, loss_fn, optimizer, lr_scheduler, device, params, memo): """ Runs train and validation every epoch while updating `model`'s parameters using the `loss_fn` and `optimizers`. `optimizer`'s learning rates are updated during the training via `lr_scheduler`. Before calling this function, put the model in `device`, and *then* set this model's parameters as optimizer's target variables: ```python # Example model = model.to(device) optimizer = optim.Adam(model.parameters(), lr=dummy_lr) # dummy_lr because it will always be # overwritten by lr_scheduler's returned value dataloaders = {'train': train_dl, 'val': val_dl} loss_fn = nn.CrossEntropy(...) lr_scheduler = TriangleLR(...) params = {'max_epoch': , # default: 50, 'batch_size': , # default: 8 , 'n_classes' : 21, #default: 21 'fill_value': , # default: 255 , 'save_every': , # default: 10 , 'print_every': , # default: 5, 'ignore_idx': 255, # default: 255 'ignore_bg': True # default: True 'debug': False # default: False } result = run(model, dataloaders, loss_fn, optimizer, lr_scheduler, device, params) ``` Assumes - model is already sent to the `device` - optimizer has been constructed with model.parameters() as input - `device` will be used to send the data (returned from dataloaders) to the `device` that is the same device where the input `model` is located """ # Get parameters max_epoch = params.get('max_epoch', 50) batch_size = params.get('batch_size', 8) best_iou = params.get('best_iou', -100) fill_value = params.get('fill_value', 255) ignore_gt = params.get( 'ignore_gt', 0 ) # None or, one of the class labels to be removed from acc computation save_every = params.get('save_every', 10) # unit of epoch print_every = params.get('print_every', 5) # unit of epoch log_fn = params.get( 'log_fn', f'/data/rl/log/{model.name}-fill-{fill_value}-bs-{batch_size}/{now2str()}/log.txt' ) # Get dataloaders train_dl, val_dl = dataloaders['train'], dataloaders['val'] assert batch_size == train_dl.batch_size, "batch_size must be the batchsize of train_dl" # Set log file directory if not existing yet if not isinstance(log_fn, Path): log_fn = Path(log_fn) if not log_fn.parent.exists(): log_fn.parent.mkdir(parents=True) print('Created : ', str(log_fn.parent)) print('log file: ', log_fn) # Start experiment all_train_losses = [] # from each iteration all_train_accs = [] ep_train_losses = [] # averge losses from each epoch ep_train_accs = [] val_losses = [] # from each epoch val_accs = [] val_ious = [] #best_iou is set above where the parameters are read from the input `params` dict exp_start = time.time() for epoch in tqdm(range(max_epoch), desc='Epoch'): # Train start = time.time() train_result = train(model, train_dl, loss_fn, optimizer, lr_scheduler, device, params, memo) end = time.time() # Collect train metrics all_train_losses.extend(train_result['train_losses']) all_train_accs.extend(train_result['train_accs']) ep_train_losses.append(train_result['loss_meter'].avg) ep_train_accs.append(train_result['acc_meter'].avg) # Logging after each train epoch append2file( f"{'='*80}" f'\nEpoch: {epoch}/{max_epoch}' f'\n\tTrain took: {(end-start)/60.0:.3f} mins' f'\n\tTrain loss: {ep_train_losses[-1]:9.5f}' f'\n\tTrain acc: {ep_train_accs[-1]:9.5f}%', log_fn) ## train_result = {'running_metrics': running_metrics, # 'loss_meter': loss_meter, # 'acc_meter': acc_meter, # 'train_losses': train_losses, # 'train_accs': train_accs} # Evaluate val_result = evaluate(model, val_dl, loss_fn, device, params) # Collect validation metrics val_losses.append(val_result['loss_meter'].avg) val_accs.append(val_result['acc_meter'].avg) score, class_iou = val_result['running_metrics'].get_scores( ignore_gt=ignore_gt) val_ious.append(score["Mean IoU : \t"]) # Log evaluation for this epoch mean_acc = score['Mean Acc : \t'] append2file( f"\n\tVal loss: {val_losses[-1]:9.5f}" f"\n\tVal acc: {val_accs[-1]:9.5f}%" f"\n\t\t vs. {mean_acc}" f"\n\tVal mean IOU: {val_ious[-1]:9.5f}", log_fn) for k, v in score.items(): # print(k, v) ##printing here append2file(f'\tVal_metric/{k} -- {v:9.5f}', log_fn) for k, v in class_iou.items(): append2file(f'\tVal Class IOU/{k} -- {v:9.5f}', log_fn) # Save current state if it achieves the best IOU on validation set if best_iou < 0 or (score["Mean IoU : \t"] - best_iou) / best_iou > 0.05: best_iou = score["Mean IoU : \t"] state = { "epoch": epoch + 1, "model_state": model.state_dict(), "optimizer_state": optimizer.state_dict(), "best_iou": best_iou, } torch.save(state, log_fn.parent / 'best_state.pth') print( f"Updated a new best state. ep: {state['epoch']}, iou: {state['best_iou']}" ) for k, v in score.items(): print(k, v) ##printing here # Save the current model if it's time to save intermittently if (epoch + 1) % save_every == 0: print("=" * 80) print('Epoch: ', epoch, ' Saved model') out_fn = log_fn.parent / f'{model.name}_{epoch}.pth' save_checkpt(epoch, model, optimizer, ep_train_losses[-1], out_fn) ## result = {'running_metrics': running_metrics, # 'loss_meter': loss_meter, # 'acc_meter': acc_meter} # Log this experiment's train and val losses out_fn = log_fn.parent / (log_fn.stem + '_losses.npz') np.savez_compressed(out_fn, train_losses=all_train_losses, train_accs=all_train_accs, val_losses=val_losses, val_accs=val_accs) print('Saved the losses to...: ', out_fn) print(f'Experiement took : {(time.time() - exp_start)/60:.3f} mins') result = { 'train': { 'loss': ep_train_losses, 'acc': ep_train_accs }, 'val': { 'loss': val_losses, 'acc': val_accs } } return model, result
def _run_epoch_eval(self, sess, y_prob, labels, metric): score = evaluate.evaluate(labels, y_prob, metric=metric, average=True) return score
def main(): args = parser.parse_args() # torch setting torch.random.manual_seed(args.seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # os setting path = args.dataset_path train_path = os.path.join(path, "train/train.txt") validation_path = os.path.join(path, "valid/valid.txt") test_path = os.path.join(path, "test/test.txt") params_path = os.path.join(args.model_dir, 'params.json') checkpoint_dir = os.path.join(args.model_dir, 'checkpoint') tensorboard_log_dir = os.path.join(args.model_dir, 'log') utils.check_dir(tensorboard_log_dir) entity2id, relation2id = data_loader.create_mappings(train_path) # params params = utils.Params(params_path) params.device = torch.device( 'cuda') if torch.cuda.is_available() else torch.device('cpu') # dataset train_set = data_loader.FB15KDataset(train_path, entity2id, relation2id) train_generator = torch_data.DataLoader(train_set, batch_size=params.batch_size) validation_set = data_loader.FB15KDataset(validation_path, entity2id, relation2id) validation_generator = torch_data.DataLoader( validation_set, batch_size=params.validation_batch_size) test_set = data_loader.FB15KDataset(test_path, entity2id, relation2id) test_generator = torch_data.DataLoader( test_set, batch_size=params.validation_batch_size) # model model = net.Net(entity_count=len(entity2id), relation_count=len(relation2id), dim=params.embedding_dim, margin=params.margin, device=params.device, norm=params.norm) # type: torch.nn.Module model = model.to(params.device) optimizer = optim.SGD(model.parameters(), lr=params.learning_rate) summary_writer = tensorboard.SummaryWriter(log_dir=tensorboard_log_dir) start_epoch_id = 1 step = 0 best_score = 0.0 print("Training Dataset: entity: {} relation: {} triples: {}".format( len(entity2id), len(relation2id), len(train_set))) print("Validation Dataset: triples: {}".format(len(validation_set))) print("Test Dataset: triples: {}".format(len(test_set))) print(model) # Train for epoch_id in range(start_epoch_id, params.epochs + 1): print("Epoch {}/{}".format(epoch_id, params.epochs)) loss_impacting_samples_count = 0 samples_count = 0 model.train() with tqdm(total=len(train_generator)) as t: for local_heads, local_relations, local_tails in train_generator: local_heads, local_relations, local_tails = (local_heads.to( params.device), local_relations.to( params.device), local_tails.to(params.device)) positive_triples = torch.stack( (local_heads, local_relations, local_tails), dim=1) # Preparing negatives. # Generate binary tensor to replace either head or tail. 1 means replace head, 0 means replace tail. head_or_tail = torch.randint(high=2, size=local_heads.size(), device=params.device) random_entities = torch.randint(high=len(entity2id), size=local_heads.size(), device=params.device) broken_heads = torch.where(head_or_tail == 1, random_entities, local_heads) broken_tails = torch.where(head_or_tail == 0, random_entities, local_tails) negative_triples = torch.stack( (broken_heads, local_relations, broken_tails), dim=1) optimizer.zero_grad() loss, pd, nd = model(positive_triples, negative_triples) loss.mean().backward() summary_writer.add_scalar('Loss/train', loss.mean().data.cpu().numpy(), global_step=step) summary_writer.add_scalar('Distance/positive', pd.sum().data.cpu().numpy(), global_step=step) summary_writer.add_scalar('Distance/negative', nd.sum().data.cpu().numpy(), global_step=step) loss = loss.data.cpu() loss_impacting_samples_count += loss.nonzero().size()[0] samples_count += loss.size()[0] optimizer.step() step += 1 t.set_postfix(loss=loss_impacting_samples_count / samples_count * 100) t.update() summary_writer.add_scalar('Metrics/batch_loss', loss_impacting_samples_count / samples_count * 100, global_step=epoch_id) # validation if epoch_id % params.validation_freq == 0: model.eval() _, _, hits_at_10, _ = evaluate( model=model, data_generator=validation_generator, entities_count=len(entity2id), device=params.device, summary_writer=summary_writer, epoch_id=epoch_id, metric_suffix="val") score = hits_at_10 if score > best_score: best_score = score utils.save_checkpoint(checkpoint_dir, model, optimizer, epoch_id, step, best_score) # Testing the best checkpoint on test dataset utils.load_checkpoint(checkpoint_dir, model, optimizer) best_model = model.to(params.device) best_model.eval() scores = evaluate(model=best_model, data_generator=test_generator, entities_count=len(entity2id), device=params.device, summary_writer=summary_writer, epoch_id=1, metric_suffix="test") print("Test scores: \n hit%1: {} \n hit%3: {} \nh it%10: {} \n mrr: {}". format(scores[0], scores[1], scores[2], scores[3])) eval_path = os.path.join(args.model_dir, 'eval.json') evals_params = utils.Params(eval_path) evals_params.hit_1 = scores[0] evals_params.hit_3 = scores[1] evals_params.hit_10 = scores[2] evals_params.mrr = scores[3] evals_params.best_score = best_score evals_params.save(eval_path)
def main(args, configs): print("Prepare training ...") preprocess_config, model_config, train_config = configs # Get dataset dataset = Dataset("train.txt", preprocess_config, train_config, model_config, sort=True, drop_last=True) batch_size = train_config["optimizer"]["batch_size"] group_size = 4 # Set this larger than 1 to enable sorting in Dataset assert batch_size * group_size < len(dataset) loader = DataLoader( dataset, batch_size=batch_size * group_size, shuffle=True, collate_fn=dataset.collate_fn, ) # Prepare model model, optimizer = get_model(args, configs, device, train=True) model = nn.DataParallel(model) num_param = get_param_num(model) Loss = FastSpeech2Loss(preprocess_config, model_config).to(device) print("Number of FastSpeech2 Parameters:", num_param) # Load checkpoint if exists if args.restore_path is not None and os.path.isfile(args.restore_path): checkpoint = torch.load(args.restore_path) pretrained_dict = checkpoint['model'] if not any(key.startswith('module.') for key in pretrained_dict): pretrained_dict = { 'module.' + k: v for k, v in pretrained_dict.items() } dem1 = 0 dem2 = 0 model_dict = model.state_dict() for k, v in pretrained_dict.items(): if k in model_dict and v.size() == model_dict[k].size(): # print('Load weight in ', k) dem1 += 1 else: print(f'Module {k} is not same size') dem2 += 1 dem2 += dem1 print(f'### Load {dem1}/{dem2} modules') # 1. filter out unnecessary keys pretrained_dict = { k: v for k, v in pretrained_dict.items() if k in model_dict and v.size() == model_dict[k].size() } # 2. overwrite entries in the existing state dict model_dict.update(pretrained_dict) # 3. load the new state dict model.load_state_dict(model_dict) # model.load_state_dict(checkpoint['model']) # optimizer.load_state_dict(checkpoint['optimizer']) print("\n---Model Restored at Step {}---\n".format(args.restore_step)) # Load vocoder vocoder = get_vocoder(model_config, device) # Init logger for p in train_config["path"].values(): os.makedirs(p, exist_ok=True) train_log_path = os.path.join(train_config["path"]["log_path"], "train") val_log_path = os.path.join(train_config["path"]["log_path"], "val") os.makedirs(train_log_path, exist_ok=True) os.makedirs(val_log_path, exist_ok=True) train_logger = SummaryWriter(train_log_path) val_logger = SummaryWriter(val_log_path) # Training step = args.restore_step + 1 epoch = 1 grad_acc_step = train_config["optimizer"]["grad_acc_step"] grad_clip_thresh = train_config["optimizer"]["grad_clip_thresh"] total_step = train_config["step"]["total_step"] log_step = train_config["step"]["log_step"] save_step = train_config["step"]["save_step"] synth_step = train_config["step"]["synth_step"] val_step = train_config["step"]["val_step"] outer_bar = tqdm(total=total_step, desc="Training", position=0) outer_bar.n = args.restore_step outer_bar.update() while True: inner_bar = tqdm(total=len(loader), desc="Epoch {}".format(epoch), position=1) for batchs in loader: for batch in batchs: batch = to_device(batch, device) # Forward output = model(*(batch[2:])) # Cal Loss losses = Loss(batch, output) total_loss = losses[0] # Backward total_loss = total_loss / grad_acc_step total_loss.backward() if step % grad_acc_step == 0: # Clipping gradients to avoid gradient explosion nn.utils.clip_grad_norm_(model.parameters(), grad_clip_thresh) # Update weights optimizer.step_and_update_lr() optimizer.zero_grad() if step % log_step == 0: losses = [l.item() for l in losses] message1 = "Step {}/{}|".format(step, total_step) message2 = "|Total Loss: {:.4f}|Mel Loss: {:.4f}|Mel PostNet Loss: {:.4f}|Pitch Loss: {:.4f}|Energy Loss: {:.4f}|Duration Loss: {:.4f}|".format( *losses) # with open(os.path.join(train_log_path, "log.txt"), "a") as f: # f.write(message1 + message2 + "\n") outer_bar.write(message1 + message2) log(train_logger, step, losses=losses) if step % synth_step == 0: output_preidiction = model(*(batch[2:6])) fig, wav_reconstruction, wav_prediction, tag = synth_one_sample( batch, output_preidiction, vocoder, model_config, preprocess_config, ) log( train_logger, fig=fig, tag="Training/step_{}_{}".format(step, tag), ) sampling_rate = preprocess_config["preprocessing"][ "audio"]["sampling_rate"] log( train_logger, audio=wav_reconstruction, sampling_rate=sampling_rate, tag="Training/step_{}_{}_reconstructed".format( step, tag), ) log( train_logger, audio=wav_prediction, sampling_rate=sampling_rate, tag="Training/step_{}_{}_synthesized".format( step, tag), ) if step % val_step == 0: model.eval() message = evaluate(model, step, configs, val_logger, vocoder) # with open(os.path.join(val_log_path, "log.txt"), "a") as f: # f.write(message + "\n") outer_bar.write(message) model.train() if step % save_step == 0: torch.save( { "model": model.module.state_dict(), "optimizer": optimizer._optimizer.state_dict(), }, os.path.join( train_config["path"]["ckpt_path"], "{}.pth.tar".format(step), ), ) if step == total_step: quit() step += 1 outer_bar.update(1) inner_bar.update(1) epoch += 1
net.forward(patch, mask, training=True) # label通道1->9 mask = label2multichannel(mask.cpu(), class_num) mask = mask.to(device) elbo = net.elbo(mask) ### reg_loss = l2_regularisation(net.posterior) + l2_regularisation(net.prior) + l2_regularisation(net.fcomb.layers) loss = -elbo + 1e-5 * reg_loss losses += loss if step%10 == 0: print("-- [step {}] loss: {}".format(step, loss)) # evaluate(net, test_loader, device, test=True) # break optimizer.zero_grad() loss.backward() optimizer.step() # 评估 losses /= (step+1) print("Loss (Train): {}".format(losses)) # evaluate(net, train_eval_loader, device, test=False) evaluate(net, test_loader, device, test=True) except KeyboardInterrupt as e: print('KeyboardInterrupt: {}'.format(e)) except Exception as e: print('Exception: {}'.format(e)) finally: # 保存模型 print("saving the trained net model -- unet_{}.pt".format(class_num)) save_model(net, path='model/unet_{}.pt'.format(class_num))
def train(args, train_loader, test_loader, net, criterion, device): """ Args: args: parsed command line arguments. train_loader: an iterator over the training set. test_loader: an iterator over the test set. net: the neural network model employed. criterion: the loss function. device: using CPU or GPU. Outputs: All training losses, training accuracies, test losses, and test accuracies on each evaluation during training. """ optimizer = load_optim(params=net.parameters(), optim_method=args.optim_method, eta0=args.eta0, alpha=args.alpha, c=args.c, milestones=args.milestones, T_max=args.train_epochs * len(train_loader), n_batches_per_epoch=len(train_loader), nesterov=args.nesterov, momentum=args.momentum, weight_decay=args.weight_decay) if args.optim_method == 'SGD_ReduceLROnPlateau': scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=args.alpha, patience=args.patience, threshold=args.threshold) # Choose loss and metric function loss_function = metrics.get_metric_function('softmax_loss') all_train_losses = [] all_train_accuracies = [] all_test_losses = [] all_test_accuracies = [] for epoch in range(1, args.train_epochs + 1): net.train() for data in train_loader: inputs, labels = data inputs, labels = inputs.to(device), labels.to(device) optimizer.zero_grad() if args.optim_method.startswith('SLS'): closure = lambda: loss_function( net, inputs, labels, backwards=False) optimizer.step(closure) else: outputs = net(inputs) loss = criterion(outputs, labels) loss.backward() if 'Polyak' in args.optim_method: optimizer.step(loss.item()) else: optimizer.step() # Evaluate the model on training and validation dataset. if args.optim_method == 'SGD_ReduceLROnPlateau' or (epoch % args.eval_interval == 0): train_loss, train_accuracy = evaluate(train_loader, net, criterion, device) all_train_losses.append(train_loss) all_train_accuracies.append(train_accuracy) test_loss, test_accuracy = evaluate(test_loader, net, criterion, device) all_test_losses.append(test_loss) all_test_accuracies.append(test_accuracy) print('Epoch %d --- ' % (epoch), 'train: loss - %g, ' % (train_loss), 'accuracy - %g; ' % (train_accuracy), 'test: loss - %g, ' % (test_loss), 'accuracy - %g' % (test_accuracy)) if args.optim_method == 'SGD_ReduceLROnPlateau': scheduler.step(test_loss) return (all_train_losses, all_train_accuracies, all_test_losses, all_test_accuracies)
def train_and_evaluate(model, meta_train_classes, meta_val_classes, meta_test_classes, task_type, optimizer, scheduler, loss_fn, metrics, params, model_dir, restore_file=None): """ Train the model and evaluate every `save_summary_steps`. Args: model: TPN model meta_train_classes: (list) the classes for meta-training meta_val_classes: (list) the classes for meta-validating meta_test_classes: (list) the classes for meta-testing task_type: (subclass of FewShotTask) a type for generating tasks optimizer: (torch.optim) optimizer for parameters of model scheduler: (torch.optim.lr_scheduler) scheduler for decaying learning rate loss_fn: a loss function metrics: (dict) a dictionary of functions that compute a metric using the output and labels of each batch params: (Params) hyperparameters model_dir: (string) directory containing config, weights and log restore_file: (string) optional- name of file to restore from (without its extension .pth.tar) """ # reload weights from restore_file if specified if restore_file is not None: restore_path = os.path.join(args.model_dir, args.restore_file + '.pth.tar') logging.info("Restoring parameters from {}".format(restore_path)) utils.load_checkpoint(restore_path, model, optimizer) # params information num_classes = params.num_classes num_samples = params.num_samples num_query = params.num_query # validation accuracy best_val_loss = float('inf') # For plotting to see summerized training procedure plot_history = { 'train_loss': [], 'train_acc': [], 'val_loss': [], 'val_acc': [], 'test_loss': [], 'test_acc': [] } with tqdm(total=params.num_episodes) as t: for episode in range(params.num_episodes): # Run one episode logging.info("Episode {}/{}".format(episode + 1, params.num_episodes)) scheduler.step() # Train a model on a single task (episode). # TODO meta-batch of tasks task = task_type(meta_train_classes, num_classes, num_samples, num_query) dataloaders = fetch_dataloaders(['train', 'test'], task) _ = train_single_task(model, optimizer, loss_fn, dataloaders, metrics, params) # print(episode, _) # Evaluate on train, val, test dataset given a number of tasks (params.num_steps) if (episode + 1) % params.save_summary_steps == 0: train_metrics = evaluate(model, loss_fn, meta_train_classes, task_type, metrics, params, 'train') val_metrics = evaluate(model, loss_fn, meta_val_classes, task_type, metrics, params, 'val') test_metrics = evaluate(model, loss_fn, meta_test_classes, task_type, metrics, params, 'test') train_loss = train_metrics['loss'] val_loss = val_metrics['loss'] test_loss = test_metrics['loss'] train_acc = train_metrics['accuracy'] val_acc = val_metrics['accuracy'] test_acc = test_metrics['accuracy'] is_best = val_loss <= best_val_loss # Save weights utils.save_checkpoint({ 'episode': episode + 1, 'state_dict': model.state_dict(), 'optim_dict': optimizer.state_dict() }, is_best=is_best, checkpoint=model_dir) # If best_test, best_save_path if is_best: logging.info("- Found new best accuracy") best_val_loss = val_loss # Save best test metrics in a json file in the model directory best_train_json_path = os.path.join( model_dir, "metrics_train_best_weights.json") utils.save_dict_to_json(train_metrics, best_train_json_path) best_val_json_path = os.path.join( model_dir, "metrics_val_best_weights.json") utils.save_dict_to_json(val_metrics, best_val_json_path) best_test_json_path = os.path.join( model_dir, "metrics_test_best_weights.json") utils.save_dict_to_json(test_metrics, best_test_json_path) # Save latest test metrics in a json file in the model directory last_train_json_path = os.path.join( model_dir, "metrics_train_last_weights.json") utils.save_dict_to_json(train_metrics, last_train_json_path) last_val_json_path = os.path.join( model_dir, "metrics_val_last_weights.json") utils.save_dict_to_json(val_metrics, last_val_json_path) last_test_json_path = os.path.join( model_dir, "metrics_test_last_weights.json") utils.save_dict_to_json(test_metrics, last_test_json_path) plot_history['train_loss'].append(train_loss) plot_history['train_acc'].append(train_acc) plot_history['val_loss'].append(val_loss) plot_history['val_acc'].append(val_acc) plot_history['test_loss'].append(test_loss) plot_history['test_acc'].append(test_acc) utils.plot_training_results(args.model_dir, plot_history) t.set_postfix( tr_acc='{:05.3f}'.format(train_acc), te_acc='{:05.3f}'.format(test_acc), tr_loss='{:05.3f}'.format(train_loss), te_loss='{:05.3f}'.format(test_loss)) print('\n') t.update()
def train(): writer = SummaryWriter( log_dir= f"./runs/{model_name}/{datetime.datetime.now().replace(microsecond=0).isoformat()}{'-' + os.environ['REMARK'] if 'REMARK' in os.environ else ''}" ) if not os.path.exists('checkpoint'): os.makedirs('checkpoint') try: pretrained_word_embedding = torch.from_numpy( np.load('./data/train/pretrained_word_embedding.npy')).float() except FileNotFoundError: pretrained_word_embedding = None if model_name == 'DKN': try: pretrained_entity_embedding = torch.from_numpy( np.load( './data/train/pretrained_entity_embedding.npy')).float() except FileNotFoundError: pretrained_entity_embedding = None try: pretrained_context_embedding = torch.from_numpy( np.load( './data/train/pretrained_context_embedding.npy')).float() except FileNotFoundError: pretrained_context_embedding = None model = Model(config, pretrained_word_embedding, pretrained_entity_embedding, pretrained_context_embedding).to(device) else: model = Model(config, pretrained_word_embedding).to(device) print(model) dataset = BaseDataset('./data/train/behaviors_parsed.tsv', './data/train/news_parsed.tsv', './data/train/roberta') print(f"Load training dataset with size {len(dataset)}.") ############################################### ''' dataloader = DataLoader(dataset, batch_size=config.batch_size, shuffle=True, num_workers=config.num_workers, drop_last=True, pin_memory=True)''' ############################################### # In the step we need to tranform the dataset in federated manner ''' federated_train_loader = sy.FederatedDataLoader(datasets.MNIST( '../data', train=True, download=True, transform=transforms.Compose( [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] ) ) federated_train_loader = sy.FederatedDataLoader( # <-- this is now a FederatedDataLoader dataset.federate((bob, alice)), # <-- NEW: we distribute the dataset across all the workers, it's now a FederatedDataset batch_size=args.batch_size, shuffle=True, **kwargs) dataloader = iter(sy.FederatedDataLoader(dataset.federate((bob, alice)), batch_size=config.batch_size, shuffle=True, #num_workers=config.num_workers, drop_last=True, #pin_memory=True )) ''' #print(dataset) dataloader = sy.FederatedDataLoader(dataset.federate((bob, alice)), batch_size=config.batch_size, shuffle=True, num_workers=config.num_workers, drop_last=True, pin_memory=True) ############################################### print(f"The training dataset has been loaded!") criterion = nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), lr=config.learning_rate) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.95, last_epoch=-1) start_time = time.time() loss_full = [] exhaustion_count = 0 step = 0 early_stopping = EarlyStopping() checkpoint_dir = os.path.join('./checkpoint', model_name) Path(checkpoint_dir).mkdir(parents=True, exist_ok=True) checkpoint_path = latest_checkpoint(checkpoint_dir) ''' if checkpoint_path is not None: print(f"Load saved parameters in {checkpoint_path}") checkpoint = torch.load(checkpoint_path) early_stopping(checkpoint['early_stop_value']) step = checkpoint['step'] model.load_state_dict(checkpoint['model_state_dict']) optimizer.load_state_dict(checkpoint['optimizer_state_dict']) model.train() ''' #for i in tqdm(range(1,config.num_epochs * len(dataset) // config.batch_size + 1),desc="Training"): for i, (minibatch, target) in enumerate(dataloader): ##### Get a mini batch of data from federated dataset #minibatch ,_ = next(dataloader) #print(minibatch) #print(minibatch.size()) #exit() #minibatch = next(dataloader) step += 1 if model_name == 'LSTUR': y_pred = model(minibatch["user"], minibatch["clicked_news_length"], minibatch["candidate_news"], minibatch["clicked_news"]) elif model_name == 'HiFiArk': y_pred, regularizer_loss = model(minibatch["candidate_news"], minibatch["clicked_news"]) elif model_name == 'TANR': y_pred, topic_classification_loss = model( minibatch["candidate_news"], minibatch["clicked_news"]) else: ################################################# # Send the model model.send(minibatch.location) minibatch, target = minibatch.to(device), target.to(device) #minibatch = minibatch.to(device) ################################################# y_pred = model(minibatch) #y = torch.zeros(config.batch_size).long().to(device) #print(y_pred.get().size()) #print(y.size()) loss = criterion(y_pred, target) if model_name == 'HiFiArk': if i % 10 == 0: writer.add_scalar('Train/BaseLoss', loss.get(), step) writer.add_scalar('Train/RegularizerLoss', regularizer_loss.get(), step) writer.add_scalar('Train/RegularizerBaseRatio', regularizer_loss.get() / loss.get(), step) loss += config.regularizer_loss_weight * regularizer_loss elif model_name == 'TANR': if i % 10 == 0: writer.add_scalar('Train/BaseLoss', loss.item(), step) writer.add_scalar('Train/TopicClassificationLoss', topic_classification_loss.item(), step) writer.add_scalar( 'Train/TopicBaseRatio', topic_classification_loss.item() / loss.item(), step) loss += config.topic_classification_loss_weight * topic_classification_loss optimizer.zero_grad() loss.backward() optimizer.step() scheduler.step() model.get() loss = loss.get().detach().cpu().item() loss_full.append(loss) if i % 10 == 0: writer.add_scalar('Train/Loss', loss, step) if i % config.num_batches_show_loss == 0: #print(loss_full) #print(type(loss_full)) tqdm.write( f"Time {time_since(start_time)}, batches {i}, current loss {loss:.4f}, average loss: {np.mean(loss_full):.4f}, latest average loss: {np.mean(loss_full[-256:]):.4f}" ) if i % config.num_batches_validate == 0: (model if model_name != 'Exp1' else models[0]).eval() val_auc, val_mrr, val_ndcg5, val_ndcg10 = evaluate( model if model_name != 'Exp1' else models[0], './data/val', 200000) (model if model_name != 'Exp1' else models[0]).train() writer.add_scalar('Validation/AUC', val_auc, step) writer.add_scalar('Validation/MRR', val_mrr, step) writer.add_scalar('Validation/nDCG@5', val_ndcg5, step) writer.add_scalar('Validation/nDCG@10', val_ndcg10, step) tqdm.write( f"Time {time_since(start_time)}, batches {i}, validation AUC: {val_auc:.4f}, validation MRR: {val_mrr:.4f}, validation nDCG@5: {val_ndcg5:.4f}, validation nDCG@10: {val_ndcg10:.4f}, " ) early_stop, get_better = early_stopping(-val_auc) if early_stop: tqdm.write('Early stop.') break elif get_better: try: torch.save( { 'model_state_dict': (model if model_name != 'Exp1' else models[0]).state_dict(), 'optimizer_state_dict': (optimizer if model_name != 'Exp1' else optimizefrs[0]).state_dict(), 'step': step, 'early_stop_value': -val_auc }, f"./checkpoint/{model_name}/ckpt-{step}.pth") except OSError as error: print(f"OS error: {error}")
def main(argv): if len(argv) > 1: raise app.UsageError("Too many command-line arguments.") tf.random.set_seed(FLAGS.seed) if FLAGS.loudness_traindata_proto_file_pattern is None: raise app.UsageError( "Must provide --loudness_data_proto_file_pattern.") log_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), FLAGS.logs_dir) if not os.path.exists(log_dir): os.mkdir(log_dir) logging.info("TensorFlow seed: %d", FLAGS.seed) input_shape = None if FLAGS.mode == "test": raise NotImplementedError("Did not implement mode test.") data = get_datasets(FLAGS.loudness_testdata_proto_file_pattern, 1, carfac=FLAGS.use_carfac) logging.info("Created testing datasets") model = tf.keras.models.load_model(FLAGS.load_model_from_file) logging.info("Loaded model") elif FLAGS.mode == "train": data = get_datasets(FLAGS.loudness_traindata_proto_file_pattern, FLAGS.batch_size, carfac=FLAGS.use_carfac, extra_file_pattern=FLAGS. extra_loudness_traindata_proto_file_pattern) frequency_bins = None for example in data["train"].take(1): input_example, target_example = example input_shape = input_example.shape carfac_channels = input_example.shape[1] frequency_bins = input_example.shape[2] logging.info("Created model") elif FLAGS.mode == "eval_once": data = get_testdata(FLAGS.loudness_testdata_proto_file_pattern, carfac=FLAGS.use_carfac) frequency_bins = None for example in data["test"].take(1): input_example, target_example, _ = example input_shape = input_example.shape carfac_channels = input_example.shape[1] frequency_bins = input_example.shape[2] model = LoudnessPredictor( frequency_bins=frequency_bins, carfac_channels=carfac_channels, num_rows_channel_kernel=FLAGS.num_rows_channel_kernel, num_cols_channel_kernel=FLAGS.num_cols_channel_kernel, num_filters_channels=FLAGS.num_filters_channels, num_rows_bin_kernel=FLAGS.num_rows_bin_kernel, num_cols_bin_kernel=FLAGS.num_cols_bin_kernel, num_filters_bins=FLAGS.num_filters_bins, dropout_p=FLAGS.dropout_p, use_channels=FLAGS.use_carfac, seed=FLAGS.seed) if FLAGS.load_from_checkpoint: path_to_load = os.path.join(log_dir, FLAGS.load_from_checkpoint) logging.info("Attempting to load model from %s", path_to_load) loaded = False try: model.load_weights(path_to_load) loaded = True logging.info("Loaded model") except Exception as err: logging.info( "Unable to load log dir checkpoint %s, trying " "'load_from_checkpoint' flag: %s", path_to_load, err) path_to_load = FLAGS.load_from_checkpoint try: model.load_weights(path_to_load) loaded = True except Exception as err: logging.info("Unable to load flag checkpoint %s: %s", path_to_load, err) else: loaded = False example_image_batch = [] if FLAGS.mode == "train": data_key = "train" for example in data[data_key].take(4): input_example, target = example input_shape = input_example.shape tf.print("(start train) input shape: ", input_shape) tf.print("(start train) target phons shape: ", target.shape) input_example = tf.expand_dims(input_example[0], axis=0) example_image_batch.append([input_example, target]) elif FLAGS.mode == "eval_once": data_key = "test" for example in data[data_key].take(4): input_example, target, _ = example input_shape = input_example.shape tf.print("(start eval) input shape: ", input_shape) tf.print("(start eval) target phons shape: ", target.shape) input_example = tf.expand_dims(input_example[0], axis=0) example_image_batch.append([input_example, target]) callbacks = [helpers.StepIncrementingCallback()] callbacks.append( tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1, update_freq="batch", write_graph=True)) model.build(input_shape) logging.info("Model summary") model.summary() if FLAGS.extra_loudness_traindata_proto_file_pattern: extra_data = True else: extra_data = False save_ckpt = log_dir + "/cp_carfac{}_extradata{}".format( FLAGS.use_carfac, extra_data) + "_{epoch:04d}.ckpt" logging.info("Save checkpoint to: %s" % save_ckpt) callbacks.append( tf.keras.callbacks.ModelCheckpoint(filepath=save_ckpt, save_weights_only=True, verbose=1, period=5)) if FLAGS.mode == "train": logging.info("Starting training for %d epochs" % FLAGS.epochs) if FLAGS.extra_loudness_traindata_proto_file_pattern: steps_per_epoch = (317 + 639) // FLAGS.batch_size else: steps_per_epoch = 317 // FLAGS.batch_size train(model, data["train"], data["validate"], FLAGS.learning_rate, FLAGS.epochs, steps_per_epoch, callbacks) elif FLAGS.mode == "test": raise NotImplementedError("Mode test not implemented.") evaluate(model, data["test"], batch_size=FLAGS.eval_batch_size) elif FLAGS.mode == "eval_once": if not loaded: raise ValueError( "Trying to eval. a model with unitialized weights.") save_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), log_dir) write_predictions(model, data["test"], batch_size=1, save_directory=save_dir, save_file=FLAGS.save_predictions_file) return else: raise ValueError("Specified value for '--mode' (%s) unknown", FLAGS.mode)
def train_and_evaluate(model, data_loader, optimizer, loss_fn, metrics, params, model_dir, restore_file=None): """Train the model and evaluate every epoch. Args: model: (torch.nn.Module) the neural network train_data: (dict) training data with keys 'data' and 'labels' val_data: (dict) validaion data with keys 'data' and 'labels' optimizer: (torch.optim) optimizer for parameters of model loss_fn: a function that takes batch_output and batch_labels and computes the loss for the batch metrics: (dict) a dictionary of functions that compute a metric using the output and labels of each batch params: (Params) hyperparameters model_dir: (string) directory containing config, weights and log restore_file: (string) optional- name of file to restore from (without its extension .pth.tar) """ ema = utils.EMA(model, params.ema_decay) # reload weights from restore_file if specified if restore_file is not None: restore_path = os.path.join(args.model_dir, args.restore_file + '.pth.tar') logging.info("Restoring parameters from {}".format(restore_path)) utils.load_checkpoint(restore_path, model, optimizer) best_val_acc = 0.0 for epoch in range(params.num_epochs): # Run one epoch logging.info("Epoch {}/{}".format(epoch + 1, params.num_epochs)) # compute number of batches in one epoch (one full pass over the training set) num_steps = (params.train_size + 1) // params.batch_size train_data_iterator = data_loader.data_iterator( split='train', batch_size=params.batch_size) train(model, optimizer, loss_fn, train_data_iterator, metrics, params, num_steps, ema) # Evaluate for one epoch on validation set ema.assign(model) num_steps = (params.val_size + 1) // params.batch_size val_data_iterator = data_loader.data_iterator( split='val', batch_size=params.batch_size) val_metrics = evaluate(model, loss_fn, val_data_iterator, metrics, params, num_steps) val_acc = val_metrics['f1'] is_best = val_acc >= best_val_acc # Save weights utils.save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'optim_dict': optimizer.state_dict() }, is_best=is_best, checkpoint=model_dir) # If best_eval, best_save_path if is_best: logging.info("- Found new best accuracy") best_val_acc = val_acc # Save best val metrics in a json file in the model directory best_json_path = os.path.join(model_dir, "metrics_val_best_weights.json") utils.save_dict_to_json(val_metrics, best_json_path) # Save latest val metrics in a json file in the model directory last_json_path = os.path.join(model_dir, "metrics_val_last_weights.json") utils.save_dict_to_json(val_metrics, last_json_path) ema.resume(model)
def train_and_evaluate(model, params, dataloader, optimizer, scheduler, loss_fn, metrics, model_dir, log_dir, threshold, cuda_present): """Train the model and evaluate every epoch. Args: model: (torch.nn.Module) the neural network train_dataloader: (DataLoader) a torch.utils.data.DataLoader object that fetches training data val_dataloader: (DataLoader) a torch.utils.data.DataLoader object that fetches validation data optimizer: (torch.optim) optimizer for parameters of model loss_fn: a function that takes batch_output and batch_labels and computes the loss for the batch metrics: (dict) a dictionary of functions that compute a metric using the output and labels of each batch params: (Params) hyperparameters model_dir: (string) directory containing config, weights and log restore_file: (string) optional- name of file to restore from (without its extension .pth.tar) """ best_val_acc = 0 for epoch in range(params.num_epochs): scheduler.step() t0 = time.time() '''Do the following for every epoch''' # Run one epoch logging.info( "Epoch {}/{} , : learning rate = {} : threshold: {}".format( epoch + 1, params.num_epochs, scheduler.get_lr(), threshold)) #train_image_dict = dataloader.load_data("train", params) train_image_dict = dataloader.Create_Img_dict("train", params) train_labels_dict = dataloader.load_labels("train", params) train_img_count = len(train_image_dict) train_data_generator = dataloader.data_iterator( params, "train", train_image_dict, train_labels_dict) # compute number of batches in one epoch (one full pass over the training set) train(model, optimizer, loss_fn, train_data_generator, metrics, params, train_img_count, threshold, cuda_present) # Evaluate for one epoch on validation set val_image_dict = dataloader.load_data("val", params) val_labels_dict = dataloader.load_labels("val", params) val_img_count = len(val_image_dict) val_data_generator = dataloader.data_iterator(params, "val", val_image_dict, val_labels_dict) (val_metrics, threshold) = evaluate.evaluate(model, loss_fn, val_data_generator, params, val_img_count, threshold, cuda_present) val_acc = val_metrics['accuracy'] is_best = val_acc > best_val_acc chk_file_name = 'train_DN73_3ch_BBG_nodataaug_Adam__' + 'epoch_' + str( epoch) + '__' best_file_name = chk_file_name + '.pth.tar' lr = params.learning_rate # Save weights every 3rd epoch #if ((epoch+1) % 3 == 0): if (is_best): best_val_acc = val_acc utils.save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'optim_dict': optimizer.state_dict() }, is_best, log_dir, best_file_name) t1 = time.time() logging.info("Time taken for this epoch = {}".format(t1 - t0)) # Save weights in the end chk_file_name += '__' + 'lr_' + str(lr) + '__' + '.pth.tar' utils.save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'optim_dict': optimizer.state_dict() }, is_best, log_dir, chk_file_name)
for j in range(n_sample): rank_order_dist[i, j] = float('inf') for i in range(n_sample): print(i) for j in range(i + 1, n_sample): index_i = numpy.argwhere(nn_list[i] == j)[0][0] index_j = numpy.argwhere(nn_list[j] == i)[0][0] if index_i <= k or index_j <= k: distance_i = 0 distance_j = 0 for r in range(min(index_i, k) + 1): if numpy.argwhere(nn_list[j] == nn_list[i, r])[0][0] > k: distance_i += 1 for r in range(min(index_j, k) + 1): if numpy.argwhere(nn_list[i] == nn_list[j, r])[0][0] > k: distance_j += 1 # 对称化距离 rank_order_dist[i, j] = rank_order_dist[ j, i] = (distance_i + distance_j) / min(index_i, index_j) return rank_order_dist if __name__ == '__main__': start_time = time() result = cluster(0.1, 200) time_diff = time() - start_time print('time:{}'.format(time_diff)) evaluate(labels, result)
for epoch in range(num_epoch): print("epoch:", epoch) train_bpr_loss = [] train_loss = [] train_data = generate_train_data(train_dict, num_item, num_neg_sample) train_batch = generate_train_batch(train_data, batch_size) for batch in train_batch: bpr_loss, loss, _ = sess.run([model.bpr_loss, model.loss, model.train_op], feed_dict=get_feed_dict(model, batch)) train_bpr_loss.append(bpr_loss) train_loss.append(loss) train_bpr_loss = sum(train_bpr_loss) / len(train_data) train_loss = sum(train_loss) / len(train_data) print("train bpr loss:", train_bpr_loss, "train loss:", train_loss) r_hat = sess.run(model.r_hat_ui, feed_dict={model.u: validate_data[:, 0], model.i: validate_data[:, 1]}) rank_list = np.reshape(r_hat, [-1, 100]).argsort()[:, ::-1].tolist() validate_hr, validate_ndcg = evaluate(rank_list, 0, 10) print("validate hit ratio:", validate_hr, "validate ndcg:", validate_ndcg) result.append([epoch, train_loss, validate_hr, validate_ndcg]) r_hat = sess.run(model.r_hat_ui, feed_dict={model.u: test_data[:, 0], model.i: test_data[:, 1]}) rank_list = np.reshape(r_hat, [-1, 100]).argsort()[:, ::-1].tolist() test_hr, test_ndcg = evaluate(rank_list, 0, 10) print("test hit ratio:", test_hr, "test ndcg:", test_ndcg) print("over!") with open('BPR_{dataset}_{num_factor}_{reg_rate}.csv'.format(dataset=dataset, num_factor=num_factor, reg_rate=reg_rate), 'w', newline='') as f: writer = csv.writer(f) for line in result: writer.writerow(line)
import os from config import workspace_path from train import train from config import Model_Config from prepare import prepare from predict import predict from evaluate import evaluate print(os.getcwd()) os.chdir(workspace_path) os.mkdir('log') os.mkdir('hdf') print(os.getcwd()) print(os.listdir('./')) if __name__ == '__main__': # prepare X_train_processed, X_val_processed, Y_train_list, Y_val_list, vocabulary_size, label_distribute_dict_list = prepare( ) # train model, history = train(X_train_processed, X_val_processed, Y_train_list, Y_val_list, vocabulary_size, label_distribute_dict_list) # predict preds = predict(X_val_processed, Model_Config.model_saved_filepath) # evaluate validate_df, df_pcf = evaluate(preds) # print precision、recall、f1 值 print(df_pcf)