def main(_): config_file = os.path.join(FLAGS.output, 'config.json') log_file = os.path.join(FLAGS.output, 'model.log') config = load_config(config_file) config['init_checkpoint'] = FLAGS.init_checkpoint logger = get_logger(log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True map_file = os.path.join(FLAGS.output, 'maps.pkl') with open(map_file, "rb") as f: tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, os.path.join(FLAGS.output, 'checkpoint'), config, logger) text = "中国你好成都" result = model.evaluate_line(sess, input_from_line(text, FLAGS.max_seq_len, tag_to_id), id_to_tag, export=True) print(result)
def predict(): batcher = get_batcher() config = load_config(FLAGS.config_file) logger = get_logger(os.path.join('log', FLAGS.log_file)) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, tag_to_id, id_to_tag = pickle.load(f) def get_test_data(char2id): sentences = [] with open('data/test.txt', 'r', encoding='utf-8') as f: for line in f: words = line.strip().split('_') ids = [ char2id[char if char in char2id else '<UNK>'] for char in words ] sentences.append([words, ids]) return sentences test_data = get_test_data(char_to_id) with tf.Session(config=tf_config) as sess: elmo_model = load_elmo() model = create_model(sess, Model, FLAGS.ckpt_path, elmo_model, config, logger) results = model.predict_batch(sess, data=test_data, id_to_tag=id_to_tag, batcher=batcher, batch_size=FLAGS.batch_size) result_to_file(results)
def main(_): config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, config, logger) while True: line = input("input sentence, please:") result = model.evaluate_line(sess, input_from_line(line, FLAGS.max_seq_len, tag_to_id), id_to_tag) print(result['entities'])
def extract(event): tf.reset_default_graph() config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, config, logger) result = model.evaluate_line(sess, input_from_line(event, FLAGS.max_seq_len, tag_to_id), id_to_tag) return result['entities']
from utils import create_model, get_logger from model import Model from loader import input_from_line from train import FLAGS, load_config, train # tornado高并发 import tornado.web import tornado.gen import tornado.concurrent from concurrent.futures import ThreadPoolExecutor # 定义端口为12306 define("port", default=12306, help="run on the given port", type=int) # 导入模型 config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = False with open(FLAGS.map_file, "rb") as f: tag_to_id, id_to_tag = pickle.load(f) sess = tf.Session(config=tf_config) model = create_model(sess, Model, FLAGS.ckpt_path, config, logger) # 测试 class HelloHandler(tornado.web.RequestHandler): def get(self): self.write('Hello World!')
from torchvision.models import vgg16 from thop import profile from thop import clever_format from pytorch_image_classification.models import * from train import load_config config = load_config() model = create_model(config) input = torch.randn(1, 3, 224, 224) #flops, params = profile(model,inputs=(input, )) #print(flops, params) flops, params = clever_format([12508275712.0,1730714.0]) print(flops, params) #resnet110 12.508G 1.731M flops, params = clever_format([6215001088.0,855770.0], "%.3f") print(flops, params) #resnet56 6.215G 855.770K flops, params = clever_format([14527502336.0,769162.0], "%.3f") print(flops, params) #densenet 14.528G 769.162K flops, params = clever_format([2019484928.0,272474.0], "%.3f") print(flops, params) # resnet20 2.019G 272.474K flops, params = clever_format([15607495680.0,138357536.0], "%.3f") print(flops, params) #vgg16 15.607G 138.358M #resnet110 12508275712.0,1730714.0 #resnet56 6215001088.0,855770.0 #densenet 14527502336.0,769162.0 #resnet20 2019484928.0,272474.0 #vgg16 15607495680.0,138357536.0
) SAVE_DIR = pathlib.Path("result") # Preload data data_files = Data_Preload(data_folder=DATA_DIR, load_augment=False, verbose=False,) model_files = Data_Preload.get_file_list(str(MODEL_DIR), ".pb") for mdl_file in model_files: np.random.seed(1) mdl_name = mdl_file.parent.name print("*** Model {} ***".format(mdl_name)) print("*** Loading Config ***") conf = load_config(str(CONF_DIR / mdl_name / "config.yaml")) exclude = conf["data"]["x_validation_exclude"] settings = conf["data"]["data_settings"] settings["skip"] = 10 print(settings["data_headings"]) print(settings["label_mapping"]) # Load model print("*** Loading model ***") model = tf.keras.models.load_model(str(mdl_file.parent)) train_data, train_label = data_files.load_data( parse_settings=settings, filter_mode=True, filter_list=exclude,
parser.add_argument('--sz-embedding', default=128, type=int) parser.add_argument('--cuda-device', default = 0, type = int) parser.add_argument('--exp', default='0', type=str, help='experiment identifier') parser.add_argument('--dir', default='default', type=str) parser.add_argument('--backend', default='faiss', choices=('torch+sklearn', 'faiss', 'faiss-gpu') ) parser.add_argument('--random-seed', default = 0, type = int) parser.add_argument('--backbone-wd', default=1e-4, type=float) parser.add_argument('--backbone-lr', default=1e-5, type=float) parser.add_argument('--embedding-lr', default=1e-5, type=float) parser.add_argument('--embedding-wd', default=1e-4, type=float) parser.add_argument('--verbose', action = 'store_true') args = vars(parser.parse_args()) config = train.load_config(config_name = 'config.json') config['dataloader']['batch_size'] = args.pop('sz_batch') config['dataloader']['num_workers'] = args.pop('num_workers') config['recluster']['mod_epoch'] = args.pop('mod_epoch') config['opt']['backbone']['lr'] = args.pop('backbone_lr') config['opt']['backbone']['weight_decay'] = args.pop('backbone_wd') config['opt']['embedding']['lr'] = args.pop('embedding_lr') config['opt']['embedding']['weight_decay'] = args.pop('embedding_wd') for k in args: if k in config: config[k] = args[k] if config['nb_clusters'] == 1: config['recluster']['enabled'] = False
# ) no_tran_data_files = Data_Preload( data_folder=NO_TRAN_FOLDER, load_augment=False, verbose=False, ) # Available models models_files = Data_Preload.get_file_list(LOG_FOLDER + "model", ".pb") for mdl_file in models_files: mdl_name = mdl_file.parent.name # -------------------------------------------------------------------------------------------- # # Load data set print("*** Loading Config ***") conf = load_config(LOG_FOLDER + "conf/" + mdl_name + "/config.yaml") exclude = conf["data"]["x_validation_exclude"] settings = conf["data"]["data_settings"] settings["skip"] = 10 # if settings["num_labels"] == 6: # TRANSITION_MODEL = False # print("Ingoring non-transition models") # continue # -------------------------------------------------------------------------------------------- # # Load model print("*** Loading model {} ***".format(mdl_name)) model = tf.keras.models.load_model(str(mdl_file.parent))
parser.add_argument('--submission_filepath', type=str, default='data/submission.csv') parser.add_argument('--config', type=str) parser.add_argument('--max_depth', type=int, default=2) parser.add_argument('--eta', type=int, default=1) parser.add_argument('--silent', type=int, default=1) parser.add_argument('--objective', type=str, default='binary:logistic') parser.add_argument('--nthread', type=int, default=4) parser.add_argument('--num_round', type=int, default=200) parser.add_argument('--columns', type=str, nargs='+', default=None) if __name__ == '__main__': args = parser.parse_args() config = load_config(args) train_filepaths = [f'data/cross-validation/cv{i}.csv' for i in range(10)] train_X, train_y = load_train_val_data(train_filepaths) test_X = pd.read_csv('data/test.csv') train_selected_X = select_columns(train_X) test_selected_X = select_columns(test_X) onehot_encoder, dummified_columns = build_onehot_encoder(train_selected_X) train_dummified_X = dummify_columns(train_selected_X, onehot_encoder, dummified_columns) test_dummified_X = dummify_columns(test_selected_X, onehot_encoder, dummified_columns)
def main(): config = load_config("config/gtsrb.json") imgsize = (config["size"], config["size"]) num_classes = config["num_classes"] model = L.Classifier(network(config["fc_size"], num_classes)) serializers.load_npz( "/mnt/sakuradata2/calland/scratch/gtsrb/oversampled/model_epoch_20", model) model.predictor.train = False model.to_gpu() val_annotations = config["test_annotation"] validation_data = FlexibleImageDataset(val_annotations, mean=config["validation_mean"], size=imgsize, normalize=True) validation_data.summary() #img = load_image("/mnt/sakuradata2/datasets/GTSRB/Final_Test/Images/00001.ppm.png", imgsize) #print "predicting image " #label = predict_label(img, model) #print label #exit() lbl_pred = [] lbl_true = [] correct = [] class_names = [str(i) for i in range(num_classes)] for i in range(len(validation_data._pairs)): if i % 100 == 0: print "{} / {}".format(i, len(validation_data._pairs)) data = validation_data.get_example(i) arr = cuda.cupy.array([data[0]]) lbl = numpy.array([data[1]]) ret = model.predictor(arr) ret = F.softmax(ret) ret.to_cpu() true_label = lbl.tolist()[0] recon_label = numpy.argmax(ret.data) #print true_label, recon_label lbl_true.append(true_label) lbl_pred.append(recon_label) if true_label == recon_label: correct.append(True) else: correct.append(False) # calculate accuracy num_correct = sum(bool(x) for x in correct) accuracy = 100.0 * (num_correct / float(len(correct))) print "Accuracy: {}%".format(accuracy) # Compute confusion matrix cnf_matrix = confusion_matrix(lbl_true, lbl_pred) numpy.set_printoptions(precision=2) # Plot normalized confusion matrix plt.figure() plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True, title='Normalized confusion matrix') plt.show()
# model_folder = "20201103-142223" # W vs (SA / SD) - 91.05% # model_folder = "20201103-143503" # SD vs (W / SD) - 89.39% model_folder = "20201103-145933" # W vs SA vs SD - 59.38% MODEL_FOLDER = pathlib.Path(model_folder) MODEL_DIR = pathlib.Path("logs\\model") CONF_DIR = pathlib.Path("logs\\conf") SAVE_DIR = pathlib.Path("result") # TEST_PARTICIPANT = ["Participant_04"] # Part of training (Seen) # TEST_PARTICIPANT = ["Participant_07"] # TEST_PARTICIPANT = ["Participant_03"] # Excluded participant (Unseen) TEST_PARTICIPANT = ["Participant_16"] conf = load_config(config_file=str(CONF_DIR / MODEL_FOLDER / "config.yaml")) conf["data"]["data_settings"]["skip"] = 0 # Don't skip anything # Load tensorflow model print("Loading model") model = tf.keras.models.load_model(str(MODEL_DIR / MODEL_FOLDER)) print("Model loaded") model.summary() # Load Data data_files = Data_Preload(data_folder=conf["data"]["folder"], load_augment=False) if SAVE_CLASS_PREDICTIONS: prediction = [] mappings = {0: 0, 3: 0, 4: 0}
# THIS FILE IS FOR HYPERPARAMETER SEARCH WITH Weights&Biases # IT SHOULDN't BE EXECUTED DIRECTLY, IT SHOULD BE RUN BY WANDB SWEEP AGENT! import wandb import train # Choose which run config to optimize RUN_CONFIG_NAME = "MNIST_CLASSIFIER_V1" # Load configs project_config = train.load_config("project_config.yaml") run_config = train.load_config("run_configs.yaml")[RUN_CONFIG_NAME] # Initialize wandb wandb.init() # Replace run config hyperparameters with the ones loaded from wandb sweep server sweep_hparams = wandb.Config._as_dict(wandb.config) for key, value in sweep_hparams.items(): if key == "_wandb": continue elif key in run_config["trainer"].keys(): run_config["trainer"][key] = value elif key in run_config["model"].keys(): run_config["model"][key] = value elif key in run_config["dataset"].keys(): run_config["dataset"][key] = value # Execute run train.train(project_config=project_config, run_config=run_config,
self.ou_a = ou_a self.ou_mu = ou_mu print(self.__dict__) def brownian_motion_log_returns(self): sqrt_delta_sigma = np.sqrt(self.delta) * self.sigma return np.random.normal(loc=0, scale=sqrt_delta_sigma, size=None) def ornstein_uhlenbeck_level(self, prev_ou_level): drift = self.ou_a * (self.ou_mu - prev_ou_level) * self.delta randomness = self.brownian_motion_log_returns() return prev_ou_level + drift + randomness if __name__ == "__main__": from train import load_config config = load_config("config_g2g.yaml") noise_params = {k: np.array(list(map(float, v.split(",")))) for k, v in config["noise_params"].items()} noise = Noise(**noise_params) ou_lvl = np.zeros(2) v = [] for i in range(1000000): # print(ou_lvl) ou_lvl = noise.ornstein_uhlenbeck_level(ou_lvl) v.append(ou_lvl) if (i+1) % 50 == 0: ou_lvl = np.zeros(2) import matplotlib.pyplot as plt l, a = list(zip(*v)) plt.hist(a, np.linspace(-1.5, 1.5, 300))