def main(_):
config_file = os.path.join(FLAGS.output, 'config.json')
log_file = os.path.join(FLAGS.output, 'model.log')
config = load_config(config_file)
config['init_checkpoint'] = FLAGS.init_checkpoint
logger = get_logger(log_file)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
map_file = os.path.join(FLAGS.output, 'maps.pkl')
with open(map_file, "rb") as f:
tag_to_id, id_to_tag = pickle.load(f)
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model,
os.path.join(FLAGS.output, 'checkpoint'), config,
logger)
text = "中国你好成都"
result = model.evaluate_line(sess,
input_from_line(text, FLAGS.max_seq_len,
tag_to_id),
id_to_tag,
export=True)
print(result)
def predict():
batcher = get_batcher()
config = load_config(FLAGS.config_file)
logger = get_logger(os.path.join('log', FLAGS.log_file))
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
with open(FLAGS.map_file, "rb") as f:
char_to_id, tag_to_id, id_to_tag = pickle.load(f)
def get_test_data(char2id):
sentences = []
with open('data/test.txt', 'r', encoding='utf-8') as f:
for line in f:
words = line.strip().split('_')
ids = [
char2id[char if char in char2id else '<UNK>']
for char in words
]
sentences.append([words, ids])
return sentences
test_data = get_test_data(char_to_id)
with tf.Session(config=tf_config) as sess:
elmo_model = load_elmo()
model = create_model(sess, Model, FLAGS.ckpt_path, elmo_model, config,
logger)
results = model.predict_batch(sess,
data=test_data,
id_to_tag=id_to_tag,
batcher=batcher,
batch_size=FLAGS.batch_size)
result_to_file(results)
def main(_):
config = load_config(FLAGS.config_file)
logger = get_logger(FLAGS.log_file)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
with open(FLAGS.map_file, "rb") as f:
tag_to_id, id_to_tag = pickle.load(f)
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, config, logger)
while True:
line = input("input sentence, please:")
result = model.evaluate_line(sess, input_from_line(line, FLAGS.max_seq_len, tag_to_id), id_to_tag)
print(result['entities'])
def extract(event):
tf.reset_default_graph()
config = load_config(FLAGS.config_file)
logger = get_logger(FLAGS.log_file)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
with open(FLAGS.map_file, "rb") as f:
tag_to_id, id_to_tag = pickle.load(f)
with tf.Session(config=tf_config) as sess:
model = create_model(sess, Model, FLAGS.ckpt_path, config, logger)
result = model.evaluate_line(sess, input_from_line(event, FLAGS.max_seq_len, tag_to_id), id_to_tag)
return result['entities']
from utils import create_model, get_logger
from model import Model
from loader import input_from_line
from train import FLAGS, load_config, train
# tornado高并发
import tornado.web
import tornado.gen
import tornado.concurrent
from concurrent.futures import ThreadPoolExecutor
# 定义端口为12306
define("port", default=12306, help="run on the given port", type=int)
# 导入模型
config = load_config(FLAGS.config_file)
logger = get_logger(FLAGS.log_file)
# limit GPU memory
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = False
with open(FLAGS.map_file, "rb") as f:
tag_to_id, id_to_tag = pickle.load(f)
sess = tf.Session(config=tf_config)
model = create_model(sess, Model, FLAGS.ckpt_path, config, logger)
# 测试
class HelloHandler(tornado.web.RequestHandler):
def get(self):
self.write('Hello World!')
from torchvision.models import vgg16 from thop import profile from thop import clever_format from pytorch_image_classification.models import * from train import load_config config = load_config() model = create_model(config) input = torch.randn(1, 3, 224, 224) #flops, params = profile(model,inputs=(input, )) #print(flops, params) flops, params = clever_format([12508275712.0,1730714.0]) print(flops, params) #resnet110 12.508G 1.731M flops, params = clever_format([6215001088.0,855770.0], "%.3f") print(flops, params) #resnet56 6.215G 855.770K flops, params = clever_format([14527502336.0,769162.0], "%.3f") print(flops, params) #densenet 14.528G 769.162K flops, params = clever_format([2019484928.0,272474.0], "%.3f") print(flops, params) # resnet20 2.019G 272.474K flops, params = clever_format([15607495680.0,138357536.0], "%.3f") print(flops, params) #vgg16 15.607G 138.358M #resnet110 12508275712.0,1730714.0 #resnet56 6215001088.0,855770.0 #densenet 14527502336.0,769162.0 #resnet20 2019484928.0,272474.0 #vgg16 15607495680.0,138357536.0
)
SAVE_DIR = pathlib.Path("result")
# Preload data
data_files = Data_Preload(data_folder=DATA_DIR, load_augment=False, verbose=False,)
model_files = Data_Preload.get_file_list(str(MODEL_DIR), ".pb")
for mdl_file in model_files:
np.random.seed(1)
mdl_name = mdl_file.parent.name
print("*** Model {} ***".format(mdl_name))
print("*** Loading Config ***")
conf = load_config(str(CONF_DIR / mdl_name / "config.yaml"))
exclude = conf["data"]["x_validation_exclude"]
settings = conf["data"]["data_settings"]
settings["skip"] = 10
print(settings["data_headings"])
print(settings["label_mapping"])
# Load model
print("*** Loading model ***")
model = tf.keras.models.load_model(str(mdl_file.parent))
train_data, train_label = data_files.load_data(
parse_settings=settings,
filter_mode=True,
filter_list=exclude,
parser.add_argument('--sz-embedding', default=128, type=int)
parser.add_argument('--cuda-device', default = 0, type = int)
parser.add_argument('--exp', default='0', type=str, help='experiment identifier')
parser.add_argument('--dir', default='default', type=str)
parser.add_argument('--backend', default='faiss',
choices=('torch+sklearn', 'faiss', 'faiss-gpu')
)
parser.add_argument('--random-seed', default = 0, type = int)
parser.add_argument('--backbone-wd', default=1e-4, type=float)
parser.add_argument('--backbone-lr', default=1e-5, type=float)
parser.add_argument('--embedding-lr', default=1e-5, type=float)
parser.add_argument('--embedding-wd', default=1e-4, type=float)
parser.add_argument('--verbose', action = 'store_true')
args = vars(parser.parse_args())
config = train.load_config(config_name = 'config.json')
config['dataloader']['batch_size'] = args.pop('sz_batch')
config['dataloader']['num_workers'] = args.pop('num_workers')
config['recluster']['mod_epoch'] = args.pop('mod_epoch')
config['opt']['backbone']['lr'] = args.pop('backbone_lr')
config['opt']['backbone']['weight_decay'] = args.pop('backbone_wd')
config['opt']['embedding']['lr'] = args.pop('embedding_lr')
config['opt']['embedding']['weight_decay'] = args.pop('embedding_wd')
for k in args:
if k in config:
config[k] = args[k]
if config['nb_clusters'] == 1:
config['recluster']['enabled'] = False
# )
no_tran_data_files = Data_Preload(
data_folder=NO_TRAN_FOLDER,
load_augment=False,
verbose=False,
)
# Available models
models_files = Data_Preload.get_file_list(LOG_FOLDER + "model", ".pb")
for mdl_file in models_files:
mdl_name = mdl_file.parent.name
# -------------------------------------------------------------------------------------------- #
# Load data set
print("*** Loading Config ***")
conf = load_config(LOG_FOLDER + "conf/" + mdl_name + "/config.yaml")
exclude = conf["data"]["x_validation_exclude"]
settings = conf["data"]["data_settings"]
settings["skip"] = 10
# if settings["num_labels"] == 6:
# TRANSITION_MODEL = False
# print("Ingoring non-transition models")
# continue
# -------------------------------------------------------------------------------------------- #
# Load model
print("*** Loading model {} ***".format(mdl_name))
model = tf.keras.models.load_model(str(mdl_file.parent))
parser.add_argument('--submission_filepath',
type=str,
default='data/submission.csv')
parser.add_argument('--config', type=str)
parser.add_argument('--max_depth', type=int, default=2)
parser.add_argument('--eta', type=int, default=1)
parser.add_argument('--silent', type=int, default=1)
parser.add_argument('--objective', type=str, default='binary:logistic')
parser.add_argument('--nthread', type=int, default=4)
parser.add_argument('--num_round', type=int, default=200)
parser.add_argument('--columns', type=str, nargs='+', default=None)
if __name__ == '__main__':
args = parser.parse_args()
config = load_config(args)
train_filepaths = [f'data/cross-validation/cv{i}.csv' for i in range(10)]
train_X, train_y = load_train_val_data(train_filepaths)
test_X = pd.read_csv('data/test.csv')
train_selected_X = select_columns(train_X)
test_selected_X = select_columns(test_X)
onehot_encoder, dummified_columns = build_onehot_encoder(train_selected_X)
train_dummified_X = dummify_columns(train_selected_X, onehot_encoder,
dummified_columns)
test_dummified_X = dummify_columns(test_selected_X, onehot_encoder,
dummified_columns)
def main():
config = load_config("config/gtsrb.json")
imgsize = (config["size"], config["size"])
num_classes = config["num_classes"]
model = L.Classifier(network(config["fc_size"], num_classes))
serializers.load_npz(
"/mnt/sakuradata2/calland/scratch/gtsrb/oversampled/model_epoch_20",
model)
model.predictor.train = False
model.to_gpu()
val_annotations = config["test_annotation"]
validation_data = FlexibleImageDataset(val_annotations,
mean=config["validation_mean"],
size=imgsize,
normalize=True)
validation_data.summary()
#img = load_image("/mnt/sakuradata2/datasets/GTSRB/Final_Test/Images/00001.ppm.png", imgsize)
#print "predicting image "
#label = predict_label(img, model)
#print label
#exit()
lbl_pred = []
lbl_true = []
correct = []
class_names = [str(i) for i in range(num_classes)]
for i in range(len(validation_data._pairs)):
if i % 100 == 0:
print "{} / {}".format(i, len(validation_data._pairs))
data = validation_data.get_example(i)
arr = cuda.cupy.array([data[0]])
lbl = numpy.array([data[1]])
ret = model.predictor(arr)
ret = F.softmax(ret)
ret.to_cpu()
true_label = lbl.tolist()[0]
recon_label = numpy.argmax(ret.data)
#print true_label, recon_label
lbl_true.append(true_label)
lbl_pred.append(recon_label)
if true_label == recon_label:
correct.append(True)
else:
correct.append(False)
# calculate accuracy
num_correct = sum(bool(x) for x in correct)
accuracy = 100.0 * (num_correct / float(len(correct)))
print "Accuracy: {}%".format(accuracy)
# Compute confusion matrix
cnf_matrix = confusion_matrix(lbl_true, lbl_pred)
numpy.set_printoptions(precision=2)
# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix,
classes=class_names,
normalize=True,
title='Normalized confusion matrix')
plt.show()
# model_folder = "20201103-142223" # W vs (SA / SD) - 91.05%
# model_folder = "20201103-143503" # SD vs (W / SD) - 89.39%
model_folder = "20201103-145933" # W vs SA vs SD - 59.38%
MODEL_FOLDER = pathlib.Path(model_folder)
MODEL_DIR = pathlib.Path("logs\\model")
CONF_DIR = pathlib.Path("logs\\conf")
SAVE_DIR = pathlib.Path("result")
# TEST_PARTICIPANT = ["Participant_04"] # Part of training (Seen)
# TEST_PARTICIPANT = ["Participant_07"]
# TEST_PARTICIPANT = ["Participant_03"] # Excluded participant (Unseen)
TEST_PARTICIPANT = ["Participant_16"]
conf = load_config(config_file=str(CONF_DIR / MODEL_FOLDER / "config.yaml"))
conf["data"]["data_settings"]["skip"] = 0 # Don't skip anything
# Load tensorflow model
print("Loading model")
model = tf.keras.models.load_model(str(MODEL_DIR / MODEL_FOLDER))
print("Model loaded")
model.summary()
# Load Data
data_files = Data_Preload(data_folder=conf["data"]["folder"], load_augment=False)
if SAVE_CLASS_PREDICTIONS:
prediction = []
mappings = {0: 0, 3: 0, 4: 0}
# THIS FILE IS FOR HYPERPARAMETER SEARCH WITH Weights&Biases
# IT SHOULDN't BE EXECUTED DIRECTLY, IT SHOULD BE RUN BY WANDB SWEEP AGENT!
import wandb
import train
# Choose which run config to optimize
RUN_CONFIG_NAME = "MNIST_CLASSIFIER_V1"
# Load configs
project_config = train.load_config("project_config.yaml")
run_config = train.load_config("run_configs.yaml")[RUN_CONFIG_NAME]
# Initialize wandb
wandb.init()
# Replace run config hyperparameters with the ones loaded from wandb sweep server
sweep_hparams = wandb.Config._as_dict(wandb.config)
for key, value in sweep_hparams.items():
if key == "_wandb":
continue
elif key in run_config["trainer"].keys():
run_config["trainer"][key] = value
elif key in run_config["model"].keys():
run_config["model"][key] = value
elif key in run_config["dataset"].keys():
run_config["dataset"][key] = value
# Execute run
train.train(project_config=project_config,
run_config=run_config,
self.ou_a = ou_a
self.ou_mu = ou_mu
print(self.__dict__)
def brownian_motion_log_returns(self):
sqrt_delta_sigma = np.sqrt(self.delta) * self.sigma
return np.random.normal(loc=0, scale=sqrt_delta_sigma, size=None)
def ornstein_uhlenbeck_level(self, prev_ou_level):
drift = self.ou_a * (self.ou_mu - prev_ou_level) * self.delta
randomness = self.brownian_motion_log_returns()
return prev_ou_level + drift + randomness
if __name__ == "__main__":
from train import load_config
config = load_config("config_g2g.yaml")
noise_params = {k: np.array(list(map(float, v.split(","))))
for k, v in config["noise_params"].items()}
noise = Noise(**noise_params)
ou_lvl = np.zeros(2)
v = []
for i in range(1000000):
# print(ou_lvl)
ou_lvl = noise.ornstein_uhlenbeck_level(ou_lvl)
v.append(ou_lvl)
if (i+1) % 50 == 0:
ou_lvl = np.zeros(2)
import matplotlib.pyplot as plt
l, a = list(zip(*v))
plt.hist(a, np.linspace(-1.5, 1.5, 300))
