def main(config):
logger = config.get_logger('train')
# fix random seeds for reproducibility
seed_everything(seed=config.config['seed'])
metric_bests = []
# logger = config.get_logger('train')
for i,train_dataloader, valid_dataloader, test_dataloader in makeDataLoader(config):
model = makeModel(config)
# logger.info(model)
criterion = makeLoss(config)
metrics = makeMetrics(config)
optimizer = makeOptimizer(config, model)
lr_scheduler = makeLrSchedule(config, optimizer, train_dataloader)
trainer = Trainer(model, criterion, metrics, optimizer,
config=config,
i_fold=i,
data_loader=train_dataloader,
valid_data_loader=valid_dataloader,
test_data_loader=test_dataloader,
lr_scheduler=lr_scheduler)
trainer.train()
metric_bests.append(trainer.mnt_best)
logger.info('metric scores:{}'.format(metric_bests))
logger.info('metric mean score: {}'.format(sum(metric_bests) / float(len(metric_bests))))
def main(config):
from data_process import makeDataLoader
#
from trainer.htqe_trainer import Trainer
logger = config.get_logger('train')
train_dataloader, valid_dataloader, test_dataloader = makeDataLoader(
config)
model = makeModel(config)
logger.info(model)
criterion = makeLoss(config)
metrics = makeMetrics(config)
optimizer = makeOptimizer(config, model)
lr_scheduler = makeLrSchedule(config, optimizer, train_dataloader.dataset)
trainer = Trainer(model,
criterion,
metrics,
optimizer,
config=config,
data_loader=train_dataloader,
valid_data_loader=valid_dataloader,
test_data_loader=test_dataloader,
lr_scheduler=lr_scheduler)
trainer.train()
def main(config):
from data_process import makeDataLoader
# 针对不同的数据，训练过程的设置略有不同。
# from trainer.weibo_trainer import Trainer # weibo
# from trainer.cnews_trainer import Trainer # cnews
from trainer.medical_question_trainer import Trainer
logger = config.get_logger('train')
train_dataloader, valid_dataloader, test_dataloader = makeDataLoader(config)
model = makeModel(config)
logger.info(model)
criterion = makeLoss(config)
metrics = makeMetrics(config)
optimizer = makeOptimizer(config, model)
lr_scheduler = makeLrSchedule(config, optimizer, train_dataloader.dataset)
trainer = Trainer(model, criterion, metrics, optimizer,
config=config,
data_loader=train_dataloader,
valid_data_loader=valid_dataloader,
test_data_loader=test_dataloader,
lr_scheduler=lr_scheduler)
trainer.train()
def active_learning(config):
from data_process import makeDataSet
from trainer.trainer import Trainer
logger = config.get_logger('train')
train_set, valid_set, query_set = makeDataSet(config)
model = makeModel(config)
logger.info(model)
criterion = makeLoss(config)
metrics = makeMetrics(config)
optimizer = makeOptimizer(config, model)
# lr_scheduler = makeLrSchedule(config, optimizer, train_set)
trainer = Trainer(
model,
criterion,
metrics,
optimizer,
config=config,
train_dataset=train_set,
valid_dataset=valid_set,
query_dataset=query_set,
test_dataset=None,
# lr_scheduler=lr_scheduler
)
trainer.train()
def main(config):
logger = config.get_logger('train')
# setup data_loader instances
data_loader = config.init_obj('data_loader', module_data)
valid_data_loader = data_loader.split_validation()
# build model architecture, then print to console
model = makeModel(config)
logger.info(model)
# get function handles of loss and metrics
criterion = makeLoss(config)
metrics = makeMetrics(config)
# build optimizer, learning rate scheduler. delete every lines containing lr_scheduler for disabling scheduler
optimizer = makeOptimizer(config, model)
lr_scheduler = config.init_obj('lr_scheduler', torch.optim.lr_scheduler,
optimizer)
trainer = Trainer(model,
criterion,
metrics,
optimizer,
config=config,
data_loader=data_loader,
valid_data_loader=valid_data_loader,
lr_scheduler=lr_scheduler)
trainer.train()
def main(config):
logger = config.get_logger('test')
# setup data_loader instances
data_loader = getattr(module_data, config['data_loader']['type'])(
config['data_loader']['args']['data_dir'],
batch_size=512,
shuffle=False,
validation_split=0.0,
training=False,
num_workers=2)
# build model architecture
model = makeModel(config)
logger.info(model)
# get function handles of loss and metrics
loss_fn = makeLoss(config)
metric_fns = makeMetrics(config)
logger.info('Loading checkpoint: {} ...'.format(config.resume))
checkpoint = torch.load(config.resume)
state_dict = checkpoint['state_dict']
if config['n_gpu'] > 1:
model = torch.nn.DataParallel(model)
model.load_state_dict(state_dict)
# prepare model for testing
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = model.to(device)
model.eval()
total_loss = 0.0
total_metrics = torch.zeros(len(metric_fns))
with torch.no_grad():
for i, (data, target) in enumerate(tqdm(data_loader)):
data, target = data.to(device), target.to(device)
output = model(data)
#
# save sample images, or do something with output here
#
# computing loss, metrics on test set
loss = loss_fn(output, target)
batch_size = data.shape[0]
total_loss += loss.item() * batch_size
for i, metric in enumerate(metric_fns):
total_metrics[i] += metric(output, target) * batch_size
n_samples = len(data_loader.sampler)
log = {'loss': total_loss / n_samples}
log.update({
met.__name__: total_metrics[i].item() / n_samples
for i, met in enumerate(metric_fns)
})
logger.info(log)
def main():
size = (256, 256)
weights = '4_50_0.1156_0.9848_0.6352_0.9524.h5'
model = makeModel(size, weights='weights/' + weights)
videoMap = [(r'D:\DiskE\Computer_Vision_Task\video_6.mp4',
'classificationLogs/video_6_classified_4_50.csv'),
(r'D:\DiskE\Computer_Vision_Task\video_2.mp4',
'classificationLogs/video_2_classified_4_50.csv')]
for srcVideoPath, classifiedVideoPath in videoMap:
logClassification(model, size, srcVideoPath, classifiedVideoPath)
def setupModel(batch, isTraining, add_summary=False):
result = {}
truth = batch["truth"]
nclasses = truth.shape.as_list()[1]
output, model = makeModel(nclasses,
batch["cpf"],
batch["npf"],
batch["sv"],
batch["globals"],
isTraining=isTraining)
result["model"] = model
prediction = tf.nn.softmax(output)
'''
bigtruth = tf.stack([tf.reduce_max(truth[:,0:nclasses-1],axis=1),truth[:,nclasses-1]],axis=1)
bigoutput = tf.stack([tf.reduce_sum(output[:,0:nclasses-1],axis=1),output[:,nclasses-1]],axis=1)
bigprediction = keras.layers.Activation('softmax')(bigoutput)
'''
cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(labels=truth,
logits=output)
loss = tf.reduce_mean(cross_entropy)
#bigcross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(labels=bigtruth,logits=bigoutput)
#bigloss = tf.reduce_mean(bigcross_entropy)
#bigcross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(labels=truth,logits=output)
#bigloss = tf.reduce_mean(cross_entropy)
result["prediction"] = prediction
result["truth"] = truth
#result["bigprediction"] = bigprediction
#result["bigtruth"] = bigtruth
accuracy, accuracy_op = tf.metrics.accuracy(tf.argmax(truth, 1),
tf.argmax(prediction, 1))
result["accuracy"] = accuracy_op
#print "Extra loss in model from Keras:", model.losses
result["loss"] = loss
#result["bigloss"] = bigloss
#result["extraloss"] = tf.reduce_sum(model.losses)
result[
"mimloss"] = loss #+tf.reduce_sum(model.losses) #bigloss+loss+tf.reduce_sum(model.losses)
return result
def train():
inputSize = (150, 150)
model = makeModel(inputSize, compileForTraining=True, weights='first_try.h5')
epochs = 50
batch_size = 64
train_generator, validation_generator = makeDataset('data', inputSize, batch_size)
nb_train_samples = train_generator.samples
nb_validation_samples = validation_generator.samples
model.fit_generator(
train_generator,
steps_per_epoch=nb_train_samples // batch_size,
epochs=epochs,
validation_data=validation_generator,
validation_steps=nb_validation_samples // batch_size)
model.save_weights('first_try.h5')
def main(config):
logger = config.get_logger('train')
train_dataloader, valid_dataloader = makeDataLoader(config)
model = makeModel(config)
logger.info(model)
# criterion = makeLoss(config)
# metrics = makeMetrics(config)
optimizer = makeOptimizer(config, model)
lr_scheduler = makeLrSchedule(config, optimizer, train_dataloader)
trainer = Trainer(model,
None,
None,
optimizer,
config=config,
data_loader=train_dataloader,
valid_data_loader=valid_dataloader,
test_data_loader=None,
lr_scheduler=lr_scheduler)
trainer.train()
saveDir = 'trained_models/'
cwd = os.getcwd()
directory=saveDir+datetime.now().strftime("%d%b_%I%M%S%P_")+ \
str(acc)+'acc_'+ str(nImg)+'I_'+str(epochs)+'ep_' +str(K)+'K'
if not os.path.exists(directory):
os.makedirs(directory)
sessFileName = directory + '/model'
#%% save testing model first.
tf.reset_default_graph()
orgT = tf.placeholder(dtype=tf.complex64, shape=(None, None, None), name='org')
csmT = tf.placeholder(tf.complex64, shape=(None, None, None, None), name='csm')
atbT, predT = mm.makeModel(orgT, csmT, initx, inity, M, N, sigma, lam, K,
False)
atbT = tf.identity(atbT, name='atb')
predT = tf.identity(predT, name='predTst')
sessFileNameTst = directory + '/modelTst'
saver = tf.train.Saver()
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
savedFile = saver.save(sess,
sessFileNameTst,
latest_filename='checkpointTst')
print('testing model saved:' + savedFile)
#%% some tensorflow code for dataset input
tf.reset_default_graph()
for i in tqdm(range(0, len(src), bs)):
X = batch(src[i:i + bs], vocab=vocab_src, padidx=pad_idx)
X = torch.from_numpy(X).to(device)
Y = greedyDecoder(X,
standardMask(X, pad_idx),
model,
startidx=sos_idx,
unk=pad_idx)
ypred.extend(translateBack(Y, inv_vocab_tgt, EOS))
ytrue.extend([' '.join(s) for s in tgt[i:i + bs]])
score = blue([s.lower() for s in ypred], [s.lower() for s in ytrue])
return score
if __name__ == '__main__':
# --model_path=models --datafile=../.data/iwslt/de-en/IWSLT16.TED.tst2014.de-en --vocab=vocab
#29.15,30.8,24.98
parser = parse()
src_vocab, tgt_vocab = loadVocab(parser.vocab)
src, tgt = readFile(parser.datafile)
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
model = makeModel(len(src_vocab), len(tgt_vocab))
model = model.to(device)
restore(model, path=parser.model_path, tocpu=False)
myscore = myEvaluate(model, src, tgt, src_vocab, tgt_vocab, device=device)
print('score:', myscore)
from keras.preprocessing.image import load_img
from keras.preprocessing.image import ImageDataGenerator
import statistics
from collections import Counter
def fixed_generator(generator):
for batch in generator:
yield (batch, batch)
INIT_LR=1e-3
EPOCHS=60
BATCH_SIZE=64
SIZE=32
autoencoder=model.makeModel()
#print(autoencoder.summary())
opt2=RMSprop(lr=INIT_LR, decay=1e-6)
autoencoder.compile(loss="mean_squared_error", optimizer = opt2)
with open('final_data.pickle','rb') as f:
data=pickle.load(f)
with open('final_labels.pickle','rb') as f:
labels=pickle.load(f)
#####
print(data.shape)
print(labels.shape)
cwd = os.getcwd()
directory=saveDir+datetime.now().strftime("%d%b_%I%M%P_")+ \
str(nLayers)+'L_'+str(K)+'K_'+str(epochs)+'E_'+gradientMethod
if not os.path.exists(directory):
os.makedirs(directory)
sessFileName = directory + '/model'
#%% save test model
tf.reset_default_graph()
csmT = tf.placeholder(tf.complex64, shape=(None, 12, 256, 232), name='csm')
maskT = tf.placeholder(tf.complex64, shape=(None, 256, 232), name='mask')
atbT = tf.placeholder(tf.float32, shape=(None, 256, 232, 2), name='atb')
out = mm.makeModel(atbT, csmT, maskT, False, nLayers, K, gradientMethod)
predTst = out['dc' + str(K)]
predTst = tf.identity(predTst, name='predTst')
sessFileNameTst = directory + '/modelTst'
saver = tf.train.Saver()
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
savedFile = saver.save(sess,
sessFileNameTst,
latest_filename='checkpointTst')
print('testing model saved:' + savedFile)
#%% read multi-channel dataset
trnOrg, trnAtb, trnCsm, trnMask = sf.getData('training')
trnOrg, trnAtb = sf.c2r(trnOrg), sf.c2r(trnAtb)
sv = tf.placeholder('float32',
shape=(None, featureDict["sv"]["max"],
len(featureDict["sv"]["branches"])),
name="sv")
event = tf.placeholder('float32',
shape=(None,
len(featureDict["globals"]["branches"])),
name="globals")
print "cpf shape: ", cpf.shape.as_list()
print "npf shape: ", npf.shape.as_list()
print "sv shape: ", sv.shape.as_list()
print "globals shape: ", event.shape.as_list()
output, model = makeModel(len(featureDict["truth"]["branches"]),
cpf,
npf,
sv,
event,
isTraining=False)
prediction = tf.nn.softmax(output, name="prediction")
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess.run(init_op)
model.loadVariables("all/model_epoch28.json", sess)
const_graph = tf.graph_util.convert_variables_to_constants(
sess, sess.graph.as_graph_def(), ["prediction"])
tf.train.write_graph(const_graph, "all", "model_epoch28.pb", as_text=False)
from data import getdata from model import makeModel # Now we wannt numpy for martix math operations import numpy as np # for making the grap we will import matplotlib from matplotlib import style import matplotlib.pyplot as plt import matplotlib.animation as animation # geting the data (only 40% of the data) data = getdata(40) #making the model model = makeModel(consts.lr, consts.th) # initialzing variables for graph fig = plt.figure() ax1 = fig.add_subplot(1, 1, 1) ''' Now we will be training the model with the live stock price graph. we will make a function named animate which will be the function which will interate with time. this function has a variable 'i' which will be the number of interation done. Then we will make a function which will take the number of interations done and output us the training data and label.