def main():
args = parse_args()
model = Model()
with open(args.model, 'rb') as file:
model.load(file)
generator = Generator(model)
if args.output is not None:
file = open(args.output, 'a', encoding='utf-8')
else:
file = stdout
for i in range(0, args.count):
text = generator.generate(args.length,
seed=args.seed,
min_n=args.min_n,
n=args.n,
break_on_end=args.break_on_end)
if args.wrap:
text = '\n'.join(wrap(text))
print(text, '\n', file=file)
if args.output is not None:
file.close()
def upload_image():
if request.method == 'POST':
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
file = request.files['file']
if file.filename == '':
flash('No image selected for uploading')
return redirect(request.url)
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
print(os.path.join(app.config['UPLOAD_FOLDER'], filename))
flash('Image successfully uploaded')
text = IT.imageToText(
os.path.join(app.config['UPLOAD_FOLDER'], filename))
summary = Model.main(text)
return render_template('index.html',
filename=filename,
text=text,
summary=summary)
else:
flash('Extensions not Allowed')
return redirect(request.url)
def train(hidden_dim_sweep=(5, 10, 25),
n_epochs=20,
out_dir='out',
data_dir='data',
device=util.device(),
Optimizer=optim.Adam,
seed=42):
out_dir, data_dir = map(Path, (out_dir, data_dir))
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
tracess = []
best_trainer = None
best_loss = util.INF
vocab = util.Vocab.load(data_dir / 'vocab.txt')
for hidden_dim in hidden_dim_sweep:
model = Model(hidden_dim=hidden_dim, vocab=vocab, out_dim=2)
loss_fn = nn.CrossEntropyLoss()
optimizer = Optimizer(model.parameters(), lr=1e-4)
trainer = Trainer(model, loss_fn, vocab, device)
traces, loss_cur = trainer.train_loop(data_dir=data_dir,
n_epochs=n_epochs,
optimizer=optimizer,
scheduler=None)
if loss_cur < best_loss:
best_trainer = trainer
best_loss = loss_cur
tracess.append((hidden_dim, traces))
out_dir.mkdir(exist_ok=True)
for h, traces in tracess:
plotting.plot_traces(traces,
out=out_dir / f'traces_{h}.png',
title=f'Loss,hidden_dim={h}')
util.jsondump(traces, out_dir / f'traces.dim_{h}.seed_{seed}.json')
L.info('Best model loss: %s', best_loss)
model_file = out_dir / 'model.pt'
L.info('Saving best model to %s', model_file)
torch.save(best_trainer.model.state_dict(), model_file)
def test(hidden_dim=25, out_dir='out', data_dir='data', device=util.device()):
out_dir, data_dir = map(Path, (out_dir, data_dir))
vocab = util.Vocab.load(data_dir / 'vocab.txt')
model = Model(hidden_dim=hidden_dim, vocab=vocab)
model.load_state_dict(torch.load(out_dir / 'model.pt'))
loss_fn = nn.CrossEntropyLoss()
trainer = Trainer(model, loss_fn, vocab, device)
with data.SentPairStream(data_dir / 'dev.tsv') as dev_data:
dev_loader = torchdata.DataLoader(dev_data,
shuffle=False,
batch_size=8)
dev_metrics = trainer.eval_(dev_loader)
L.info('Dev performance: %s', dev_metrics)
with data.SentPairStream(data_dir / 'test.tsv') as test_data:
test_loader = torchdata.DataLoader(test_data,
shuffle=False,
batch_size=8)
test_metrics = trainer.eval_(test_loader)
L.info('Test performance: %s', test_metrics)
def test_cl_rsd():
ell = np.arange(2, 500)
# Anna Porredon
cl_anna = np.loadtxt('C_ells_bin1_1_linear.txt')
# Mehdi Rezaie
cosmo = init_cosmology()
z, b, dNdz = init_sample('mock', verb=False)
th = Model(cosmo)
th.add_tracer(z, b, dNdz)
cl_null = th.compute_cl(ell, fnl=0.0, has_rsd=False, has_fnl=False)
cl_rsd = th.compute_cl(ell, fnl=0.0, has_rsd=True, has_fnl=False)
## additional runs
#cl_fnlp = th.compute_cl(ell, fnl=100.0, has_rsd=True, has_fnl=True)
#cl_fnln = th.compute_cl(ell, fnl=-100.0, has_rsd=True, has_fnl=True)
#cls_ccl = run_ccl(cosmo, (z, dNdz), (z, b), ell)
#cls_ccl_rsd = run_ccl(cosmo, (z, dNdz), (z, b), ell, has_rsd=True)
assert (abs(cl_rsd-cl_anna[2:, 1]) < 1.0e-6).all()
fig, ax = plt.subplots()
ax.plot(ell, cl_null, 'C0--', alpha=0.5, label='FFTlog')
ax.plot(ell, cl_rsd, 'C0-', lw=1, alpha=0.5, label='FFTlog+RSD')
ax.plot(cl_anna[2:, 0], cl_anna[2:, 1], 'r:', label='Anna')
## additional curves
#ax.plot(ell, cl_fnlp, 'C0-.', lw=1, label='FFTlog+RSD (fnl=100)')
#ax.plot(ell, cl_fnln, 'C1-.', lw=1, label='FFTlog+RSD (fnl=-100)')
#ax.plot(ell, a*cls_ccl, 'C1-', alpha=0.8, label='CCL (Limber)')
#ax.plot(ell, a*cls_ccl_rsd,'C1--', lw=1, label='CCL+RSD')
ax.legend(frameon=False, ncol=2, loc='lower left', fontsize=10)
ax.set(xscale='log', yscale='log', xlabel=r'$\ell$', ylabel=r'C$_{\ell}$')
ax.tick_params(direction='in', which='both', axis='both', right=True, top=True)
ax.grid(True, ls=':', color='grey', which='both', lw=0.2)
# ax0 = fig.add_axes([0.2, 0.2, 0.4, 0.3])
# add_plot(ax0)
# ax0.set(xlim=(1.9, 5), ylim=(0.9e-6, 2.1e-6))
# ax0.set_xticks([2, 3, 4])
ax.set_ylim(8.0e-8, 2.0e-5)
fig.savefig('cl_fftlog_ccl_benchmark.png',
dpi=300, bbox_inches='tight', facecolor='w')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--model_name', type=str, required=True)
parser.add_argument('--vocabulary_file', type=str, required=True)
parser.add_argument('--output_file', type=str, required=True)
parser.add_argument('--seed', type=str, default="Once upon a time, ")
parser.add_argument('--sample_length', type=int, default=1500)
parser.add_argument('--log_frequency', type=int, default=100)
args = parser.parse_args()
model_name = args.model_name
vocabulary_file = args.vocabulary_file
output_file = args.output_file
seed = args.seed.decode('utf-8')
sample_length = args.sample_length
log_frequency = args.log_frequency
model = Model(model_name)
model.restore()
classifier = model.get_classifier()
vocabulary = Vocabulary()
vocabulary.retrieve(vocabulary_file)
sample_file = codecs.open(output_file, 'w', 'utf_8')
stack = deque([])
for i in range(0, model.sequence_length - len(seed)):
stack.append(u' ')
for char in seed:
if char not in vocabulary.vocabulary:
print char,"is not in vocabulary file"
char = u' '
stack.append(char)
sample_file.write(char)
with tf.Session() as sess:
tf.global_variables_initializer().run()
saver = tf.train.Saver(tf.global_variables())
ckpt = tf.train.get_checkpoint_state(model_name)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
for i in range(0, sample_length):
vector = []
for char in stack:
vector.append(vocabulary.binary_vocabulary[char])
vector = np.array([vector])
prediction = sess.run(classifier, feed_dict={model.x: vector})
predicted_char = vocabulary.char_lookup[np.argmax(prediction)]
stack.popleft()
stack.append(predicted_char)
sample_file.write(predicted_char)
if i % log_frequency == 0:
print "Progress: {}%".format((i * 100) / sample_length)
sample_file.close()
print "Sample saved in {}".format(output_file)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--training_file', type=str, required=True)
parser.add_argument('--vocabulary_file', type=str, required=True)
parser.add_argument('--model_name', type=str, required=True)
parser.add_argument('--epoch', type=int, default=200)
parser.add_argument('--batch_size', type=int, default=50)
parser.add_argument('--sequence_length', type=int, default=50)
parser.add_argument('--log_frequency', type=int, default=100)
parser.add_argument('--learning_rate', type=int, default=0.002)
parser.add_argument('--units_number', type=int, default=128)
parser.add_argument('--layers_number', type=int, default=2)
args = parser.parse_args()
training_file = args.training_file
vocabulary_file = args.vocabulary_file
model_name = args.model_name
epoch = args.epoch
batch_size = args.batch_size
sequence_length = args.sequence_length
log_frequency = args.log_frequency
learning_rate = args.learning_rate
batch = Batch(training_file, vocabulary_file, batch_size, sequence_length)
input_number = batch.vocabulary.size
classes_number = batch.vocabulary.size
units_number = args.units_number
layers_number = args.layers_number
print "Start training with epoch: {}, batch_size: {}, log_frequency: {}," \
"learning_rate: {}".format(epoch, batch_size, log_frequency, learning_rate)
if not os.path.exists(model_name):
os.makedirs(model_name)
model = Model(model_name)
model.build(input_number, sequence_length, layers_number, units_number, classes_number)
classifier = model.get_classifier()
cost = tf.reduce_mean(tf.square(classifier - model.y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
expected_prediction = tf.equal(tf.argmax(classifier, 1), tf.argmax(model.y, 1))
accuracy = tf.reduce_mean(tf.cast(expected_prediction, tf.float32))
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
iteration = 0
while batch.dataset_full_passes < epoch:
iteration += 1
batch_x, batch_y = batch.get_next_batch()
batch_x = batch_x.reshape((batch_size, sequence_length, input_number))
sess.run(optimizer, feed_dict={model.x: batch_x, model.y: batch_y})
if iteration % log_frequency == 0:
acc = sess.run(accuracy, feed_dict={model.x: batch_x, model.y: batch_y})
loss = sess.run(cost, feed_dict={model.x: batch_x, model.y: batch_y})
print("Iteration {}, batch loss: {:.6f}, training accuracy: {:.5f}".format(iteration * batch_size,
loss, acc))
batch.clean()
print("Optimization done")
saver = tf.train.Saver(tf.global_variables())
checkpoint_path = "{}/{}.ckpt".format(model_name, model_name)
saver.save(sess, checkpoint_path, global_step=iteration * batch_size)
print("Model saved in {}".format(model_name))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--training_file', type=str, required=True)
parser.add_argument('--vocabulary_file', type=str, required=True)
parser.add_argument('--model_name', type=str, required=True)
parser.add_argument('--epoch', type=int, default=200)
parser.add_argument('--batch_size', type=int, default=50)
parser.add_argument('--sequence_length', type=int, default=50)
parser.add_argument('--log_frequency', type=int, default=100)
parser.add_argument('--learning_rate', type=int, default=0.002)
parser.add_argument('--units_number', type=int, default=128)
parser.add_argument('--layers_number', type=int, default=2)
args = parser.parse_args()
training_file = args.training_file
vocabulary_file = args.vocabulary_file
model_name = args.model_name
epoch = args.epoch
batch_size = args.batch_size
sequence_length = args.sequence_length
log_frequency = args.log_frequency
learning_rate = args.learning_rate
batch = Batch(training_file, vocabulary_file, batch_size, sequence_length)
input_number = batch.vocabulary.size
classes_number = batch.vocabulary.size
units_number = args.units_number
layers_number = args.layers_number
print "Start training with epoch: {}, batch_size: {}, log_frequency: {}," \
"learning_rate: {}".format(epoch, batch_size, log_frequency, learning_rate)
if not os.path.exists(model_name):
os.makedirs(model_name)
model = Model(model_name)
model.build(input_number, sequence_length, layers_number, units_number, classes_number)
classifier = model.get_classifier()
cost = tf.reduce_mean(tf.square(classifier - model.y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
expected_prediction = tf.equal(tf.argmax(classifier, 1), tf.argmax(model.y, 1))
accuracy = tf.reduce_mean(tf.cast(expected_prediction, tf.float32))
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
iteration = 0
while batch.dataset_full_passes < epoch:
iteration += 1
batch_x, batch_y = batch.get_next_batch()
batch_x = batch_x.reshape((batch_size, sequence_length, input_number))
sess.run(optimizer, feed_dict={model.x: batch_x, model.y: batch_y})
if iteration % log_frequency == 0:
acc = sess.run(accuracy, feed_dict={model.x: batch_x, model.y: batch_y})
loss = sess.run(cost, feed_dict={model.x: batch_x, model.y: batch_y})
print("Iteration {}, batch loss: {:.6f}, training accuracy: {:.5f}".format(iteration * batch_size,
loss, acc))
batch.clean()
print("Optimization done")
saver = tf.train.Saver(tf.global_variables())
checkpoint_path = "{}/{}.ckpt".format(model_name, model_name)
saver.save(sess, checkpoint_path, global_step=iteration * batch_size)
print("Model saved in {}".format(model_name))
], p=1.0)
t_dataset=MelanomaDataset(df=df, imfolder=test,
train=False, transforms=test_transform, meta_features=meta_features)
print('Length of test set is {}'.format(len(t_dataset)))
testloader=DataLoader(t_dataset, batch_size=8, shuffle=False, num_workers=8)
"""Testing"""
# model = ResNetModel()()
# model = EfficientModel()
# model = EfficientModel(n_meta_features=len(meta_features))
model = Model(arch='efficientnet-b1')
# model.load_state_dict(torch.load("../checkpoint/fold_1/efficient_256/efficientb0_256_14_0.9212.pth", map_location=torch.device(device)))
model.load_state_dict(torch.load("..//checkpoint/fold_1/efficient_320/efficientb1_320_14_0.9293.pth", map_location=torch.device(device)))
model.to(device)
model.eval()
test_prob_stack = []
img_ids = []
with torch.no_grad():
for i in range(15):
test_prob = []
for img, meta, img_id in tqdm(testloader):
if train_on_gpu:
img, meta = img.to(device), meta.to(device)
logits = model.forward(img)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--model_name', type=str, required=True)
parser.add_argument('--vocabulary_file', type=str, required=True)
parser.add_argument('--output_file', type=str, required=True)
parser.add_argument('--seed', type=str, default="Once upon a time, ")
parser.add_argument('--sample_length', type=int, default=1500)
parser.add_argument('--log_frequency', type=int, default=100)
args = parser.parse_args()
model_name = args.model_name
vocabulary_file = args.vocabulary_file
output_file = args.output_file
seed = args.seed.decode('utf-8')
sample_length = args.sample_length
log_frequency = args.log_frequency
model = Model(model_name)
model.restore()
classifier = model.get_classifier()
vocabulary = Vocabulary()
vocabulary.retrieve(vocabulary_file)
sample_file = codecs.open(output_file, 'w', 'utf_8')
stack = deque([])
for i in range(0, model.sequence_length - len(seed)):
stack.append(u' ')
for char in seed:
if char not in vocabulary.vocabulary:
print char,"is not in vocabulary file"
char = u' '
stack.append(char)
sample_file.write(char)
with tf.Session() as sess:
tf.global_variables_initializer().run()
saver = tf.train.Saver(tf.global_variables())
ckpt = tf.train.get_checkpoint_state(model_name)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
for i in range(0, sample_length):
vector = []
for char in stack:
vector.append(vocabulary.binary_vocabulary[char])
vector = np.array([vector])
prediction = sess.run(classifier, feed_dict={model.x: vector})
predicted_char = vocabulary.char_lookup[np.argmax(prediction)]
stack.popleft()
stack.append(predicted_char)
sample_file.write(predicted_char)
if i % log_frequency == 0:
print "Progress: {}%".format((i * 100) / sample_length)
sample_file.close()
print "Sample saved in {}".format(output_file)
def signIn(_data):
_user = Model.UserExists(_data)
if _user:
auth.logInUser(_user)
return True
return False
import numpy as np
import matplotlib.pyplot as plt
from IPython.display import display, Math
import zeus
from modules import init_cosmology, init_sample, Model
import sys
sys.path.append('/Users/rezaie/github/LSSutils')
from lssutils.utils import histogram_cell
# --- theory
cosmo = init_cosmology()
z, b, dNdz = init_sample()
model = Model(cosmo)
model.add_tracer(z, b, dNdz)
# data
cl_mocks = np.load('cl_mocks_1k.npz', allow_pickle=True)
cl_full = cl_mocks['full'] # select full sky mocks
# bin measurements
bins = np.arange(1, 501, 20)
cl_fullb = []
for i in range(cl_full.shape[0]):
x, clb_ = histogram_cell(cl_full[i, :], bins=bins)
cl_fullb.append(clb_)
#print('.', end='')
cl_fullb = np.array(cl_fullb)
y = cl_fullb.mean(axis=0)
nmocks, nbins = cl_fullb.shape
hf = (nmocks - 1.0) / (nmocks - nbins - 2.0)
t_dataset=MelanomaDataset(df=df_train, imfolder=train,
train=True, transforms=train_transform, meta_features=meta_features)
v_dataset=MelanomaDataset(df=df_valid, imfolder=train,
train=True, transforms=valid_transform, meta_features=meta_features)
print('Length of training and validation set are {} {}'.format(
len(t_dataset), len(v_dataset)))
trainloader=DataLoader(t_dataset, batch_size=32, shuffle=True, num_workers=8)
validloader=DataLoader(v_dataset, batch_size=32, shuffle=False, num_workers=8)
""" Training """
# model = ResNetModel()
# model = EfficientModelwithoutMeta()
model = Model(arch='efficientnet-b2')
# model = EfficientModel(n_meta_features=len(meta_features))
model.to(device)
# model = nn.DataParallel(model)
criterion=nn.BCEWithLogitsLoss()
optimizer=torch.optim.AdamW(model.parameters(), lr=3e-4, betas=(0.9, 0.999))
scheduler=torch.optim.lr_scheduler.OneCycleLR(
optimizer, max_lr=3e-4, div_factor=10, pct_start=1 / epochs, steps_per_epoch=len(trainloader), epochs=epochs)
writer = SummaryWriter(f'../checkpoint/fold_{fold}/efficient_{resolution}')
print(f'Training Started Fold_{fold}')
training_loss = []
validation_loss = []
c_acc = 0.0