def test(self):
"""Test Function."""
print("Testing the results")
self.inputs = data_loader.load_data(self._dataset_name,
self._size_before_crop, False,
self._do_flipping)
self.model_setup()
saver = tf.train.Saver()
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
chkpt_fname = tf.train.latest_checkpoint(self._checkpoint_dir)
saver.restore(sess, chkpt_fname)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
self._num_imgs_to_save = cyclegan_datasets.DATASET_TO_SIZES[
self._dataset_name]
self.save_images(sess, 0)
coord.request_stop()
coord.join(threads)
def main():
# Load config
config = get_config_from_json('config/model.config')
# Load data
[X_train, Y_train, X_CV, Y_CV, X_test, Y_test] = load_data(0.18)
# Generate dataset
test_dataset = MyDataset(X_test, Y_test)
# Create Data Loaders
test_loader = DataLoader(dataset=test_dataset,
batch_size=config.model.batch_size,
shuffle=False)
# Build model
model = CNNModel()
model = model.double()
model.eval()
if os.path.isfile(config.model.path):
model.load_state_dict(torch.load(config.model.path))
print('Loaded checkpoint..')
else:
print('checkpoint not found..')
evaluate(model, test_loader)
def mainClassification():
# Load config
config = get_config_from_json('config/modelClassification.config')
# Load data
[X_train, Y_train, X_CV, Y_CV, X_test, Y_test] = load_data(0.18)
# Generate dataset
train_dataset = MyDataset(X_train, Y_train)
val_dataset = MyDataset(X_CV, Y_CV)
# Create Data Loaders
train_loader = DataLoader(dataset=train_dataset,
batch_size=config.model.batch_size,
shuffle=True)
val_loader = DataLoader(dataset=val_dataset,
batch_size=config.model.batch_size,
shuffle=False)
# Build model
model = CNNModel()
model = model.double()
print(model)
# Train model
train(model, config, train_loader, val_loader)
def load_data(filepath, cfg, feature_list, start_feature=5, mask_value=0):
X_train_data, y_train_data = data_loader.load_data(
datafile=filepath + 'normalized_training.csv',
flare_label=cfg.flare_label,
series_len=cfg.seq_len,
start_feature=start_feature,
n_features=cfg.n_features,
mask_value=mask_value,
feature_list=feature_list)
X_train_data = np.reshape(X_train_data,
(len(X_train_data), cfg.n_features))
X_train_data = X_train_data.astype(np.float32)
y_train_data = data_loader.label_transform(y_train_data)
X_valid_data, y_valid_data = data_loader.load_data(
datafile=filepath + 'normalized_validation.csv',
flare_label=cfg.flare_label,
series_len=cfg.seq_len,
start_feature=start_feature,
n_features=cfg.n_features,
mask_value=mask_value,
feature_list=feature_list)
X_valid_data = np.reshape(X_valid_data,
(len(X_valid_data), cfg.n_features))
X_valid_data = X_valid_data.astype(np.float32)
y_valid_data = data_loader.label_transform(y_valid_data)
X_test_data, y_test_data = data_loader.load_data(
datafile=filepath + 'normalized_testing.csv',
flare_label=cfg.flare_label,
series_len=cfg.seq_len,
start_feature=start_feature,
n_features=cfg.n_features,
mask_value=mask_value,
feature_list=feature_list)
X_test_data = np.reshape(X_test_data, (len(X_test_data), cfg.n_features))
X_test_data = X_test_data.astype(np.float32)
y_test_data = data_loader.label_transform(y_test_data)
return X_train_data, X_valid_data, X_test_data, y_train_data, y_valid_data, y_test_data
def mainSiamese():
# Load config
config = get_config_from_json('config/modelSiamese.config')
# Load data
[X_train, Y_train, X_CV, Y_CV, X_test, Y_test] = load_data(0.18)
# Generate dataset
test_dataset = MySiameseDataset(X_test, Y_test)
# Create Data Loaders
test_loader = DataLoader(dataset=test_dataset, batch_size=1, shuffle=True)
# Build model
model = SiameseModel()
model = model.double()
print(model)
model.eval()
if os.path.isfile(config.model.path):
model.load_state_dict(torch.load(config.model.path))
print('Loaded checkpoint..')
else:
print('checkpoint not found..')
dataiter = iter(test_loader)
x0, _, _ = next(dataiter)
plt.tight_layout()
plt.subplot(4, 3, 2)
plt.axis('off')
plt.imshow(x0[0][0], 'gray')
plt.title('Original Image', fontdict={'fontsize': 10})
for i in range(9):
_, x1, label = next(dataiter)
output0, output1 = model(x0, x1)
output0 = output0.type(torch.DoubleTensor)
output1 = output1.type(torch.DoubleTensor)
euclidean_distance = F.pairwise_distance(output0, output1)
plt.subplot(4, 3, i + 4)
plt.axis('off')
plt.imshow(x1[0][0], 'gray')
plt.title(str(round(euclidean_distance.item(), 2)),
fontdict={'fontsize': 10})
plt.show()
def train(self):
"""Training Function."""
# Load Dataset from the dataset folder
self.inputs = data_loader.load_data(self._dataset_name,
self._size_before_crop, True,
self._do_flipping)
# Build the network
self.model_setup()
# Loss function calculations
self.compute_losses()
# Initializing the global variables
init = (tf.global_variables_initializer(),
tf.local_variables_initializer())
saver = tf.train.Saver()
max_images = cyclegan_datasets.DATASET_TO_SIZES[self._dataset_name]
with tf.Session() as sess:
sess.run(init)
# Restore the model to run the model from last checkpoint
if self._to_restore:
chkpt_fname = tf.train.latest_checkpoint(self._checkpoint_dir)
saver.restore(sess, chkpt_fname)
writer = tf.summary.FileWriter(self._output_dir)
if not os.path.exists(self._output_dir):
os.makedirs(self._output_dir)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
# Training Loop
for epoch in range(sess.run(self.global_step), self._max_step):
print("In the epoch ", epoch)
saver.save(sess,
os.path.join(self._output_dir, "cyclegan"),
global_step=epoch)
# Dealing with the learning rate as per the epoch number
if epoch < 100:
curr_lr = self._base_lr
else:
curr_lr = self._base_lr - \
self._base_lr * (epoch - 100) / 100
self.save_images(sess, epoch)
for i in range(0, max_images):
print("Processing batch {}/{}".format(i, max_images))
inputs = sess.run(self.inputs)
# Optimizing the G_A network
_, fake_B_temp, summary_str = sess.run(
[
self.g_A_trainer, self.fake_images_b,
self.g_A_loss_summ
],
feed_dict={
self.input_a: inputs['images_i'],
self.input_b: inputs['images_j'],
self.learning_rate: curr_lr
})
writer.add_summary(summary_str, epoch * max_images + i)
fake_B_temp1 = self.fake_image_pool(
self.num_fake_inputs, fake_B_temp, self.fake_images_B)
# Optimizing the D_B network
_, summary_str = sess.run(
[self.d_B_trainer, self.d_B_loss_summ],
feed_dict={
self.input_a: inputs['images_i'],
self.input_b: inputs['images_j'],
self.learning_rate: curr_lr,
self.fake_pool_B: fake_B_temp1
})
writer.add_summary(summary_str, epoch * max_images + i)
# Optimizing the G_B network
_, fake_A_temp, summary_str = sess.run(
[
self.g_B_trainer, self.fake_images_a,
self.g_B_loss_summ
],
feed_dict={
self.input_a: inputs['images_i'],
self.input_b: inputs['images_j'],
self.learning_rate: curr_lr
})
writer.add_summary(summary_str, epoch * max_images + i)
fake_A_temp1 = self.fake_image_pool(
self.num_fake_inputs, fake_A_temp, self.fake_images_A)
# Optimizing the D_A network
_, summary_str = sess.run(
[self.d_A_trainer, self.d_A_loss_summ],
feed_dict={
self.input_a: inputs['images_i'],
self.input_b: inputs['images_j'],
self.learning_rate: curr_lr,
self.fake_pool_A: fake_A_temp1
})
writer.add_summary(summary_str, epoch * max_images + i)
writer.flush()
self.num_fake_inputs += 1
sess.run(tf.assign(self.global_step, epoch + 1))
coord.request_stop()
coord.join(threads)
writer.add_graph(sess.graph)
# Enabling cross-site scripting (might need to be removed later):
# cors = CORS(app)s
# Parameters for filtering template spreadsheets:
# fiterParams = api.model( "Diagram data filter application",{
# 'pmid' : fields.String(description="Pubmed ID of a requested publication", required=False, default=False),
# 'efo' : fields.String(description="EFO id of the term", required=False, default=False),
# 'pvalue' : fields.String(description="Upper boundary of the p-value (eg. 1e-8)", required=False, default=False),
# })
# Loading data - loaded once, filtered after:
parent_mapping_file = Configuration.parent_mapping_file
association_file = Configuration.association_file
ancestry_file = Configuration.ancestry_file
gwas_data = load_data(parent_mapping_file,association_file,ancestry_file)
@api.route('/v1/filter')
@api.expect(fiterParams, validate=True)
class diagarmFilter(Resource):
@api.doc('Filter diagram data')
def post(self):
global gwas_data
# Parsing and validating input paramters
parameters = eu.validate_paramters(fiterParams.parse_args())
print(parameters)
# Get filtered dataset:
filteredData = filter(gwas_data, parameters)
from data_loader.data_loader import load_data
from utils.load_model_optimizer_lr import load_model_op_lr
from utils.get_result import get_result
from utils.utils import get_config_from_json
config_file = 'config1.json'
config = get_config_from_json(config_file)
for k, v in config.items():
print(k, ': ', v)
# 1.加载数据
print('loading data.')
if config['only_train']:
train_loader = load_data(config)
else:
train_loader, valid_loader, test_loader = load_data(config)
print('loaded data.')
# 2.加载模型、优化器、损失函数、学习率计划器
sentiment_model, optimizer, criterion, scheduler = load_model_op_lr(config, len(train_loader))
def validation(sentiment_model):
y_pred = []
y_true = []
total_eval_loss = 0
sentiment_model.eval() # eval模式下dropout不会工作
for i, (inp, tar) in enumerate(valid_loader):
for idx in range(len(words_row)):
if labels_row[idx] == "B" and idx + 1 < len(
words_row) and labels_row[idx + 1] == "I":
keywords_tmp.append(words_row[idx] + " " + words_row[idx + 1])
elif labels_row[idx] == "I":
if idx > 0 and labels_row[idx - 1] == "B":
continue
else:
keywords_tmp.append(words_row[idx])
keywords.append(keywords_tmp)
return keywords
if __name__ == "__main__":
dataset = load_data(".\\kpwr-1.1\\*\\result.csv")
features = create_features_list(dataset)
dataset["features"] = features
train, test = train_test_split(dataset)
CRF.train(train['features'], train['label_base'])
preds = CRF.test(test['features'])
keywords_true = test['base_keywords_in_text']
keywords_pred = get_keywords_from_labels(test['base_words_list'], preds)
prec_h, rec_h, f1_h = evaluator.hard_evaluation(keywords_true,
keywords_pred)
prec_s, rec_s, f1_s = evaluator.soft_evaluation(keywords_true,
keywords_pred)