def test_value(name, test_set):
report = Report()
fref = open('data/%s.txt'%test_set, 'r', encoding='utf8')
fcan = open('result/%s.%s'%(test_set, name), 'r', encoding='utf8')
reference_all = fref.readlines()
candidate_all = fcan.readlines()
fref.close()
fcan.close()
ref_count = 0
can_count = 0
acc_count = 0
for reference, candidate in zip(reference_all, candidate_all):
reference = reference.strip()
candidate = candidate.strip()
ref_words_len, can_words_len, acc_word_len = report.compare_line(reference, candidate)
ref_count += ref_words_len
can_count += can_words_len
acc_count += acc_word_len
P = acc_count / ref_count * 100
R = acc_count / can_count * 100
F1 = (2 * P * R) / (P+R)
print(name)
print('准确率', P)
print('召回率', R)
print('F1', F1)
print()
def validate(val_loader, epoch):
net.train(False)
reporter = Report()
epoch_bce_loss = 0
epoch_dice_loss = 0
epoch_loss = 0
with torch.no_grad():
for inputs, labels in val_loader:
inputs = inputs.cuda()
labels = labels.cuda()
preds = sigmoid(net(inputs.detach()).detach())
reporter.feed(preds, labels)
bce_crit.weight = get_weight_vector(labels, relative_weight,
is_cuda)
loss = criterion(preds, labels)
epoch_bce_loss += last_bce_loss
epoch_dice_loss += last_dice_loss
epoch_loss += loss.item()
del inputs, labels, preds, loss
avg_bce_loss = epoch_bce_loss / float(len(val_loader))
avg_dice_loss = epoch_dice_loss / float(len(val_loader))
avg_loss = epoch_loss / float(len(val_loader))
avg_acc = reporter.accuracy()
print("[Valid] Epoch({}) Avg BCE Loss: {:.4f} Avg Dice Loss: {:.4f} \
Avg Loss: {:.4f}".format(epoch, avg_bce_loss, avg_dice_loss, avg_loss))
print(reporter)
print(reporter.stats())
return avg_loss, avg_acc
def train(train_loader, epoch):
net.train(True)
reporter = Report()
epoch_bce_loss = 0
epoch_dice_loss = 0
epoch_loss = 0
for inputs, labels in train_loader:
optimizer.zero_grad()
if is_cuda:
inputs = inputs.cuda()
labels = labels.cuda()
outputs = sigmoid(net(inputs))
reporter.feed(outputs, labels)
bce_crit.weight = get_weight_vector(labels, relative_weight, is_cuda)
loss = criterion(outputs, labels)
epoch_bce_loss += last_bce_loss
epoch_dice_loss += last_dice_loss
epoch_loss += loss.item()
loss.backward()
optimizer.step()
scheduler.step()
del inputs, labels, outputs, loss
avg_bce_loss = epoch_bce_loss / float(len(train_loader))
avg_dice_loss = epoch_dice_loss / float(len(train_loader))
avg_loss = epoch_loss / float(len(train_loader))
avg_acc = reporter.accuracy()
print("\n[Train] Epoch({}) Avg BCE Loss: {:.4f} Avg Dice Loss: {:.4f} \
Avg Loss: {:.4f}".format(epoch, avg_bce_loss, avg_dice_loss, avg_loss))
print(reporter)
print(reporter.stats())
return avg_loss, avg_acc
offset_mean=offset_mean,
offset_scale=offset_scale)
session.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=0)
summarizer = tf.summary.FileWriter(path.join(MDLDEF, 'train',
args.model),
graph=session.graph)
model.save(saver, mdldir, 0, meta=True)
epochs = 0
while True:
epochs += 1
session.run(train_iterator.initializer)
train_report = Report(epochs, mode='t')
while True:
try:
stroke, loss, sse, _ = model.predict(
*session.run(train_example),
train=True,
learning_rate=args.learning_rate,
clip_threshold=args.clip_threshold,
keep_prob=args.keep_prob)
train_report.report(loss, sse)
if train_report.iterations % 10 == 0:
model.save(saver, mdldir, epochs)
except tf.errors.OutOfRangeError:
break
print('', file=sys.stderr)
# (to avoid long excel loads)
if len(sys.argv) == 2 and sys.argv[1] == 'frompickle':
with open('arr.pickle', 'rb') as outp:
arr = pickle.load(outp)
# otherwise open excel file and pickle it for future use
else:
file = xlrd.open_workbook('input.xlsx')
sheet = file.sheet_by_name('Final')
arr = []
for row in range(1, sheet.nrows-1):
arr.append(sheet.row_values(row))
with open('arr.pickle', 'wb') as inp:
pickle.dump(arr, inp)
r = Report()
for row in arr[0:3]:
aquiror_name = row[3]
target_name = row[2]
deal_date = row[13]
deal_date = xlrd.xldate_as_tuple(deal_date, 0)[0:5]
deal_date = datetime(*deal_date)
# get aquiror and target tickers
aquiror_ticker, msg1 = find_single_ticker(aquiror_name)
target_ticker, msg2 = find_single_ticker(target_name)
#print('\n\n'+ msg1 + '\n' + msg2)
print(aquiror_ticker, target_ticker, deal_date)
def inference(target_dir):
volume_size = args.size*3 if len(args.size) == 1 else args.size
dataloader = DataLoader(MRIDataset(target_dir,
volume_size,
sampling_mode='center',
deterministic=True),
batch_size=1,
num_workers=4)
input_paths = dataloader.dataset.inputs
label_paths = dataloader.dataset.labels
reporter = Report()
preds_list = list()
labels_list = list()
sum_hd = 0
sum_sd = 0
num_voxels_hd = list()
for i, (inputs, labels) in enumerate(dataloader):
inputs = inputs.cuda()
preds = sigmoid(net(inputs.detach()).detach())
full_labels = dataloader.dataset._load_full_label(label_paths[i])
full_preds = dataloader.dataset._project_full_label(input_paths[i],
preds.cpu())
preds = full_preds
labels = full_labels
reporter.feed(preds, labels)
temp_reporter = Report()
temp_reporter.feed(preds, labels)
sum_hd += temp_reporter.hard_dice()
sum_sd += temp_reporter.soft_dice()
num_voxels_hd.append(
(labels.view(-1).sum().item(), temp_reporter.hard_dice()))
del temp_reporter
preds_list.append(preds.cpu())
labels_list.append(labels.cpu())
del inputs, labels, preds
print("Micro Averaged Dice {}, {}".format(sum_hd / len(dataloader),
sum_sd / len(dataloader)))
if len(args.dump_name):
# dump preds for visualization
pickle.dump([input_paths, preds_list, labels_list],
open('preds_dump_{}.pickle'.format(args.dump_name), 'wb'))
print(reporter)
print(reporter.stats())
preds = torch.stack(preds_list).view(-1).numpy()
labels = torch.stack(labels_list).view(-1).numpy().astype(int)
print(num_voxels_hd)
return preds, labels
except FileNotFoundError:
print('File {} not found.'.format(file))
quit()
try:
sheet = file.sheet_by_name('Competitors list')
except:
print('Sheet "Competitors list" not found.')
quit()
arr = []
for row in range(1, sheet.nrows - 1):
arr.append(sheet.row_values(row))
with open('arr.pickle', 'wb') as inp:
pickle.dump(arr, inp)
r = Report()
# remember first deal number
deal_num = int(arr[0][0])
# iterate over data array
for row in arr:
# check current deal number
current_deal_num = int(row[0])
if current_deal_num != deal_num:
r.start_new_row()
deal_num = current_deal_num
# get ticker value
ticker = row[2]
# get part before dot and strip spaces