def view_all_salient_features(model_options, labels=None, num_examples=None, folderpath='', data=None):
imgs = []
for label in range(10):
if data is None:
imgs_label, _ = read_results(label, folderpath)[-1]
else:
imgs_label = data[label]
imgs_label = imgs_label[0:1, :, :]
salient_features = get_salient_features(model_options, imgs_label, label,
grad_sign='pos', num_keep=200, as_binary=True, mul_weights=False)
imgs.append(salient_features)
imgs = np.array(imgs)
if labels:
imgs = imgs[labels]
if num_examples:
imgs = imgs[:, :num_examples]
shape = list(imgs.shape[:2])
shape += [model_options['image_dim_size']] * 2
graph_grid(imgs, shape=shape)
def _start_session(self):
"""Starts a session (either a new one or one that has been loaded)."""
exam_data = self.exam_data
session_cfg = exam_data.session
self.save_pattern = os.path.join(self.session_dir, 'captures',
session_cfg['save-filename-pattern'])
self.answers_file = os.path.join(self.session_dir,
'eyegrade-answers.csv')
try:
self._read_student_ids()
except Exception as e:
self.interface.show_error(('The student list cannot be read: '
+ str(e)))
return
self.imageproc_options = imageproc.ExamCapture.get_default_options()
if exam_data.id_num_digits and exam_data.id_num_digits > 0:
self.imageproc_options['read-id'] = True
self.imageproc_options['id-num-digits'] = exam_data.id_num_digits
self.imageproc_options['left-to-right-numbering'] = \
exam_data.left_to_right_numbering
# Set the debug options in imageproc_options:
self._action_debug_changed()
self.imageproc_context.open_camera()
if self.imageproc_context.camera is None:
self.interface.show_error(('No camera found. Connect a camera and '
'try to start the session again.'))
return
if not os.path.isfile(self.answers_file):
self.image_id = 1
else:
results = utils.read_results(self.answers_file,
allow_question_mark=True)
self.image_id = 1 + max([int(r['seq-num']) for r in results])
self._start_search_mode()
def proc_all_results(num_examples,
save_filename=None,
res=None,
folderpath=''):
# TODO: modify this to use graph_grid
num_results = 10
fig = plt.figure(figsize=(12, 12))
axs = ImageGrid(fig,
rect=111,
nrows_ncols=(num_results, num_examples),
axes_pad=0.35,
label_mode='L')
for label in range(num_results):
# TODO: don't hardcode timestep
if res is None:
imgs, softmaxs = read_results(label, folderpath)[-1]
else:
imgs, softmaxs = res[label]
axs_label = axs[label * num_examples:(label + 1) * num_examples]
proc_results_label(label, imgs, softmaxs, axs_label)
plt.tight_layout()
if save_filename:
fig.savefig(save_filename, bbox_inches='tight')
else:
plt.show()
def proc_all_results(num_examples, save_filename=None, res=None, folderpath=''):
# TODO: modify this to use graph_grid
num_results = 10
fig = plt.figure(figsize=(12, 12))
axs = ImageGrid(fig, rect=111, nrows_ncols=(num_results, num_examples),
axes_pad=0.35, label_mode='L')
for label in range(num_results):
# TODO: don't hardcode timestep
if res is None:
imgs, softmaxs = read_results(label, folderpath)[-1]
else:
imgs, softmaxs = res[label]
axs_label = axs[label * num_examples: (label + 1) * num_examples]
proc_results_label(label, imgs, softmaxs, axs_label)
plt.tight_layout()
if save_filename:
fig.savefig(save_filename, bbox_inches='tight')
else:
plt.show()
def modify_results(results_filename, exam_config_filename,
output_filename, invalidate, set_correct):
config = utils.read_config()
results = utils.read_results(results_filename)
exam_data = utils.ExamConfig(exam_config_filename)
for result in results:
modify(result, exam_data, invalidate, set_correct)
utils.write_results(results, output_filename, config['csv-dialect'])
def eval_model(dataset_file, model_filename, results_filename):
"""
Docstring for eval_model.
"""
model = None
# Load your best model.
if model_filename:
model_filename = Path(model_filename)
model = torch.load(model_filename)
if CUDA:
model = model.cuda()
print("\nLoading model from", model_filename.absolute())
if model:
N_SUBJ = 160
data = utils.read_memfile(dataset_file,
shape=(N_SUBJ, 3750),
dtype='float32')
results = utils.read_results(results_filename)
# y is generated as we do not have predictions here.
fake_y = np.random.randint(1, 32, size=((N_SUBJ, 4)))
fake_y[0, -1] = 32
data = {'X': data, 'y': fake_y}
data = utils.Data(precomputed=data, augmentation=False)
# Overwrite ymap in Data with one computed using real data.
data.ymap = results['ymap']
dataloader = torch.utils.data.DataLoader(data,
batch_size=64,
num_workers=2)
# Generate predictions with model.
results = experiments.evalu_loop(model, dataloader, return_preds=True)
y_pred = results['preds']
# Convert ID column back to original format.
ids = y_pred[:, -1]
ids = data.ymap.inverse_transform(ids.astype(np.int))
y_pred[:, -1] = ids
else:
print("\nYou did not specify a model, generating dummy data instead!")
c = 32
n = 10
y_pred = np.concatenate(
[np.random.rand(n, 3),
np.random.randint(0, c, (n, 1))], axis=1).astype(np.float32)
return (y_pred)
def analyze(results_filenames, exam_cfg_filename):
exam_data = utils.ExamConfig(exam_cfg_filename)
results = []
for results_file in results_filenames:
results.extend(utils.read_results(results_file, exam_data.permutations))
stats_q = stats_by_question(results, exam_data.num_questions)
print_stats_by_question(stats_q)
print
stats_m = stats_by_model(results, exam_data.num_questions,
exam_data.models)
print_stats_by_model(stats_m)
plot_stats_by_question(stats_q)
return results
def augment_data(data, folderpath=''):
images = data.train.images
labels = data.train.labels
images = images.reshape((-1, 28, 28, 1))
imgs = [x for i in range(10) for x in read_results(i, folderpath)[-1][0]]
imgs = np.array(imgs)[:, :, :, None]
lbls = np.empty((imgs.shape[0], 10))
lbls.fill(1/10)
images = np.concatenate((imgs, images))
labels = np.concatenate((lbls, labels))
images, labels = _shuffle_images_labels(images, labels)
new_train = input_data.DataSet(images, labels)
data.train = new_train
def view_all_salient_features(model_options,
labels=None,
num_examples=None,
folderpath='',
data=None):
imgs = []
for label in range(10):
if data is None:
imgs_label, _ = read_results(label, folderpath)[-1]
else:
imgs_label = data[label]
imgs_label = imgs_label[0:1, :, :]
salient_features = get_salient_features(model_options,
imgs_label,
label,
grad_sign='pos',
num_keep=200,
as_binary=True,
mul_weights=False)
imgs.append(salient_features)
imgs = np.array(imgs)
if labels:
imgs = imgs[labels]
if num_examples:
imgs = imgs[:, :num_examples]
shape = list(imgs.shape[:2])
shape += [model_options['image_dim_size']] * 2
graph_grid(imgs, shape=shape)
"MDA", "PWT", "DDC", "POT", "MST.UN", "AYA", "DRG.UN", "PHY.U",
"VRX", "SLW"
]:
continue
stock_yield = get_3_month_yield(stock, date)
# Filter out error in data (we suppose no yield of over x100)
if not stock_yield > 100:
yields.append(stock_yield)
print(date, stocks, yields)
return yields
def average_yields(stocks):
total_yield = {}
for date in stocks:
yields = get_stocks_yields(stocks[date], date)
average_yield = utils.list_average(yields)
total_yield[date] = average_yield
return total_yield
results = utils.read_results("./res.txt")
stocks_yield = average_yields(results)
index = yf.Ticker("^GSPTSE")
his = index.history(start="2009-12-01", end="2020-03-01")
index_yield = utils.df_to_yield_dic(his)
utils.plot_yields(stocks_yield, index_yield)
})
return output
def to_plotly_json(results, years, metric, tags, fname):
plotly_obj = plotlify_results(results, years, metric, tags)
with codecs.open(fname, "w+", "utf-8") as f:
json.dump(plotly_obj, f)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="evaluation")
parser.add_argument("files", nargs='+')
parser.add_argument("-t", "--tags", default="most_freq")
parser.add_argument("-o", "--output", required=True)
parser.add_argument("-m", "--metric", required=True)
args = parser.parse_args()
results, years = read_results(args.files)
if args.tags == "most_freq":
tags = [u'MD', u'VAN', u'VBP', u'Q', u'C', u'VB', u'NS', u'VBD', u'FW', u'PRO$',
u'NPR', u'ADV', u'.', u'ADJ', u',', u'CONJ', u'D', u'PRO', u'P', u'N']
else:
tags = intersection(*[r[0] for r in results])
metrics = {"f1": f1, "precision": precision, "recall": recall}
to_plotly_json(results, years, metrics[args.metric], tags, args.output)
# from penn_data import pos_from_range
# counts = dict()
# test = (1400, 1500, 2000)
# test_counts = counts_per_tag(pos_from_range(*test))
# for start in range(1400, 1850, 50):
# train = (start, start + 100, 10000)