def __init__(self, models_to_load=[], device='cpu'):
super(FullModelInterface, self).__init__()
self.models = models_to_load
self.dps = {
k:DefaultProcessor(
model_info[k][0],
Hierarchy.from_dict(read_pickle(os.path.join(model_info[k][1], 'hierarchy.pkl')))\
if model_info[k][1] is not None else self.hierarchy,
model_file=os.path.join(model_info[k][1], 'model_state.tpkl') if model_info[k][1] is not None else None,
device=device,
cluster=True)
for k in self.models}
self.trained_queries = {k:get_queries(os.path.join(model_info[k][1], 'used_targets.txt'))
if model_info[k][1] is not None else list(self.hierarchy.descriptions.keys())
for k in self.models}
def main():
img_files = glob.glob("oxbuild-images/*.jpg")
# img_names = ['all_souls_000000', ...]
# query_indices = [11, 21, ...]
img_names = utils.get_images_names(img_files)
query_indices = utils.get_queries(img_names)
files_for_codebook = img_files
for index in query_indices:
del files_for_codebook[index]
del img_names[index]
img_names = utils.get_images_names(img_files)
land_marks = [
'all_souls', 'ashmolean', 'balliol', 'bodleian', 'christ_church',
'cornmarket', 'hertford', 'keble', 'magdalen', 'pitt_rivers',
'radcliffe_camera'
]
#64
print("leyendo codebook...")
codebook = np.loadtxt("clusters64.csv", delimiter=",")
print("leyendo vlad matrix...")
vlad = np.loadtxt("vlad64.csv", delimiter=",")
print("listo")
print("vlad matrix shape: ")
print(vlad.shape)
test = Test(vlad, codebook, img_names, "euclidean")
precisions = []
for lm in land_marks:
for i in range(5):
index = str(i + 1)
precision = test.do_query(lm + "_" + index)
precisions.append(precision)
print("64 euclidean map = "),
print(np.average(precisions))
test = Test(vlad, codebook, img_names, "hellinger")
precisions = []
for lm in land_marks:
for i in range(5):
index = str(i + 1)
precision = test.do_query(lm + "_" + index)
precisions.append(precision)
print("64 hellinger map = "),
print(np.average(precisions))
def main():
img_files = glob.glob("oxbuild-images/*.jpg")
# img_names = ['all_souls_000000', ...]
# query_indices = [11, 21, ...]
img_names = utils.get_images_names(img_files)
query_indices = utils.get_queries(img_names)
files_for_codebook = img_files
for index in query_indices:
del files_for_codebook[index]
del img_names[index]
img_names = utils.get_images_names(img_files)
land_marks = ['all_souls','ashmolean','balliol','bodleian','christ_church','cornmarket','hertford','keble','magdalen','pitt_rivers','radcliffe_camera']
#64
print("leyendo codebook...")
codebook = np.loadtxt("clusters64.csv", delimiter=",")
print("leyendo vlad matrix...")
vlad = np.loadtxt("vlad64.csv", delimiter=",")
print("listo")
print("vlad matrix shape: ")
print(vlad.shape)
test = Test(vlad,codebook,img_names,"euclidean")
precisions = []
for lm in land_marks:
for i in range(5):
index = str(i+1)
precision = test.do_query(lm + "_" + index)
precisions.append(precision)
print("64 euclidean map = "),
print(np.average(precisions))
test = Test(vlad,codebook,img_names,"hellinger")
precisions = []
for lm in land_marks:
for i in range(5):
index = str(i+1)
precision = test.do_query(lm + "_" + index)
precisions.append(precision)
print("64 hellinger map = "),
print(np.average(precisions))
print("Reading training data:")
print("[First]:\nRead label files to relations...")
relations, relation_labeler = read_lablers_to_relations(
config_data["labels"])
print("[Second]:\nSet relations as train instances...")
print("Reading data index ...")
index = pyndri.Index(config_data["index"])
token2id, _, _ = index.get_dictionary()
externalDocId = {}
for doc_id in range(index.document_base(),
index.maximum_document()): # type: int
extD_id, _ = index.document(doc_id)
externalDocId[extD_id] = doc_id
train_queries = get_queries(config_data["train_queries"])
print("x_train preparation...")
# the model needs list of 3 input arrays :
v_q_words = []
v_d_words = []
v_rel_labels = []
# print(train_queries)
print(list(relations)[0])
for relation in tqdm(relations):
# get q_word_ids from index
q_words = [
token2id[qi] if qi in token2id else 0
for qi in train_queries[relation[0]].strip().split()
def main():
# img_files = ['oxbuild-images/all_souls_000000.jpg', ...]
img_files = glob.glob("oxbuild-images/*.jpg")
# img_names = ['all_souls_000000', ...]
img_names = utils.get_images_names(img_files)
# query_indices = [11, 21, ...]
query_indices = utils.get_queries(img_names)
files_for_codebook = img_files
for index in query_indices:
del files_for_codebook[index]
del img_names[index]
# Training
#---------------------------------------------------------------------------
# Extracting descriptors
#
# start = time()
# descriptors_count = train.calculate_descriptors(files_for_codebook)
# end = time()
# elapsed_time = utils.humanize_time(end - start)
# print("Elapsed time getting the descriptors {0}.".format(elapsed_time))
#
# #Get the sample of 100k descriptors
# start = time()
# sample = train.get_sample()
# end = time()
# elapsed_time = utils.humanize_time(end - start)
# print("Elapsed time getting the sample {0}.".format(elapsed_time))
#
# # Clustering
#
# k = 64
# start = time()
# clusters = train.get_clusters(k, sample)
# end = time()
# elapsed_time = utils.humanize_time(end - start)
# print("Elapsed time clustering for k={0} {1}".format(k, elapsed_time))
# np.savetxt("clusters64.csv", clusters, delimiter=",")
#
# k = 128
# start = time()
# clusters = train.get_clusters(k, sample)
# end = time()
# elapsed_time = utils.humanize_time(end - start)
# print("Elapsed time clustering for k={0} {1}".format(k, elapsed_time))
# np.savetxt("clusters128.csv", clusters, delimiter=",")
#
# k = 256
# start = time()
# clusters = train.get_clusters(k, sample)
# end = time()
# elapsed_time = utils.humanize_time(end - start)
# print("Elapsed time clustering for k={0} {1}".format(k, elapsed_time))
# np.savetxt("clusters256.csv", clusters, delimiter=",")
# Vlad
#k = 64
# clusters = np.loadtxt("clusters64.csv", delimiter=",")
# vlad = Vlad(clusters, 64)
# vlad_matrix = None
# i = 0
# start = time()
# for image_path in files_for_codebook:
# print(str(i) + "/" + str(len(files_for_codebook)))
# descriptors = train.get_descriptor_from_image_path(image_path)
#
# vlad_imagen = vlad.get_image_vlad(descriptors)
# if vlad_matrix is None:
# vlad_matrix = vlad_imagen
# else:
# vlad_matrix = np.vstack((vlad_matrix,vlad_imagen))
# i += 1
# end = time()
# elapsed_time = utils.humanize_time(end - start)
# print("Elapsed time vlad {0}.".format(elapsed_time))
# np.savetxt("vlad64.csv",vlad_matrix,delimiter=",")
#
# #k = 256
# clusters = np.loadtxt("clusters256.csv", delimiter=",")
# vlad = Vlad(clusters, 64)
# vlad_matrix = None
# i = 0
# start = time()
# for image_path in files_for_codebook:
# print(str(i) + "/" + str(len(files_for_codebook)))
# descriptors = train.get_descriptor_from_image_path(image_path)
#
# vlad_imagen = vlad.get_image_vlad(descriptors)
# if vlad_matrix is None:
# vlad_matrix = vlad_imagen
# else:
# vlad_matrix = np.vstack((vlad_matrix,vlad_imagen))
# i += 1
# end = time()
# elapsed_time = utils.humanize_time(end - start)
# print("Elapsed time vlad {0}.".format(elapsed_time))
# np.savetxt("vlad256.csv",vlad_matrix,delimiter=",")
# Testing
#---------------------------------------------------------------------------
# Hacer queries
#64
query_name = "hertford_4"
print("leyendo codebook de 64 clusters...")
codebook = np.loadtxt("clusters64.csv", delimiter=",")
print("leyendo vlad matrix...")
vlad = np.loadtxt("vlad64.csv", delimiter=",")
print("listo")
test = Test(vlad, codebook, img_names, "euclidean")
ranking = test.do_query_and_get_ranking(query_name)
print("ranking: ")
print(ranking)
file_name = "ranking_" + query_name + "_64"
list_to_file(ranking.tolist(), file_name)
#128
print("leyendo codebook de 128 clusters...")
codebook = np.loadtxt("clusters128.csv", delimiter=",")
print("leyendo vlad matrix...")
vlad = np.loadtxt("vlad128.csv", delimiter=",")
print("listo")
test = Test(vlad, codebook, img_names, "euclidean")
ranking = test.do_query_and_get_ranking(query_name)
file_name = "ranking_" + query_name + "_128"
list_to_file(ranking.tolist(), file_name)
#255
print("leyendo codebook de 256 clusters...")
codebook = np.loadtxt("clusters256.csv", delimiter=",")
print("leyendo vlad matrix...")
vlad = np.loadtxt("vlad256.csv", delimiter=",")
print("listo")
test = Test(vlad, codebook, img_names, "euclidean")
ranking = test.do_query_and_get_ranking(query_name)
file_name = "ranking_" + query_name + "_256"
list_to_file(ranking.tolist(), file_name)
from sklearn import base
from google.cloud import bigquery
import utils as u
queries = u.get_queries('queries')
bq_client = bigquery.Client()
class PostCodeExtractor(base.TransformerMixin, base.BaseEstimator):
'''Extract the post code from the address column
and drop the address column'''
def __inint__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X):
X_c = X.copy()
X_c['post_code'] = X_c['address'].str.extract(r'.*(\d{4}).*',
expand=False)
return X_c
class PostCodeEnricher(base.TransformerMixin, base.BaseEstimator):
'''Extract the post code data from bigquery,
merge it with the training data.'''
def __init__(self):
pass
def fit(self, X, y=None):
import glob
import utils
img_files = glob.glob("oxbuild-images/*.jpg")
img_names = utils.get_images_names(img_files)
query_indices = utils.get_queries(img_names)
files_for_codebook = img_files
for index in query_indices:
del files_for_codebook[index]
print len(files_for_codebook)
import glob
import utils
img_files = glob.glob("oxbuild-images/*.jpg")
img_names = utils.get_images_names(img_files)
query_indices = utils.get_queries(img_names)
files_for_codebook = img_files
for index in query_indices:
del files_for_codebook[index]
print len(files_for_codebook)
import os
from pytt.utils import read_pickle
from utils import get_queries
dataset = '/home/jered/Documents/data/mimic-iii-clinical-database-1.4/preprocessed/reports_and_codes_expanded'
# need to add support for ancestors
# code_graph_file =
# ancestors = True
rebalanced = True
counts_file = os.path.join(dataset, 'counts.pkl')
used_targets_file = os.path.join(dataset, 'used_targets.txt')
used_targets = get_queries(used_targets_file)
counts = read_pickle(counts_file)
micro_counts = [[], [], []]
macro_scores = [[], [], []]
for k, v in counts.items():
if k not in used_targets: continue
total = v[0] + v[1]
true_positives = v[1] / 2 if rebalanced else v[1] * v[1] / total
micro_counts[0] += [true_positives]
positives = total / 2 if rebalanced else v[1]
micro_counts[1] += [positives]
relevants = v[1]
micro_counts[2] += [relevants]
if positives != 0:
p = true_positives / positives
macro_scores[0] += [p]
if relevants != 0: