def create_visual_keys(project_directory, config, location_keys_map):
raw_file_directory = project_directory + config.get('RAW-DATA','visual_descriptors')
preprocessed_file_directory = project_directory + config.get('PREPROCESSED-DATA','visual_directory')
files = helpers.load_directory(raw_file_directory)
r = redis.Redis(host=config.get('CACHE','host'), port=int(config.get('CACHE','port')))
for filepath in files:
location_name = helpers.tokenize(helpers.get_file_name(filepath), " ")[0]
model = helpers.tokenize(helpers.get_file_name(filepath), " ")[1]
location_id = location_keys_map[location_name]
file = helpers.load_text_file(filepath)
index = 0
for line in file:
image_id = helpers.tokenize(line, ",")[0]
key = location_id + "_" + model + "_" + image_id
value = str(index)
r.set(key, value)
key = location_id + "_" + model + "_" + str(index)
value = str(image_id)
r.set(key, value)
index = index + 1
def generate_visual_descriptors_map(self):
visual_descriptors = {}
visual_descriptors_directory = helpers.load_directory(
self.project_directory +
self.config.get("RAW-DATA", "visual_descriptors"))
for file in visual_descriptors_directory:
location_name = helpers.get_file_name(file).split()[0]
location_id = self.cache.hgetall('location_map')[
location_name.encode('utf-8')].decode('utf-8')
model = helpers.get_file_name(file).split()[1]
if location_id not in visual_descriptors:
visual_descriptors[location_id] = {}
if model not in visual_descriptors[location_id]:
visual_descriptors[location_id][model] = []
file = helpers.load_text_file(file)
lines = []
for line in file:
line = helpers.tokenize(line, ",")
line = helpers.to_float(line[1:len(line) - 1])
lines.append(line)
visual_descriptors[location_id][model] = lines
return visual_descriptors
def reduce_lda(self, model, k):
files = helpers.load_directory(
self.project_directory +
self.config.get("PREPROCESSED-DATA", "visual_directory"))
data = {}
for filepath in files:
filename = helpers.get_file_name(filepath)
file_model = filename.split()[1]
location_name = filename.split()[0]
location_id = self.cache.hgetall('location_map')[
location_name.encode('utf-8')].decode('utf-8')
if model != file_model:
continue
file = (scipy.sparse.load_npz(filepath)).todense()
if location_id not in data:
data[location_id] = {}
if file_model not in data[location_id]:
data[location_id][file_model] = []
data[location_id][file_model] = file
return data
def generate_maps(self):
maps = {}
maps['user'] = {}
maps['image'] = {}
maps['location'] = {}
image_to_user = {}
user_to_image = {}
image_to_location = {}
location_to_image = {}
location_to_user = {}
user_to_location = {}
xml_directory = helpers.load_directory(self.project_directory + self.config.get("RAW-DATA", "xmls"))
for file in xml_directory:
filename = helpers.get_file_name(file)
location_id = self.cache.hgetall('location_map')[filename.encode('utf-8')].decode('utf-8')
mydoc = minidom.parse(file)
photos = mydoc.getElementsByTagName('photo')
for elem in photos:
image_to_user[elem.attributes['id'].value] = []
image_to_location[elem.attributes['id'].value] = []
user_to_location[elem.attributes['userid'].value] = []
user_to_image[elem.attributes['userid'].value] = []
location_to_image[location_id] = []
location_to_user[location_id] = []
for elem in photos:
image_to_user[elem.attributes['id'].value].append(elem.attributes['userid'].value)
image_to_location[elem.attributes['id'].value].append(location_id)
if location_id not in user_to_location[elem.attributes['userid'].value]:
user_to_location[elem.attributes['userid'].value].append(location_id)
user_to_image[elem.attributes['userid'].value].append(elem.attributes['id'].value)
location_to_image[location_id].append(elem.attributes['id'].value )
if elem.attributes['userid'].value not in location_to_user[location_id]:
location_to_user[location_id].append(elem.attributes['userid'].value )
maps['user']['location'] = user_to_location
maps['user']['image'] = user_to_image
maps['image']['user'] = image_to_user
maps['image']['location'] = image_to_location
maps['location']['user'] = location_to_user
maps['location']['image'] = location_to_image
return maps
def normalize_visual_descriptors(project_directory, config, location_keys_map):
raw_file_directory = project_directory + config.get('RAW-DATA','visual_descriptors')
preprocessed_file_directory = project_directory + config.get('PREPROCESSED-DATA','visual_directory')
files = helpers.load_directory(raw_file_directory)
for filepath in files:
temp_visual_descriptor = []
file = helpers.load_text_file(filepath)
filename = helpers.get_file_name(filepath)
model = filename.split()[1]
print("generating for:",filename)
if "CM" in model:
cm(preprocessed_file_directory, filename, file)
continue
elif "CN" in model:
cn(preprocessed_file_directory, filename, file)
continue
elif "HOG" in model:
hog(preprocessed_file_directory, filename, file)
continue
elif "LBP3x3" in model:
lbp(preprocessed_file_directory, filename, file, 1000)
elif "LBP" in model:
lbp(preprocessed_file_directory, filename, file, 100)
continue
elif "CSD" in model:
csd(preprocessed_file_directory, filename, file)
continue
elif "GLRLM" in model:
glrlm(preprocessed_file_directory, filename, file)
continue