def extract():
print("{} Descriptor Extractor: [{}]".format(memory_usage_psutil(), color(FEATURE_DETECTOR_ALG.upper(), fontcolor=FONT_COLOR, bold=True)))
# Pega lista de imagens
images_list = make_image_list(DATASET_PATH)
print("{} Images: [{}]".format(memory_usage_psutil(), color(len(images_list), fontcolor=FONT_COLOR, bold=True)))
# Connect to DATABASE
print("{} Connecting to the MySQL database...".format(memory_usage_psutil()), end=' ')
try:
DB_CONN = Connect(DATABASE_SETTINGS["host"], DATABASE_SETTINGS["user"], DATABASE_SETTINGS["password"], DATABASE_SETTINGS["database"])
DB_CURSOR = DB_CONN.cursor()
except Exception as e:
raise e
cprint("OK\n", fontcolor=FONT_COLOR, bold=True)
# Tenta apagar tabela antiga de imagens
print("{} Dropping the existing image table...".format(memory_usage_psutil()), end=' ')
try:
DB_CURSOR.execute("DROP TABLE {};".format(DATABASE_SETTINGS["table"]))
except Exception as e:
pass
cprint("OK\n", fontcolor=FONT_COLOR, bold=True)
print("{} Creating image table...".format(memory_usage_psutil()), end=' ')
try:
DB_CURSOR.execute('''
CREATE TABLE {}
(
_id INT PRIMARY KEY,
_path VARCHAR(255) NOT NULL UNIQUE,
_descriptors JSON
);'''.format(DATABASE_SETTINGS["table"]))
except Exception as e:
raise e
cprint("OK\n", fontcolor=FONT_COLOR, bold=True)
# Instancia classe de extração de descritores
des_extract = DescriptorExtractor()
print("{} Extracting Descriptors".format(memory_usage_psutil()))
for i in tqdm(iterable=range(0, len(images_list)), ncols=90, unit='images'):
# Extract image descriptors
des = des_extract.extract(img_name=images_list[i], feature_detector_alg=FEATURE_DETECTOR_ALG)
# Insert a img to images collection
if des:
if FEATURE_DETECTOR_ALG in BINARY_FEATURE_DETECTOR_ALG_LIST:
try:
DB_CURSOR.execute('''
INSERT INTO {}
VALUES ({}, '{}', '{}')
;
'''.format(
DATABASE_SETTINGS["table"], # Nome da tabela
int(images_list[i].split('/')[-1]), #ID da imagem em INT
images_list[i], # Caminho até a imagem
dumps([to_bit_string(elem) for elem in des]) # bit-string-descriptors em JSON
)
)
except Exception as e:
raise e
elif FEATURE_DETECTOR_ALG in NUMERICAL_FEATURE_DETECTOR_ALG_LIST:
try:
DB_CURSOR.execute('''
INSERT INTO {}
VALUES ({}, '{}', '{}')
;
'''.format(
DATABASE_SETTINGS["table"], # Nome da tabela
i, #int(images_list[i].split('/')[-1]), #ID da imagem em INT
images_list[i], # Caminho até a imagem
dumps([elem for elem in des]) # array-descriptors em JSON
)
)
except Exception as e:
raise e
else:
raise Exception("Algoritmo não foi adicionado à lista de algoritmos válidos!")
des = None
del des
print("{} Descriptors: [{}]".format(memory_usage_psutil(), color(des_extract.size, fontcolor=FONT_COLOR, bold=True)))
print("{} Descriptors per image: [{}]".format(memory_usage_psutil(), color(des_extract.size/len(images_list), fontcolor=FONT_COLOR, bold=True)))
images_list = None
del images_list, des_extract
# Confirma alterações no BD
print("{} Writing data in the database...".format(memory_usage_psutil()), end=' ')
try:
DB_CONN.commit()
except Exception as e:
raise e
cprint("OK\n", fontcolor=FONT_COLOR, bold=True)
# Fecha conexão
print("{} Closing the connection to the database...".format(memory_usage_psutil()), end=' ')
try:
DB_CONN.close()
except Exception as e:
raise e
cprint("OK\n", fontcolor=FONT_COLOR, bold=True)
def image_search(query_path, hist_comp_method=4, top_n_results=4):
global QUERY_HISTOGRAM
global MATCHED_IMAGES
QUERY_HISTOGRAM = {}
MATCHED_IMAGES = {}
# Instancia classe de extração de descritores
des_extract = DescriptorExtractor()
bit_string_vector = des_extract.extract(
img_name=query_path, feature_detector_alg=FEATURE_DETECTOR_ALG)
# Percorre lista de descritores da imagem
for bit_string_des in bit_string_vector:
# Explora recursivamente a VocabTree
explore(VOCAB_TREE.tree, bit_string_des, len(bit_string_vector))
bit_string_vector = None
del bit_string_vector, des_extract
# global MATCHED_IMAGES
# print("[{}] MATCHED_IMAGES: {}".format(len(MATCHED_IMAGES), MATCHED_IMAGES))
# Percorre lista de imagens que podem ser semelhantes
for d_image in MATCHED_IMAGES.keys():
# Percorre lista de nodos pelos quais os descritores da Query passaram
for node in QUERY_HISTOGRAM.keys():
# Testa se imagem passou pelo nodo em quastão
if d_image in VOCAB_TREE.inverted_file[node].keys():
# Se sim, adiciona qi ao histograma
MATCHED_IMAGES[d_image].append(
VOCAB_TREE.inverted_file[node][d_image])
else:
# Semão, adiciona 0 ao histograma
MATCHED_IMAGES[d_image].append(0)
# Mostra lista de imagens
# print("\n\nd_images: ")
# Percorre chaves do dicionário de imagens possivelmente semelhantes
for key in MATCHED_IMAGES.keys():
# Normaliza histograma da imagem da base de dados
MATCHED_IMAGES[key] = np.array(tuple(
MATCHED_IMAGES[key])) / np.linalg.norm(
np.array(tuple(MATCHED_IMAGES[key])), ord=1)
MATCHED_IMAGES[key] = MATCHED_IMAGES[key].astype('float32')
# Mostra histrograma
# print("\n\n[{}] {} ~> {}".format(MATCHED_IMAGES[key].size, key, MATCHED_IMAGES[key]))
# Normaliza histograma da query
q_values = np.array(tuple(QUERY_HISTOGRAM.values())) / np.linalg.norm(
np.array(tuple(QUERY_HISTOGRAM.values())), ord=1)
q_values = q_values.astype('float32')
# Mostra histograma da query
# print("\n\n[{}] {} ~> {}".format(q_values.size, "query", q_values))
# CALCULA SCORE
score = []
for _image_id in MATCHED_IMAGES.keys():
if q_values.size != MATCHED_IMAGES[_image_id].size:
raise Exception(
"Query histogram and database image histogram are different!")
# Compara histogramas
s = compare_hist(q_values, MATCHED_IMAGES[_image_id], hist_comp_method)
score.append((s, _image_id))
q_values = None
del q_values
# Ordena score
score.sort()
del score[top_n_results:]
image_list = []
# Connect to DATABASE
try:
DB_CONN = Connect(DATABASE_SETTINGS["host"], DATABASE_SETTINGS["user"],
DATABASE_SETTINGS["password"],
DATABASE_SETTINGS["database"])
DB_CURSOR = DB_CONN.cursor()
except Exception as e:
raise e
for _s, _image_id in score:
try:
DB_CURSOR.execute('''
SELECT _path
FROM {}
WHERE _id = {}
;
'''.format(DATABASE_SETTINGS["table"], _image_id))
query_image_path = DB_CURSOR.fetchall()[0][0].split('/')[-1]
image_list.append((_s, query_image_path))
except Exception as e:
raise Exception("Error to read image path from database!!")
# Fecha conexão
try:
DB_CONN.close()
except Exception as e:
raise e
return image_list
endpoint_2 = r'http://10.30.31.121/heat_log/2018-04-03_0000-cluster_nodes.log'
# 2.数据库信息
DB_USER = "******"
# DB_HOST = "10.30.30.121"
DB_HOST = "10.118.252.202"
DB_PASSWD = "123456"
DB_DBNAME = "mk"
try:
conn = Connect(host=DB_HOST, user=DB_USER, passwd=DB_PASSWD, db=DB_DBNAME)
except Exception as e:
print "连接数据库失败: " + str(e)
exit()
# 3.开始
try:
heat = HeatMain(conn, endpoint_1)
heat.fetch_log()
# print len(heat.logs)
heat.parse_log()
# print len(heat.recordss)
raw_input("解析数据完毕，确认写入数据库(Ctrl-C退出):")
heat.store_log()
except Exception as e:
print "运行失败: " + str(e)
else:
print "运行成功"
finally:
heat.statistics()
conn.close()
class Model:
def __init__(self):
self._conn = None
self._cursor = None
def connect(self, host, user, password, database):
'''
Método para conectar-se à Base de Dados
Args:
host : str
Nome do host da Base de Dados
user : str
Usuário da Base de Dados
password : str
Senha da Base de Dados
database : str
Nome do Database a ser utilizado
'''
try:
self._conn = Connect(host, user, password, database)
self._cursor = self._conn.cursor()
except Exception as e:
raise e
else:
print("Connecting to '{}' database...".format(database))
def close(self):
'''
Método para fechar a conexão com a Base de Dados
'''
try:
self._conn.close()
except Exception as e:
raise e
else:
print("Closing connection...")
def drop_table(self, table):
'''
Método para deletar uma tabela da Base de Dados
Args:
table : str
Nome da tabela da Base de Dados
'''
try:
self._cursor.execute(f"DROP TABLE {table};")
except:
pass
else:
print("Dropping '{}' table...".format(table))
def commit(self):
'''
Método para escrever os dados na Base de Dados
'''
try:
self._conn.commit()
except Exception as e:
raise e
else:
pass
def select(self, columns=[], tables=[], value=None):
'''
Método para ler dados da Base de Dados
Args:
columns : list
Lista de colunas a serem selecionadas
tables : list
Lista de tabelas
value : str
Valor a ser usado com a cláusula WHERE
Returns
n_rows : int
Quantidade de linhas retornadas
'''
columns = [columns] if not isinstance(columns, list) else columns
tables = [tables] if not isinstance(tables, list) else tables
try:
n_rows = self._cursor.execute(
'''SELECT {} FROM {} WHERE {}={};'''.format(
', '.join(map(lambda item: str(item), columns)),
', '.join(map(lambda item: str(item), tables)), columns[1],
self._add_aspas(value)))
except Exception as e:
raise e
else:
return n_rows
def select_all(self):
'''
Método para ler todos os dados da Base de Dados
Returns
self._cursor.fetchall() : tuple
Dados lidos da Base de Dados
'''
try:
ret = self._cursor.execute('''
SELECT Point.pointLAT,
Point.pointLNG,
Point.pointStreetName,
Point.pointHouseNumber,
Suburb.suburbName,
City.cityName,
Point.pointPostalCode,
State.stateUF,
Country.countryName
FROM Point
LEFT JOIN Suburb
ON Point.suburbID = Suburb.id
LEFT JOIN City
ON Suburb.cityID = City.id
LEFT JOIN State
ON City.stateID = State.id
LEFT JOIN Country
ON State.countryID = Country.id;
''')
except:
return None
else:
return self._cursor.fetchall()
def _add_aspas(self, item):
'''
Método para formatar dados: se o item é uma String,
são adicionadas aspas ('') no item.
Args:
item : str | int | float
Item a ser formatado
Returns
str(item) : str
Item formatado
'''
if isinstance(item, str):
return f"'{item.lower()}'"
return str(item)
def insert(self, table=None, columns=[], values=[]):
'''
Método para inserir dados na Base de Dados.
Se a coluna for do tipo UNIQUE (countryName, stateUF, cityName, suburbName),
é testado se o dado já existe na Base de Dados. Se já existir, o ID do dado
é retornado; se o dado não existir, o dado é adicionado à Base de Dados e
seu ID é retornado.
Args:
table : str
Tabela onde os dados serão inseridos
columns : list
Lista de colunas
values : list
Lista de valores correspondentes às colunas
Returns
_id | self._cursor.lastrowid : int
ID do dado adicionado ou do dado já existente
'''
columns = [columns] if not isinstance(columns, list) else columns
values = [values] if not isinstance(values, list) else values
# Busca o ID e o valor Único da tabela na base de dados para
# verificar se já existe (countryName, stateUF, cityName, suburbName)
n_rows = self.select(columns=['id', columns[0]],
tables=table,
value=values[0])
if n_rows > 0: # Se o valor já existe, retorna o ID
_data = self._cursor.fetchall()[0]
# Testa se os valores são iguais
_id, _value = _data
if values[0] == _value:
return _id
# Se o valor não existe, insere ele
try:
ret = self._cursor.execute(
'''INSERT INTO {} ({}) VALUES ({});'''.format(
table, ', '.join(map(lambda item: str(item), columns)),
', '.join(map(lambda item: self._add_aspas(item),
values))))
except Exception as e:
raise e
else:
return self._cursor.lastrowid # retorna o ID da nova inserção
def create_tables(self):
'''
Método para criar as tabelas
'''
try:
self._cursor.execute('''
CREATE TABLE IF NOT EXISTS Country (
id INT NOT NULL AUTO_INCREMENT,
countryName VARCHAR(50) NOT NULL UNIQUE,
PRIMARY KEY (id)
);
''')
self._cursor.execute('''
CREATE TABLE IF NOT EXISTS State (
id INT NOT NULL AUTO_INCREMENT,
stateUF VARCHAR(5) NOT NULL UNIQUE,
countryID INT NOT NULL,
PRIMARY KEY (id),
FOREIGN KEY (countryID) REFERENCES Country(id)
);
''')
self._cursor.execute('''
CREATE TABLE IF NOT EXISTS City (
id INT NOT NULL AUTO_INCREMENT,
cityName VARCHAR(50) NOT NULL UNIQUE,
stateID INT NOT NULL,
PRIMARY KEY (id),
FOREIGN KEY (stateID) REFERENCES State(id)
);
''')
self._cursor.execute('''
CREATE TABLE IF NOT EXISTS Suburb (
id INT NOT NULL AUTO_INCREMENT,
suburbName VARCHAR(100) NOT NULL UNIQUE,
cityID INT NOT NULL,
PRIMARY KEY (id),
FOREIGN KEY (cityID) REFERENCES City(id)
);
''')
self._cursor.execute('''
CREATE TABLE IF NOT EXISTS Point (
id INT NOT NULL AUTO_INCREMENT,
pointLAT FLOAT,
pointLNG FLOAT,
pointStreetName VARCHAR(100),
pointHouseNumber VARCHAR(20),
pointPostalCode VARCHAR(20),
suburbID INT(11) NOT NULL,
PRIMARY KEY (id),
FOREIGN KEY (suburbID) REFERENCES Suburb(id)
);
''')
except Exception as e:
raise e
else:
print("Creating tables...")