def getWebResultsForQuery(self): searcher = Searcher(self.query, self.rsz, log_level=self.log_level) result = searcher.search() resultQueue = [] resultsDict = dict() if result: for i in range(len(result)): resultQueue.append(result[i]['url']) sentimentOutput = sentiment.SentimentAnalysis() sentimentScores = [] sentimentTypes = [] totalScore = 0 for i in range(len(resultQueue)): query = resultQueue[i] response = sentimentOutput.makeRequest(query) sentimentResult = sentimentOutput.parseResponse(response) if sentimentResult == None: continue sentimentResultType = sentimentResult["type"] sentimentResultScore = sentimentResult["score"] totalScore += float(sentimentResultScore) typeScoreList = [] typeScoreList.append(sentimentResultType) typeScoreList.append(sentimentResultScore) resultsDict[query] = typeScoreList resultsDict["averageScore"] = totalScore/self.rsz return resultsDict
class Solver:
def __init__(self, name, heuristic, Open, Closed):
self.name = name
self.heuristic = heuristic
self.Open = Open
self.Closed = Closed
self.openset = None
self.closedset = None
self.board = None
self.solution = None
self.path = None
def solve(self, board, root, goal):
self.board = board
self.openset = self.Open()
self.closedset = self.Closed()
self.solution = Searcher(self.openset, self.closedset, root, goal, self.heuristic)
if self.solution.search():
trace = self.solution.trace(self.solution.Goal)
self.path = trace[0]
print self.name + ' solved ' + board + ' with cost', trace[-1]
else:
print self.name + ' could not solve ' + board
def visualize(self, world, steps, color):
for pos in steps:
world.drawDot(pos, color)
#world.dump()
def save(self):
world.image.save('solutions/' + self.board + '-' + self.name + '.png', 'PNG')
def __init__(self, company="", cache_prefix="", conf="", index_head="", info_head="", prefix_black_tableid=""):
logger.error("===================%s %s %s %s %s" %(company, cache_prefix, conf, index_head, info_head))
Searcher.__init__(self,company=company, cache_prefix=cache_prefix)
self.indexer = Xywy_Indexer(conf, index_head)
self.info = Xywy_Info(conf, info_head)
self.company = company
self.blacklist = Xywy_Blacklist(prefix_black_tableid)
def main():
with open("15/input.txt", encoding="UTF-8") as file:
content = file.read()
matrix = get_matrix(content)
searcher = Searcher(matrix)
path = searcher.get_shortest_path()
print(path)
def doSearch(tree):
""" Searches the tree for the search phrase requested from the user """
input = prompt('Give search phrase', type='str')
if not input:
print 'Cannot give an empty search phrase!'
return
searcher = Searcher(tree, input)
print 'Searching...'
searcher.search(printPretty=True)
def __init__(self, initialPosition, initialWidth, initialHeight, frame,
parametersNew):
#########################################
#########################################
self.initFrame = frame
self.initPos = initialPosition
self.initW = initialWidth
self.initH = initialHeight
self.KM = KalmanFilter()
self.MF = MaskingFilter()
self.KM.setStatePost(
np.array([initialPosition[0], initialPosition[1], 0.,
0.]).reshape(4, 1))
self.selectionWidth = initialWidth
self.selectionHeight = initialHeight
self.prevFrameGray = None
self.frameCounter = 0 #Shape = {tuple}: (x, 1, 2)
#Por ejemplo: [[[x1 y1]]\n\n [[x2 y2]]\n\n [[x3 y3]]]
#es decir una matriz de x filas y 1 columna, donde cada elemento
#de la unica columna es una coordenada [x y].
if self.MF.mask is not self.MF.maskingType["FILTER_OFF"]:
self.MF.calculateNewMask(
frame, frame[int(initialPosition[1] -
initialHeight / 2):int(initialPosition[1] +
initialHeight / 2),
int(initialPosition[0] -
initialWidth / 2):int(initialPosition[0] +
initialWidth / 2)])
frame = self.MF.filterFrame(frame)
self.prevFrameGray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)
self.SC = Searcher(self.initFrame, initialHeight, initialWidth,
initialPosition[0], initialPosition[1],
cv.cvtColor(frame, cv.COLOR_BGR2GRAY))
self.SC.features, self.SC.trackingError = self.SC.ST.recalculateFeatures(
self.prevFrameGray[int(initialPosition[1] -
initialHeight / 2):int(initialPosition[1] +
initialHeight / 2),
int(initialPosition[0] -
initialWidth / 2):int(initialPosition[0] +
initialWidth / 2)])
self.SC.features = self.SC.featureTranslate(
initialPosition[0] - initialWidth / 2,
initialPosition[1] - initialHeight / 2, self.SC.features)
self.SC.LK.prevFeatures = self.SC.features
#OPTIMIZACIÓN
self.bins_var = 1
self.kernel_blur_var = 1
self.mask_blur_var = 1
self.low_pth_var = 200
def run_one_query(query, num_of_docs=50, query_id=None, query_desc=None):
# run one query, return it ranked documents and the time it took to calculate it
global is_stemming
global dict_cache_path
print ('{} - RUN query {} - '.format(time.strftime("%H:%M:%S"), query))
query_id = 999 if query_id is None else query_id
start_time = datetime.now()
searcher = Searcher(query, is_stemming.get(), dict_cache_path.get(), query_desc)
ranked_documents = searcher.search_query()
elapsed_time = (datetime.now() - start_time).seconds
return ranked_documents[:num_of_docs], query, elapsed_time, query_id
def load(self):
self.indexer = Indexer(self.posting_path)
if self.to_stem:
self.indexer.to_stem = True
self.languages = self.indexer.load()
self.avg_doc_length = self.indexer.docs_avg_length
self.searcher = Searcher(self.main_path, self.posting_path,
self.indexer.terms_dict,
self.indexer.cities_dict,
self.indexer.docs_dict, self.avg_doc_length,
self.to_stem, self.with_semantics)
self.searcher.model = Word2Vec.load(self.posting_path + '//model.bin')
def search():
if request.method == 'POST':
query = cv2.imdecode(np.fromstring(request.files['img'].read(), np.uint8), cv2.IMREAD_COLOR)
features = cd.describe(query)
searcher = Searcher(INDEX)
results = searcher.search(features, 10)
res = []
for (score, resultID) in results:
res.append({"Image": str(resultID), "Score": str(score)})
context = {"images": res}
print(context)
return render_template('index.html', context=context)
res = []
context = {"images": res}
return render_template('index.html', context=context)
def runit(self):
try:
self.search = Searcher(self.gui)
self.thread1 = threading.Thread(target=self.search.orcestrator)
self.thread1.start()
except FileNotFoundError:
messagebox.showinfo('Error', 'File not Found')
def run_Scrapper_0(self, value, companyname, date):
obj_local = None
if (value == 1):
obj_local = self.obj
elif (value == 2):
obj_local = self.obj1
elif (value == 3):
obj_local = self.obj2
elif (value == 4):
obj_local = self.obj3
elif (value == 5):
obj_local = self.obj4
obj_local.Post_Factiva_Home_Request()
obj1 = Searcher(company_name=companyname,
date=date,
driver=obj_local.driver)
obj1.Select_CustomDateRange()
obj1.Enter_FromDate()
obj1.Enter_ToDate()
obj1.Enter_Company()
obj1.SubmitSearch()
obj2 = Scrapper(driver=obj_local.driver, session=obj_local.session)
obj2.MigrateSeleniumtoRequests()
count_ = 1
while True:
obj2.GetAllArticles()
obj2.Parse_Articles()
check = obj2.Navigate_NextPage(count=count_)
if (check):
break
count_ += 1
obj2.Save_ToDataFrame(companyname=companyname, date=date)
class PySearcherTestCases(unittest.TestCase):
def setUp(self):
self.reader = WordReader()
self.finder = Trie(self.reader)
self.searcher = Searcher(self.finder, '')
def tearDown(self):
self.reader = None
self.finder = None
self.searcher = None
def testRandomWord(self):
""" Tests that non-empty words are found and they are not the same """
word1 = self.searcher.randomWord()
word2 = self.searcher.randomWord()
self.assertTrue(len(word1) > 1, 'Word length too short')
self.assertTrue(len(word2) > 1, 'Word length too short')
self.assertNotEqual(word1, word2, 'Found the same word')
def testRandomWords(self):
""" Tests that a set of random words do not contain the same words """
words = self.searcher.randomWord(5)
self.assertTrue(len(set(words)) == 5, 'Did not find 5 unique words')
def testBinaryOperationsAreWorking(self):
"""
Checks that operations are not identic and that correct number of hits
is returned for every known result.
"""
self.reader.addFileName(MaterialFilePath, readNow=True)
self.finder.addFromReader()
results = []
for operation in operations:
results.append(self.searcher.search(operations[operation],
returnCount=True))
self.assertTrue(len(set(results)) == 6, #i.e. operations are not identic
'Searcher failed binary operation check')
for searchTerm in binaryOperationsSearch:
self.assertEqual(self.searcher.search(searchTerm, returnCount=True),
binaryOperationsSearch[searchTerm],
'Searcher found wrong number of hits on some search')
def solve(self, board, root, goal):
self.board = board
self.openset = self.Open()
self.closedset = self.Closed()
self.solution = Searcher(self.openset, self.closedset, root, goal, self.heuristic)
if self.solution.search():
trace = self.solution.trace(self.solution.Goal)
self.path = trace[0]
print self.name + ' solved ' + board + ' with cost', trace[-1]
else:
print self.name + ' could not solve ' + board
def controlQueriesOfFreeText(text, semantic, list_of_cities = None):
global __stem_suffix, __index_path, __results
dic_after_parse_by_addons = None
dictionary_of_queries = ReadQuery.create_dictionary_from_free_text_query(text)
dic_after_parse = Parser.parse(dictionary_of_queries, "Query")# { term : { query : tf_in_query } }
if semantic == True:
dictionary_of_queries_by_addons = semanticCare(dictionary_of_queries)
dic_after_parse_by_addons = Parser.parse(dictionary_of_queries_by_addons, "Query") # { term : { query : tf_in_query } }
searcher = Searcher(ReadFile.docs_dictionary, Indexer.main_dictionary, __avdl,__stem_suffix, __index_path, ReadFile.city_dictionary)
searcher.set_cities_filter_list(list_of_cities)
searcher.search(dic_after_parse, dic_after_parse_by_addons)
__results = searcher.get_final_result()
return __results
def controlQueriesOfFile(path_of_queries_file, semantic, list_of_cities = None):
global __stem_suffix, __index_path, __results
dictionary_of_queries_by_title, dictionary_of_queries_by_addons = ReadQuery.create_dictionary_of_file(path_of_queries_file)
dic_after_parse_by_title = Parser.parse(dictionary_of_queries_by_title, "Query") # { term : { query : tf_in_query } }
if semantic == True:
dictionary_of_queries_by_addons = semanticCare(dictionary_of_queries_by_title, dictionary_of_queries_by_addons)
dic_after_parse_by_addons = Parser.parse(dictionary_of_queries_by_addons, "Query") # { term : { query : tf_in_query } }
searcher = Searcher(ReadFile.docs_dictionary, Indexer.main_dictionary, __avdl,__stem_suffix ,__index_path, ReadFile.city_dictionary)
searcher.set_cities_filter_list(list_of_cities)
searcher.search(dic_after_parse_by_title, dic_after_parse_by_addons)
__results = searcher.get_final_result()
__results = collections.OrderedDict(sorted(__results.items()))
return __results
def initSidebar(self):
#---------------------
""" Initializes the sidebar with the tables list and configures it"""
toList = ["table", "view", "procedure",
"trigger", "scalarfunction", "tablefunction"]
statusID = self.statusbar.get_context_id("sidebar")
self.statusbar.push(statusID, "Initializing Sidebar")
scrolls = gtk.ScrolledWindow(gtk.Adjustment(), gtk.Adjustment())
scrolls.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC)
treestore = gtk.TreeStore(str)
self.sidebar = gtk.TreeView(treestore)
cell = gtk.CellRendererText()
column = gtk.TreeViewColumn("Database Objects", cell, text=0)
self.sidebar.append_column(column)
masterTable = []
for item in toList:
result = self.database.query(self.listAll+" "+item)
if "Error" not in result:
statusID = self.statusbar.get_context_id("success")
self.statusbar.push(statusID, "Success.")
parser = Parser(result)
table = parser.getTable()[:] #the [:] makes a clone
table.insert(0, [item])
masterTable.append(table)
rows = parser.getTableIterator()
iter = treestore.append(None, [item.title()])
while rows.hasNext():
treestore.append(iter, [str(rows.getNext()[0])])
else:
statusID = self.statusbar.get_context_id("error")
self.statusbar.push(statusID, result)
self.searcher = Searcher(masterTable)
self.sidebar.connect("row-activated", self.rowClicked, masterTable)
scrolls.add(self.sidebar)
self.vpanedPanel.add(scrolls)
scrolls.show()
self.sidebar.show()
self.statusbar.push(statusID, "Sidebar Loaded")
class Driver:
def __init__(self):
self.timestep_secs = 10
self.searcher = Searcher()
self.composer = Composer()
self.mailer = Mailer()
def sleep(self):
print('==================')
print('Going to sleep')
time.sleep(self.timestep_secs)
print('Have woken up')
def loop(self):
while (True):
self.sleep()
data = self.searcher.search()
data = self.composer.compose(data)
self.mailer.send(data)
def __init__(self, editor): Signal.__init__(self) from MatchIndexer import Indexer Indexer(self, editor) from MatchSelector import Selector Selector(self, editor) from MatchNavigator import Navigator Navigator(self, editor) from Reseter import Reseter Reseter(self, editor) from MatchColorer import Colorer Colorer(self, editor) from Marker import Marker Marker(self, editor) from Searcher import Searcher Searcher(self, editor) from RegexCreator import Creator Creator(self, editor) from PatternCreator import Creator Creator(self, editor) from SelectionDetector import Detector Detector(self, editor)
def __init__(self, editor):
GObject.__init__(self)
self.__init_attributes(editor)
from SearchCaseDetector import Detector
Detector(self, editor)
from MatchSelector import Selector
Selector(self, editor)
from GUI.Manager import Manager
Manager(self, editor)
from BoundaryManager import Manager
Manager(self, editor)
from PatternCreator import Creator
Creator(self, editor)
from RegexCreator import Creator
Creator(self, editor)
from CursorMarker import Marker
Marker(self, editor)
from Searcher import Searcher
Searcher(self, editor)
from MatchMapper import Mapper
Mapper(self, editor)
from Marker import Marker
Marker(self, editor)
from MatchColorer import Colorer
Colorer(self, editor)
from MatchIndexer import Indexer
Indexer(self, editor)
from SelectionMatchColorer import Colorer
Colorer(self, editor)
from MatchNavigator import Navigator
Navigator(self, editor)
from ConfigurationManager import Manager
Manager(self, editor)
from ReplaceMatchColorer import Colorer
Colorer(self, editor)
from ReplaceManager import Manager
Manager(self, editor)
return true;
} """
snippet = """
EditText myEditText = (EditText) findViewById(R.id.myEditText);
// Check if no view has focus:
View view = this.getCurrentFocus();
if (view != null) {
InputMethodManager imm = (InputMethodManager)getSystemService(
Context.INPUT_METHOD_SERVICE);
imm.hideSoftInputFromWindow(view.getWindowToken(), 0);
}
"""
index = Searcher(snippet, "/Users/Raphael/Downloads/linkgithub") ##????
sources = index.more_like_this()
#print "Sources", sources
r = Resolver(snippet, sources)
r.class_PQN_to_FQN_map()
r.method_PQN_to_FQN_map()
r.resolve_snippet_variable_class_PQN_to_FQN()
r.resolve_snippet_unkown_variable_class()
r.resolve_snippet_method_class()
r.resolve_snippet_PQN_to_FQN()
#print dict(r.class_PQN_to_FQN), dict(r.method_PQN_to_FQN)
# -*-coding:utf8;-*-
# qpy:2
# qpy:console
from Document import Document
from Searcher import Searcher
if __name__ == "__main__":
searcher = Searcher(
[
"This is a test",
"These are some longer words in a sentence and these are additional words",
"The quick, brown fox jumped over the lazy hen",
]
)
documents = searcher.get_corpus().get_documents()
for doc in documents:
print "Document %d => %s\n" % (doc.get_id(), doc.get_stems())
search_results = searcher.search("these")
for idx, result in enumerate(search_results):
print idx
print "%s\n" % result
class MainTests(TestCase):
def setUp(self):
self.search = Searcher()
def test_ContainsPartString1(self):
self.assertEqual("содержит на позиции 0", self.search.contains("abcdef", "abc").__str__())
def test_ContainsPartString2(self):
self.assertEqual("содержит на позиции 3", self.search.contains("abcdef", "def").__str__())
def test_NotContainsPartString1(self):
self.assertEqual("не содержит", self.search.contains("abcdef", "dg").__str__())
def test_NotContainsPartString2(self):
self.assertEqual("не содержит", self.search.contains("abcdef", "drf").__str__())
def test_ContainsFullString(self):
self.assertEqual("содержит на позиции 0", self.search.contains("abcdef", "abcdef").__str__())
def test_OutOfLenght(self):
with self.assertRaises(SearcherException):
self.search.contains("abcdef", "acvhfjfcsm").__str__()
def test_EmptyString(self):
with self.assertRaises(SearcherException):
self.search.contains("", "acvhfjfcsm").__str__()
def test_EmptySubString(self):
with self.assertRaises(SearcherException):
self.search.contains("ahfjkvl", "").__str__()
with opener('w')(outdir + '/all.hmm') as pressfile:
pressfile.writelines(all_lines)
call(['../binaries/hmmpress', '-f', outdir + '/all.hmm'])
#----------------------------------------------------------------------------------------
#build_hmms()
infname = 'head-simulated-seqs.csv'
#infname = 'head-data.csv' # bzgrep -m100 . /shared/silo_researcher/Matsen_F/MatsenGrp/data/bcr/output_sw/C/01-C-N_merged.tsv.bz2 | sed 's/[ \t][ \t]*/,/g'|cut -f2 -d, > head-data.csv
with opener('r')(infname) as infile:
germlines = utils.read_germlines('../../../recombinator')
reader = csv.DictReader(infile)
for inline in reader:
print 'searching'
# inline['seq'] = inline['seq'][-130:]
searcher = Searcher(inline['seq'], debug=True, n_matches_max=2)
searcher.search()
inferred_group_str = ''
true_group_str = ''
outline = {}
outline['seq'] = inline['seq']
print 'RESULT ',
for region in utils.regions:
inferred_name = searcher.get_best_match_name(region)
outline[region + '_gene'] = utils.unsanitize_name(inferred_name)
true_name = utils.sanitize_name(inline[region + '_gene'])
inferred_group_str += inferred_name
true_group_str += true_name
if inferred_name == 'none':
print ' none',
def main():
graveyard_red = (-500, -500)
graveyard_green = (400, -600)
game_params = get_gui_setup()
mode = str(game_params["game_mode"])
online_game = game_params["online_game"]
player_side = game_params["player_side"]
initial = game_params["initial_setup"]
load_game = game_params["load_new"]
if load_game == "Load Previous Game":
initial = load_previous_game()
HOST_UR1, PORT_UR1 = game_params["ur1_ip"], game_params["ur1_port"]
HOST_UR2, PORT_UR2 = game_params["ur2_ip"], game_params["ur2_port"]
manipulator = game_params["manipulator"]
hist = [Position(initial, 0, (True, True), (True, True), 0, 0)]
searcher = Searcher()
visualization = GameDisplay()
est_chessboard = EstimateChessboard()
if mode == '3':
transform_UR1 = CoordinatesTransform(manipulator)
transform_UR2 = CoordinatesTransform(manipulator)
if manipulator == "UR1":
HOST_UR2, PORT_UR2 = HOST_UR1, PORT_UR1
if manipulator == "UR2":
HOST_UR1, PORT_UR1 = HOST_UR2, PORT_UR2
else:
transform_UR1 = CoordinatesTransform("UR1")
transform_UR2 = CoordinatesTransform("UR2")
if mode == '2':
opening_moves = get_opening()
img = cv2.imread("basic_frame.png")
if online_game == 'Online':
img = warm_up_camera()
chessboard = est_chessboard.estimate_chessboard_postion(img)
visualization.display(initial, 'green')
while True:
################## Green's Move ###########################
print_pos(hist[-1])
move = None
board = hist[-1].get_board()
execute_flag = None
if mode == '0' or (mode == '1' and player_side == 'green'):
move = input_move(hist, "green")
execute_flag = True
if (mode == '1' and player_side == "red"):
move = calculate_move(searcher, hist)
execute_flag = True
if mode == '2':
if opening_moves and load_game == "Play New Game":
move = opening_moves.pop(0)
move = parse(move[0:2]), parse(move[2:4])
else:
move = calculate_move(searcher, hist)
if mode == '3' and player_side == "green":
move = vision_move(hist, player_side, chessboard)
execute_flag = False
if mode == "3" and player_side == "red":
move = calculate_move(searcher, hist)
execute_flag = True
move1s = render(move[0])
move2s = render(move[1])
move1 = chessboard[move1s]
move2 = chessboard[move2s]
move_1 = transform_UR1.coords_conversion((move1[0], move1[1]))
move_2 = transform_UR1.coords_conversion((move2[0], move2[1]))
if execute_flag and online_game == 'Offline':
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#s.connect((HOST_UR1, PORT_UR1))
script = RobotScript('prog', s)
execute_move(move1s, move2s, chessboard, transform_UR1, script,
board, graveyard_red, move_1, move_2, None, None)
s.close()
hist.append(hist[-1].move(move))
print_pos(hist[-1].rotate())
time.sleep(1)
mate_flag = check_mate(hist, searcher)
if mate_flag:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST_UR1, PORT_UR1))
script = RobotScript('prog', s)
visualization.display(hist[-1][0], "Green Won!")
execute_mate_move(hist, chessboard, s, script, transform_UR1,
"green")
s.close()
visualization.display(hist[-1].rotate()[0], 'Red')
save_game(hist)
######################### Red's Move ########################################
if mode == "0" or (mode == "1" and player_side == "red"):
move = input_move(hist, "red")
execute_flag = True
if (mode == "1" and player_side == "green"):
move = calculate_move(searcher, hist)
execute_flag = True
if mode == '2' and load_game == "Play New Game":
if opening_moves:
move = opening_moves.pop(0)
move = 119 - parse(move[0:2]), 119 - parse(move[2:4])
else:
move = calculate_move(searcher, hist)
if mode == "3" and player_side == "green":
move = calculate_move(searcher, hist)
execute_flag = True
if mode == "3" and player_side == "red":
move = vision_move(hist, player_side, chessboard)
execute_flag = False
board = hist[-1].get_board()
hist.append(hist[-1].move(move))
move2_s = render(move[1])
move1_s = render(move[0])
move1s = render(119 - move[0])
move2s = render(119 - move[1])
move1 = chessboard[move1s]
move2 = chessboard[move2s]
move_1 = transform_UR2.coords_conversion((move1[0], move1[1]))
move_2 = transform_UR2.coords_conversion((move2[0], move2[1]))
time.sleep(1)
visualization.display(hist[-1][0], 'Green')
if execute_flag and online_game == 'Offline':
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#s.connect((HOST_UR2, PORT_UR2))
script = RobotScript('prog', s)
execute_move(move1_s, move2_s, chessboard, transform_UR2, script,
board, graveyard_green, move_1, move_2, move1s,
move2s)
s.close()
save_game(hist)
mate_flag = check_mate(hist, searcher)
if mate_flag:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST_UR2, PORT_UR2))
script = RobotScript('prog', s)
visualization.display(hist[-1][0], "Red Won!")
execute_mate_move(hist, chessboard, s, script, transform_UR1,
"red")
s.close()
from Searcher import Searcher print(Searcher.InsertionSearch([1, 4, 2, 11, 5, 304, 21, 3, 23, 54, 1, 2, 90]))
# -*- coding: utf-8 -*-
"""
Created on Wed Jun 6 18:52:06 2018
@author: sitandon
"""
#import sys
#sys.path.append("F:\Sid\Learnings\Python\Collective Intelligence by Toby\Code\chapter4\SearchEngine_Practice")
from Crawler_Prac import crawler
from Searcher import Searcher
RUN_CRAWLER = 0
if RUN_CRAWLER == 1:
crawler = crawler(".", "SearchEngine", "username", "Password")
crawler.crawl(["https://en.wikipedia.org/wiki/Google"], depth=2)
searcher = Searcher(".", "SearchEngine", "username", "Password")
searcher.query("india hyderabad", [
searcher.inboundLinksScore, searcher.pageRankScore,
searcher.frequencyScores
])
#coding: utf-8 import CONSTS from Comparator import Comparator from Searcher import Searcher from Generator import Generator if __name__ == '__main__': s1 = Searcher(CONSTS.O_DIR1) s2 = Searcher(CONSTS.D_DIR1) orig_files = s1.search() dev_files = s2.search() for f1 in orig_files: f2 = [item for item in dev_files if item['filename'] == f1['filename']] if len(f2) <= 0: continue f2 = f2[0] c = Comparator( "%s/%s" % (f1['dir'], f1['filename']), "%s/%s" % (f2['dir'], f2['filename']) ) c.compare() modified = c.get_modified_method() if len(modified) <= 0: continue #print f1['filename'] #for methodinfo in modified: # print "\t%s" % methodinfo g = Generator(f1['filename'], modified)
else:
infname = 'data/human-beings/' + human + '/' + naivety + '/' + data_type + '/head-data.csv'
baseoutdir = 'data/human-beings/' + human + '/' + naivety + '/' + data_type
print 'opening ',infname
print ' output',baseoutdir
with opener('r')(infname) as infile:
germlines = utils.read_germlines('../../../recombinator')
reader = csv.DictReader(infile)
il = 0
for inline in reader:
il += 1
print inline['seq'][-100:]
# if len(inline['seq']) != 130:
# assert 'simulated' in infname
searcher = Searcher(inline['seq'][-100:], debug=False, n_matches_max=5)
found_str = searcher.search()
values['found_strings']['v'][0].append(found_str) # toss them in ['v'][0] -- doesn't really make sense, but they're fine anywhere
if found_str != 'vjd': # skip the ones where we didn't find matches in this order (see freqs above).
continue
for region in utils.regions:
for imatch in range(len(searcher.matches[region])):
if imatch > 2:
break
match = searcher.matches[region][imatch]
if imatch == 0 and region == 'd':
# print '%s (%3d%3d) --> (%3d%3d %s)' % (region, match['ali_from'], match['ali_to'], match['ali_from'] - 1, len(searcher.query_seqs[region]) - match['ali_to'], searcher.query_seqs[region]),
values['vd_insertion_length'][region][imatch].append(match['ali_from'] - 1) # NOTE these are index *one* counting (!!!)
values['dj_insertion_length'][region][imatch].append(len(searcher.query_seqs[region]) - match['ali_to']) # NOTE these are index *one* counting (!!!)
values['evalue'][region][imatch].append(match['evalue'])
values['ali_from'][region][imatch].append(match['ali_from'])
def __init__(self):
self.timestep_secs = 10
self.searcher = Searcher()
self.composer = Composer()
self.mailer = Mailer()
class MainUI:
#------------------------------------------------------------------------------
""" Generates the GUI for Veranda"""
#------------------
def __init__(self):
#------------------
"""Initialize the program & ui"""
# Define some instance variables
self.name = "Veranda" # App name
self.version = "0.1.0" # Version
self.newNumbers = [] # for naming new tabs
self.database = DBusSql() # SQL Access driver
self.bottomState = False # False: notebookBottom closed
self.getObject = "get object" # Versaplex command for get object
self.listAll = "list all" # Versaplex command for list all
self.searcher = "" # Becomes a searcher object later
self.exposeEventID = 0 # Used to disconnect a signal
self.bindings = gtk.AccelGroup()# Keyboard bindings group
# Import Glade's XML and load it
self.gladeTree = gtk.glade.XML("ui.glade")
dic = {"on_exit":(gtk.mainquit)}
self.gladeTree.signal_autoconnect(dic)
# Grab some of the widgets for easy access
self.sidebar = ""
self.resulter = Resulter()
self.window = self.gladeTree.get_widget("window")
self.vboxMain = self.gladeTree.get_widget("vbox-main")
self.vpanedEditor = self.gladeTree.get_widget("vpaned-editor")
self.vpanedPanel = self.gladeTree.get_widget("vpaned-panel")
self.notebookTop = self.gladeTree.get_widget("notebook-top")
self.notebookBottom = self.gladeTree.get_widget("notebook-bottom")
self.buttonRun = self.gladeTree.get_widget("button-run")
self.buttonNewTab = self.gladeTree.get_widget("button-newtab")
self.buttonClose = self.gladeTree.get_widget("button-closetab")
self.buttonNext = self.gladeTree.get_widget("button-nexttab")
self.buttonPrevious = self.gladeTree.get_widget("button-lasttab")
self.entrySearch = self.gladeTree.get_widget("entry-search")
self.statusbar = self.gladeTree.get_widget("statusbar")
# Statusbar context ids: * "sidebar"
# * "run query"
# * "error"
# * "success"
# Misc Initializations
hbox = gtk.HBox()
hbox.show()
runImage = gtk.Image()
runImage.set_from_file("run.svg")
runImage.show()
hbox.pack_start(runImage)
label = gtk.Label(" Run")
label.show()
hbox.pack_start(label)
self.buttonRun.add(hbox)
hbox = gtk.HBox()
hbox.show()
newTabImage = gtk.Image()
newTabImage.set_from_file("new.svg")
newTabImage.show()
hbox.pack_start(newTabImage)
label = gtk.Label(" New Tab")
label.show()
hbox.pack_start(label)
self.buttonNewTab.add(hbox)
hbox = gtk.HBox()
hbox.show()
newTabImage = gtk.Image()
newTabImage.set_from_file("close.svg")
newTabImage.show()
hbox.pack_start(newTabImage)
label = gtk.Label(" Close Current Tab")
label.show()
hbox.pack_start(label)
self.buttonClose.add(hbox)
hbox = gtk.HBox()
hbox.show()
newTabImage = gtk.Image()
newTabImage.set_from_file("next.svg")
newTabImage.show()
hbox.pack_start(newTabImage)
label = gtk.Label(" Next Tab")
label.show()
hbox.pack_start(label)
self.buttonNext.add(hbox)
hbox = gtk.HBox()
hbox.show()
newTabImage = gtk.Image()
newTabImage.set_from_file("previous.svg")
newTabImage.show()
hbox.pack_start(newTabImage)
label = gtk.Label(" Previous Tab")
label.show()
hbox.pack_start(label)
self.buttonPrevious.add(hbox)
# Open a first tab (comes with configured editor)
self.newTab()
# Connect events & key strokes
self.window.connect("delete_event", gtk.main_quit)
self.buttonRun.connect("clicked", self.runQuery)
self.buttonNewTab.connect("clicked", self.newTab)
self.buttonClose.connect("clicked", self.closeCurrentTab)
self.buttonNext.connect("clicked", self.nextTab)
self.buttonPrevious.connect("clicked", self.lastTab)
self.entrySearch.connect("key-release-event", self.search)
self.exposeEventID = self.window.connect("expose-event",
self.postStartInit)
self.window.add_accel_group(self.bindings)
self.buttonRun.add_accelerator("clicked", self.bindings,
ord("r"), gtk.gdk.CONTROL_MASK, gtk.ACCEL_VISIBLE)
self.buttonNewTab.add_accelerator("clicked", self.bindings,
ord("t"), gtk.gdk.CONTROL_MASK, gtk.ACCEL_VISIBLE)
self.buttonNext.add_accelerator("clicked", self.bindings,
ord("n"), gtk.gdk.CONTROL_MASK, gtk.ACCEL_VISIBLE)
self.buttonPrevious.add_accelerator("clicked", self.bindings,
ord("p"), gtk.gdk.CONTROL_MASK, gtk.ACCEL_VISIBLE)
self.buttonClose.add_accelerator("clicked", self.bindings,
ord("w"), gtk.gdk.CONTROL_MASK, gtk.ACCEL_VISIBLE)
self.buttonClose.add_accelerator("clicked", self.bindings,
ord("c"), gtk.gdk.CONTROL_MASK, gtk.ACCEL_VISIBLE)
# Show things
self.window.show()
#---------------------
def initSidebar(self):
#---------------------
""" Initializes the sidebar with the tables list and configures it"""
toList = ["table", "view", "procedure",
"trigger", "scalarfunction", "tablefunction"]
statusID = self.statusbar.get_context_id("sidebar")
self.statusbar.push(statusID, "Initializing Sidebar")
scrolls = gtk.ScrolledWindow(gtk.Adjustment(), gtk.Adjustment())
scrolls.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC)
treestore = gtk.TreeStore(str)
self.sidebar = gtk.TreeView(treestore)
cell = gtk.CellRendererText()
column = gtk.TreeViewColumn("Database Objects", cell, text=0)
self.sidebar.append_column(column)
masterTable = []
for item in toList:
result = self.database.query(self.listAll+" "+item)
if "Error" not in result:
statusID = self.statusbar.get_context_id("success")
self.statusbar.push(statusID, "Success.")
parser = Parser(result)
table = parser.getTable()[:] #the [:] makes a clone
table.insert(0, [item])
masterTable.append(table)
rows = parser.getTableIterator()
iter = treestore.append(None, [item.title()])
while rows.hasNext():
treestore.append(iter, [str(rows.getNext()[0])])
else:
statusID = self.statusbar.get_context_id("error")
self.statusbar.push(statusID, result)
self.searcher = Searcher(masterTable)
self.sidebar.connect("row-activated", self.rowClicked, masterTable)
scrolls.add(self.sidebar)
self.vpanedPanel.add(scrolls)
scrolls.show()
self.sidebar.show()
self.statusbar.push(statusID, "Sidebar Loaded")
#-----------------------
def getNewNumber(self):
#-----------------------
""" Get a unique number to number a tab """
x = 0
while (True):
if x in self.newNumbers:
x = x+1
else:
self.newNumbers.append(x)
return r" "+str(x)+r" "
#----------------------------------------
def removeNumber(self, editor, notebook):
#----------------------------------------
""" If a given page has a label with an automatic
number, remove that number from the list of numbers so that
it can be reassigned to a new fresh tab in the future"""
label = self.getLabelText(editor, notebook)
label = label.split(" ")
self.newNumbers.remove(int(label[0]))
#---------------------------------------------
def configureEditor(self, editor, textbuffer):
#---------------------------------------------
"""Sets up a gtksourceview with the common options I want."""
languagemanager = gtksourceview.LanguageManager()
textbuffer.set_language(languagemanager.get_language("sql"))
textbuffer.set_highlight_syntax(True)
editor.set_show_line_numbers(True)
editor.set_wrap_mode(gtk.WRAP_WORD_CHAR)
editor.modify_font(pango.FontDescription("monospace 10"))
#---------------------------------------------
def makeBottomTabMenu(self, label, resulter):
#---------------------------------------------
"""Returns an hbox with the title, change button, and close button
to be put in a tab"""
hbox = gtk.HBox()
label = gtk.Label(r" "+str(label)+r" ")
hbox.pack_start(label)
changeIcon = gtk.Image()
changeIcon.set_from_file("cycle.svg")
buttonMode = gtk.Button(None)
buttonMode.add(changeIcon)
hbox.pack_start(buttonMode, False, False, 1)
closeIcon = gtk.Image()
closeIcon.set_from_file("close.svg")
buttonClose = gtk.Button(None)
buttonClose.add(closeIcon)
hbox.pack_start(buttonClose, False, False, 1)
buttonClose.connect("clicked", self.closeTab, resulter)
buttonMode.connect("clicked", self.changeMode, resulter)
changeIcon.show()
closeIcon.show()
buttonMode.show()
label.show()
buttonClose.show()
hbox.show()
return hbox
#---------------------------------------
def showOutput(self, topEditor, result):
#---------------------------------------
parser = Parser(result)
if self.bottomState == False:
self.resulter.update(parser)
self.notebookBottom.show()
hbox = self.makeBottomTabMenu("Results", self.resulter)
self.newTabBottom(self.resulter.getCurrentView(), hbox)
self.bottomState = True
else :
index = self.notebookBottom.page_num(self.resulter.getCurrentView())
hbox = self.notebookBottom.get_tab_label(
self.resulter.getCurrentView())
self.resulter.update(parser)
self.notebookBottom.remove_page(index)
self.notebookBottom.insert_page(self.resulter.getCurrentView(),
hbox, index)
self.notebookBottom.set_tab_reorderable(
self.resulter.getCurrentView(), True)
self.notebookBottom.set_current_page(index)
#------------------------------------
def newTabBottom(self, widget, hbox):
#------------------------------------
"""Creates a new tab on the bottom notebook, with "widget" in the tab
and "hbox" as the label (not actually a gtk label)"""
self.notebookBottom.append_page(widget, hbox)
#----------------------------------------
def getLabelText(self, editor, notebook):
#----------------------------------------
"""Retrieves the label number from notebook with a page which contains
the given editor"""
hbox = notebook.get_tab_label(editor)
children = hbox.get_children()
labelText = children[0].get_text()
labelText = labelText.strip(' ')
return str(labelText)
#---------------------------------
def expandSidebar(self, sidebarList):
#---------------------------------
"""Will expand some of the sidebar elements to make better use
of space"""
expandMax = 18
usedSoFar = 0
for section in sidebarList:
if len(section) + usedSoFar > expandMax:
break
else:
usedSoFar += len(section)
self.sidebar.expand_to_path((sidebarList.index(section),1))
#------------------------------------
def updateSidebar(self, sidebarList):
#------------------------------------
"""Given a new list, this will change the contents of the sidebar"""
treestore = gtk.TreeStore(str)
for section in sidebarList:
iter = treestore.append(None,[section[0][0]])
for element in section[1:]:
treestore.append(iter,[element[0]])
self.sidebar.set_model(treestore)
self.expandSidebar(sidebarList)
#----------------------#
#-- CALLBACK METHODS --#
#----------------------#
#------------------------------------------
def postStartInit(self, widget, data=None):
#------------------------------------------
""" Initializes all the stuff that should only happen after the window
is already on screen"""
self.initSidebar()
widget.disconnect(self.exposeEventID)
#-------------------------------------
def runQuery(self, widget, data=None):
#-------------------------------------
"""Uses the database abstraction (initially Dbus)
To send the query that is in the current window"""
scrolls = self.notebookTop.get_nth_page(self.notebookTop.
get_current_page())
if scrolls != None:
editor = scrolls.get_children()[0]
buffer = editor.get_buffer()
#get all text, not including hidden chars
query = buffer.get_text(buffer.get_start_iter(),
buffer.get_end_iter(), False)
contextID = self.statusbar.get_context_id("run query")
self.statusbar.push(contextID, "Ran query: "+query)
result = self.database.query(query)
if "Error" not in result:
self.showOutput(editor, result)
statusID = self.statusbar.get_context_id("success")
self.statusbar.push(statusID, "Success.")
else:
statusID = self.statusbar.get_context_id("error")
self.statusbar.push(statusID, result)
else:
contextID = self.statusbar.get_context_id("error")
self.statusbar.push(contextID, "No query to run.")
#-----------------------------------
def search(self, widget, data=None):
#-----------------------------------
"""Incremental search callback. As the user types, this method
notices and modifies the sidebar"""
text = widget.get_text()
sidebarList = self.searcher.find(text)
self.updateSidebar(sidebarList)
#--------------------------------------
def newTab(self, widget=None, data=None):
#--------------------------------------
"""Open a new editor tab (top). Data is an optional title for the tab."""
scrolls = gtk.ScrolledWindow(gtk.Adjustment(), gtk.Adjustment())
scrolls.set_policy(gtk.POLICY_AUTOMATIC, gtk.POLICY_AUTOMATIC)
textbuffer = gtksourceview.Buffer()
editor = gtksourceview.View(textbuffer)
self.configureEditor(editor, textbuffer)
hbox = gtk.HBox()
if data == None:
label = gtk.Label(self.getNewNumber())
else:
label = gtk.Label(self.getNewNumber()+str(data)+" ")
hbox.pack_start(label)
closeIcon = gtk.Image()
closeIcon.set_from_file("close.svg")
buttonClose = gtk.Button(None)
buttonClose.add(closeIcon)
hbox.pack_start(buttonClose, False, False, 1)
buttonClose.connect("clicked", self.closeTab, scrolls)
scrolls.add(editor)
self.notebookTop.append_page(scrolls, hbox)
self.notebookTop.set_tab_reorderable(scrolls, True)
scrolls.show()
closeIcon.show()
label.show()
buttonClose.show()
hbox.show()
editor.show()
# KEEP THIS LINE AT THE END OR ELSE! (hours of frustration...)
self.notebookTop.set_current_page(-1)
return editor
#--------------------------------------------
def closeTab(self, sourceWidget, targetWidget):
#--------------------------------------------
"""Close a tab. targetWidget points to the contents of the notebook
tab that you want closed."""
index = -1
try:
index = self.notebookTop.page_num(targetWidget)
except TypeError:
pass
if index != -1:
self.removeNumber(targetWidget, self.notebookTop)
self.notebookTop.remove_page(index)
return
index = self.notebookBottom.page_num(targetWidget.getCurrentView())
if index != -1:
self.notebookBottom.remove_page(index)
self.bottomState = False
self.notebookBottom.queue_resize()
self.notebookTop.queue_resize()
return
if index == -1:
print "Worse Than Failure: Lost The Tab!"
#--------------------------------------------
def closeCurrentTab(self, widget, data=None):
#--------------------------------------------
"""Closes the current tab in the top editor section"""
index = self.notebookTop.get_current_page()
self.notebookTop.remove_page(index)
#------------------------------------
def nextTab(self, widget, data=None):
#------------------------------------
"""Changes to the previous tab"""
index = self.notebookTop.get_current_page()
self.notebookTop.set_current_page((index+1) % \
self.notebookTop.get_n_pages())
#------------------------------------
def lastTab(self, widget, data=None):
#------------------------------------
"""Changes to the next tab"""
index = self.notebookTop.get_current_page()
self.notebookTop.set_current_page(index-1)
#--------------------------------------
def changeMode(self, widget, resulter):
#--------------------------------------
"""After a change button is clicked, this makes the notebook tab
osscroll through the different view modes in a fixed pattern"""
pageIndex = self.notebookBottom.page_num(resulter.getCurrentView())
hbox = self.notebookBottom.get_tab_label(resulter.getCurrentView())
self.notebookBottom.remove_page(pageIndex)
self.notebookBottom.insert_page(resulter.getNextView(), hbox, pageIndex)
self.notebookBottom.set_tab_reorderable(resulter.getCurrentView(), True)
#-------------------------------------------------------------
def rowClicked(self, treeview, position, column, masterTable):
#-------------------------------------------------------------
"""
Given the position coordinates and the master table (a
list of all data that is in the sidebar), this method opens
a new editor tab which has code in it. The code is the source
code to the object that was double clicked on in the sidebar.
If the item is a table, the code is just a select statement.
"""
try:
type = masterTable[position[0]][0][0]
name = masterTable[position[0]][position[1]+1][0]
except IndexError:
contextID = self.statusbar.get_context_id("error")
self.statusbar.push(contextID,
"Can't do anything with a category title")
print "Can't do anything when a category title is clicked"
return
if type == "table":
query = "select top 100 * from [%s]" % name
contextID = self.statusbar.get_context_id("run query")
self.statusbar.push(contextID, "Ran query: "+query)
result = self.database.query(query)
if "Error" not in result:
editor = self.newTab(None, name)
buffer = editor.get_buffer()
buffer.set_text(query)
statusID = self.statusbar.get_context_id("success")
self.statusbar.push(statusID, "Success.")
self.showOutput(editor, result)
else:
statusID = self.statusbar.get_context_id("error")
self.statusbar.push(statusID, result)
else:
query = self.getObject + " " + type + " " + name
contextID = self.statusbar.get_context_id("run query")
self.statusbar.push(contextID, "Ran query: "+query)
result = self.database.query(query)
if "Error" in result:
statusID = self.statusbar.get_context_id("error")
self.statusbar.push(statusID, result)
return
else:
statusID = self.statusbar.get_context_id("success")
self.statusbar.push(statusID, "Success.")
parser = Parser(result)
data = parser.getTable()
commands = data[0][0]
com2 = commands[:]
pattern1 = re.compile(r"^create", re.I)
commands = re.sub(pattern1, r"ALTER", commands, 1)
if commands == com2:
pattern2 = re.compile(r"\ncreate", re.I)
commands = re.sub(pattern2, r"\nALTER", commands, 1)
editor = self.newTab(None, name)
buffer = editor.get_buffer()
buffer.set_text(commands)
def searchText(self):
# these may not exist if search isn't supported for this
# product so we only try to import if this function gets
# called
import xapian
from Searcher import Searcher
# A constant for the number of results that should be returned for
# every query.
MAX_NO_OF_RESULTS = 10
# A constant for the position of the file path within the document
# object.
XAPIAN_FILE_NAME_SLOT = 1
# Add the wildcard implicitly to the query string. This allows the user
# to enter only part of the query in order to get search results.
queryString = self.fields['query'] + '*'
# Get the reference to the gui class.
guiReference = self.session().value('guiReference')
# Searcher object for performing the actual search.
searcher = Searcher('/opt/tms/web2/appserver/search_index',
xapian.QueryParser.STEM_SOME)
# Configuration flags for the query parser.
# Note: Synonyms are OR'd with the query entered. Eg: if foo is a
# synonym for bar, then the query bar wil expand to ==> bar or baz.
searchFlags = xapian.QueryParser.FLAG_PHRASE \
| xapian.QueryParser.FLAG_WILDCARD \
| xapian.QueryParser.FLAG_AUTO_MULTIWORD_SYNONYMS \
| xapian.QueryParser.FLAG_SPELLING_CORRECTION \
| xapian.QueryParser.FLAG_AUTO_SYNONYMS \
| xapian.QueryParser.FLAG_BOOLEAN_ANY_CASE
# Top level results node.
results = self.doc.createElement('results');
# Counter for the number of results.
resultCount = 0
# Matches obtained on doing a search.
matches = searcher.search(searchFlags, queryString)
## @brief Creates a structure within the xml file of the form:
#
# <result title="TITLE" link="LINK" />
#
# @param titleText This is the entry that will be displayed for this result
# within the autocomplete list. It will be the same as the
# title for every pagelet.
# @param linkText This is the link to the result. This is the text that
# occurs after p= for the gui servlet.
def appendResult(titleText, linkText):
result = self.doc.createElement('result')
result.setAttribute('title', titleText)
result.setAttribute('link', linkText)
results.appendChild(result)
# Process the results
for match in matches:
# Get the name of the file without its extension.
fileName = match.document.get_value(XAPIAN_FILE_NAME_SLOT)
# Get the title for the result.
pageletTitle = guiReference.findBestFitPagelet(fileName, {}).getPageletTitle()
# If no appropriate pagelet title is found then the indexed page
# isn't present in the navbar so we can ignore this result.
if pageletTitle == '':
continue
# Increment the number of results that have been found. If that
# exceeds the maximum desired then stop adding additional elements
# to the xml document.
resultCount += 1
if resultCount > MAX_NO_OF_RESULTS:
break
# Create an xml structure representing a result within the xml
# document.
appendResult(pageletTitle, fileName)
# Add a count of the number of results available.
results.setAttribute('resultCount', str(resultCount))
# Add the results to the document.
self.doc.documentElement.appendChild(results)
# Write the xml document in the response.
self.writeXmlDoc()
from Searcher import Searcher
# desc = RGBHistogram([8 8, 8])
desc = RGBHistogram([4, 4, 4])
test_image_path = '../data/flowers/image_0030.jpg'
query_image = cv2.imread(test_image_path)
query_feature = desc.describe(query_image)
# print(query_feature)
# quit()
# load the index and initialize our searcher
index = pickle.load(open("Histogram_only_index_4bins.cpickle", "rb"))
print(index.items())
searcher = Searcher(index)
results = searcher.search(query_feature)
print(results)
for i in range(0, 5):
ret_path = '../data/' + results[i][1].replace('\\','/')
img = cv2.imread(ret_path)
cv2.imshow(str(i), img)
cv2.waitKey()
exit()
list_images = glob.glob("../data/flowers/*.jpg")
# initialize our image descriptor -- a 3D RGB histogram with
# 8 bins per channel
import argparse
import cv2
import re
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--index", required=True, help="C:\ml\Monumark")
ap.add_argument("-q", "--query", required=True, help="C:\ml\Monumark\queries")
ap.add_argument("-r",
"--result_path",
required=True,
help="C:\ml\Monumark\Final_Results")
args = vars(ap.parse_args())
cd = Descriptor((8, 12, 3))
query = cv2.imread(args["query"])
features = cd.describe(query)
results = Searcher(args["index"])
final = results.search(features)
cv2.imshow("Query", query)
for (score, resultID) in final:
print(score, resultID)
resultID = resultID.split("\\")[1]
print(score, resultID)
result = cv2.imread(args["result_path"] + "/" + resultID)
cv2.imshow("Result", result)
cv2.waitKey(0)
class Tracker:
LIG_THR_EVERY_FRAMES = 15
def __init__(self, initialPosition, initialWidth, initialHeight, frame,
parametersNew):
#########################################
#########################################
self.initFrame = frame
self.initPos = initialPosition
self.initW = initialWidth
self.initH = initialHeight
self.KM = KalmanFilter()
self.MF = MaskingFilter()
self.KM.setStatePost(
np.array([initialPosition[0], initialPosition[1], 0.,
0.]).reshape(4, 1))
self.selectionWidth = initialWidth
self.selectionHeight = initialHeight
self.prevFrameGray = None
self.frameCounter = 0 #Shape = {tuple}: (x, 1, 2)
#Por ejemplo: [[[x1 y1]]\n\n [[x2 y2]]\n\n [[x3 y3]]]
#es decir una matriz de x filas y 1 columna, donde cada elemento
#de la unica columna es una coordenada [x y].
if self.MF.mask is not self.MF.maskingType["FILTER_OFF"]:
self.MF.calculateNewMask(
frame, frame[int(initialPosition[1] -
initialHeight / 2):int(initialPosition[1] +
initialHeight / 2),
int(initialPosition[0] -
initialWidth / 2):int(initialPosition[0] +
initialWidth / 2)])
frame = self.MF.filterFrame(frame)
self.prevFrameGray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)
self.SC = Searcher(self.initFrame, initialHeight, initialWidth,
initialPosition[0], initialPosition[1],
cv.cvtColor(frame, cv.COLOR_BGR2GRAY))
self.SC.features, self.SC.trackingError = self.SC.ST.recalculateFeatures(
self.prevFrameGray[int(initialPosition[1] -
initialHeight / 2):int(initialPosition[1] +
initialHeight / 2),
int(initialPosition[0] -
initialWidth / 2):int(initialPosition[0] +
initialWidth / 2)])
self.SC.features = self.SC.featureTranslate(
initialPosition[0] - initialWidth / 2,
initialPosition[1] - initialHeight / 2, self.SC.features)
self.SC.LK.prevFeatures = self.SC.features
#OPTIMIZACIÓN
self.bins_var = 1
self.kernel_blur_var = 1
self.mask_blur_var = 1
self.low_pth_var = 200
def getTrackingError(self):
return self.SC.trackingError
def setFilter(self, filterType):
if filterType in self.MF.maskingType.keys():
self.MF.mask = self.MF.maskingType[filterType]
else:
print("Wrong filter type")
def featureTranslate(self, x, y, features):
if features is None:
return None
for i in range(features.shape[0]):
features[i][0][0] += x
features[i][0][1] += y
return features
def update(self, frame):
self.frameCounter += 1
self.KM.predict()
realframe = frame
if self.MF.mask is self.MF.maskingType["FILTER_LAB"]:
if self.frameCounter != 0 and self.frameCounter % self.MF.CIELabRecalculationNumber == 0 and self.MF.labPeriodicRecalculations is True and self.SC.trackingError is False:
vx, vy = self.getEstimatedVelocity()
if np.abs(vx) < 5 and np.abs(vy) < 5:
medx, medy = np.median(
self.SC.features[:, 0,
0]), np.median(self.SC.features[:, 0,
1])
std = np.sqrt((np.std(self.SC.features[:, 0, 0]))**2 +
(np.std(self.SC.features[:, 0, 1]))**2)
# calculate mean and std of features
mask = (self.SC.features[:, 0, 0] <
medx + self.SC.stdMultiplier * std + 0.1) & (
self.SC.features[:, 0, 0] >
medx - self.SC.stdMultiplier * std - 0.1) & (
self.SC.features[:, 0, 1] <
medy + self.SC.stdMultiplier * std +
0.1) & (self.SC.features[:, 0, 1] > medy -
self.SC.stdMultiplier * std - 0.1)
self.SC.features = self.SC.features[mask]
# remove outliers.
medx, medy = np.median(
self.SC.features[:, 0,
0]), np.median(self.SC.features[:, 0,
1])
if (~np.isnan(medx)) and (~np.isnan(medy)):
self.MF.calculateNewMask(
frame,
frame[int(medy - self.selectionHeight /
2):int(medy + self.selectionHeight / 2),
int(medx - self.selectionWidth /
2):int(medx + self.selectionWidth / 2)])
frame = self.MF.filterFrame(frame)
elif self.MF.mask is self.MF.maskingType["FILTER_CSHIFT"]:
frame = self.MF.filterFrame(frame)
#TINCHO
#Tacking error?
if self.SC.trackingError is True:
if self.SC.missAlgorithm == self.SC.missAlgorithmD["ST"]:
x, y = self.SC.searchMissing(self.KM.statePost[0][0],
self.KM.statePost[1][0], frame,
frame)
elif self.SC.missAlgorithm == self.SC.missAlgorithmD["CORR"]:
x, y = self.SC.searchMissing(self.KM.statePost[0][0],
self.KM.statePost[1][0],
realframe, frame)
if self.SC.trackingError is False:
self.KM.correct(x, y)
else:
x, y = self.SC.search(self.frameCounter, realframe, frame)
if self.SC.trackingError is False:
self.KM.correct(x, y)
def changeSettings(self, parametersNew):
self.KM.dt = parametersNew[0] #kalman_ptm
self.KM.PROCESS_COV = parametersNew[1] #kalman_pc
self.KM.MEAS_NOISE_COV = parametersNew[2] #kalman_mc
self.SC.LK.lkMaxLevel = int(parametersNew[3]) #lk_mr
if parametersNew[4] is False: #Color Filter OnOff
self.MF.mask = self.MF.maskingType["FILTER_OFF"]
self.MF.LSemiAmp = parametersNew[5] #colorFilter_LihtThr
self.MF.aSemiAmp = parametersNew[6] #colorFilter_a
self.MF.bSemiAmp = parametersNew[7] #colorFilter_b
if parametersNew[20] == True and parametersNew[19] == False:
self.SC.missAlgorithm = self.SC.missAlgorithmD["ST"]
elif parametersNew[20] == False and parametersNew[19] == True:
self.SC.missAlgorithm = self.SC.missAlgorithmD["CORR"]
if parametersNew[22] == True and parametersNew[21] == False:
self.SC.recalcAlgorithm = self.SC.recalcAlgorithmD["ST"]
elif parametersNew[22] == False and parametersNew[21] == True:
self.SC.recalcAlgorithm = self.SC.recalcAlgorithmD["CORR"]
self.SC.MASKCONDITION = parametersNew[23]
#= parametersNew[8] #Light R OnOff
#= parametersNew[9] #ligtRec_x)
#= parametersNew[10] #ligtRec_maxT
#= parametersNew[11] #Cam shift On/Off
self.SC.ST.maxcorners = int(parametersNew[13]) #shit_MaxFeat
self.SC.ST.qLevel = parametersNew[14] #shit_FeatQual
self.SC.ST.minDist = parametersNew[15] #shit_MinFeat
#= parametersNew[16] #ShiTomasiOn/ Off
self.SC.ST.frameRecalculationNumber = parametersNew[16] #shit_SPix
#self.MF.mask = self.MF.maskingType[parametersNew[??]] #MENSAJE PARA TOMI: tiene que ser un string parametersNew[??] fijate en la clase
self.MF.hist_filter.set_bins(parametersNew[9])
self.MF.hist_filter.set_mask_blur(parametersNew[10])
self.MF.hist_filter.set_kernel_blur(parametersNew[11])
self.MF.hist_filter.set_low_pth(parametersNew[12])
self.MF.ksize = parametersNew[24]
if int(self.MF.ksize) % 2 == 0:
self.MF.ksize = int(self.MF.ksize) + 1
else:
self.MF.ksize = int(self.MF.ksize)
self.MF.updateMaskFromSettings()
self.KM.updateParams()
def updateKalman(self, kalman_ptm, kalman_pc, kalman_mc):
self.KM.dt = kalman_ptm
self.KM.PROCESS_COV = kalman_pc
self.KM.MEAS_NOISE_COV = kalman_mc
self.KM.updateParams()
def updateLK(self, lk_mr):
self.SC.LK.lkMaxLevel = lk_mr
def updateColorFilter(self, CFPropOnOff, LihtThr, a, b, maskBlur_lab):
if CFPropOnOff is False: # Color Filter OnOff
self.MF.mask = self.MF.maskingType["FILTER_OFF"]
self.MF.LSemiAmp = LihtThr
self.MF.aSemiAmp = a
self.MF.bSemiAmp = b
self.MF.ksize = maskBlur_lab
if int(self.MF.ksize) % 2 == 0:
self.MF.ksize = int(self.MF.ksize) + 1
else:
self.MF.ksize = int(self.MF.ksize)
self.MF.updateMaskFromSettings()
def updateCamShift(self, CFCamShiftOnOff, bins, mb, sb, lbpt):
self.MF.hist_filter.set_bins(bins)
self.MF.hist_filter.set_mask_blur(mb)
self.MF.hist_filter.set_kernel_blur(sb)
self.MF.hist_filter.set_low_pth(lbpt)
self.MF.updateMaskFromSettings()
def updateShiT(self, MaxFeat, FeatQual, MinFeat, Rec, ShiTPropOnOff, SPix):
self.SC.ST.maxcorners = int(MaxFeat)
self.SC.ST.qLevel = FeatQual
self.SC.ST.minDist = MinFeat
self.SC.ST.frameRecalculationNumber = Rec
def updateMissinSearch(self, missCorr, missST, recCor, recST):
if missST == True and missCorr == False:
self.SC.missAlgorithm = self.SC.missAlgorithmD["ST"]
self.SC.searchHeight = self.initH
self.SC.searchWidth = self.initW
elif missST == False and missCorr == True:
self.SC.missAlgorithm = self.SC.missAlgorithmD["CORR"]
self.SC.searchHeight = self.initH
self.SC.searchWidth = self.initW
if recST == True and recCor == False:
self.SC.recalcAlgorithm = self.SC.recalcAlgorithmD["ST"]
elif recST == False and recCor == True:
self.SC.recalcAlgorithm = self.SC.recalcAlgorithmD["CORR"]
def updateMaskCond(self, maskCondition):
self.SC.MASKCONDITION = maskCondition
def updateBGR(self, color):
self.MF.calculateNewMask(None, None, True, color)
self.MF.updateMaskFromSettings()
def getFilteredFrame(self):
return self.MF.filteredFrame
def getCorrFrame(self):
return self.SC.corr_out
def getEstimatedPosition(self):
return self.KM.statePost[0][0], self.KM.statePost[1][0]
def getEstimatedVelocity(self):
return self.KM.statePost[2][0], self.KM.statePost[3][0]
def getTrajectory(self):
return self.KM.trajectory
def costChangeParamsLAB(self, x):
self.MF.LSemiAmp = x[0]
self.MF.aSemiAmp = x[1]
self.MF.bSemiAmp = x[2]
self.MF.updateMaskFromSettings()
testFrame = self.MF.filterFrame(self.initFrame)
countTotal = np.count_nonzero(testFrame)
countInside = np.count_nonzero(testFrame[
int(self.initPos[1] -
self.selectionHeight / 2):int(self.initPos[1] +
self.selectionHeight / 2),
int(self.initPos[0] -
self.selectionWidth / 2):int(self.initPos[0] +
self.selectionWidth / 2)])
countOutside = countTotal - countInside
# print(countOutside-countInside)
# return countOutside*(1/5) - countInside*(4/5)
return countOutside - countInside
def calculate_optimal_params(self):
if self.MF.mask is self.MF.maskingType["FILTER_LAB"]:
self.optimize()
params = {
"l": self.MF.LSemiAmp,
"a": self.MF.aSemiAmp,
"b": self.MF.bSemiAmp,
"blur": self.MF.ksize
}
return params
elif self.MF.mask is self.MF.maskingType["FILTER_CSHIFT"]:
for i in range(3):
self.optimize()
params = {
"bins": self.MF.hist_filter.bins_opti,
"mask_blur": self.MF.hist_filter.mask_blur_size_opti,
"kernel_blur": self.MF.hist_filter.kernel_blur_size_opti,
"low_pth": self.MF.hist_filter.low_pth_opti
}
return params
def calculate_cost(self):
test_frame = self.MF.filterFrame(self.initFrame)
count_total = np.count_nonzero(test_frame)
count_inside = np.count_nonzero(test_frame[
int(self.initPos[1] -
self.selectionHeight / 2):int(self.initPos[1] +
self.selectionHeight / 2),
int(self.initPos[0] -
self.selectionWidth / 2):int(self.initPos[0] +
self.selectionWidth / 2)])
count_outside = count_total - count_inside
return count_outside - count_inside
# return count_outside**4 - count_inside**3 #PREFERIMOS QUE ESTE VACIO AFUERA
def optimize(self):
if self.MF.mask is self.MF.maskingType["FILTER_LAB"]:
for j in range(3):
best_L = [self.MF.LSemiAmp]
best_cost = self.calculate_cost()
for i in range(10, 150):
self.MF.LSemiAmp = i
self.MF.updateMaskFromSettings()
cost = self.calculate_cost()
if cost < best_cost:
best_L.append(i)
best_cost = cost
self.MF.LSemiAmp = best_L[-1]
self.MF.updateMaskFromSettings()
best_mask_blur = [self.MF.ksize]
# best_cost = 0
for i in [1, 3, 5, 7, 9, 11, 13, 15, 17, 19]:
self.MF.ksize = i
cost = self.calculate_cost()
if cost < best_cost:
best_mask_blur.append(i)
best_cost = cost
self.MF.ksize = best_mask_blur[-1]
best_a = [self.MF.aSemiAmp]
for i in range(5, 100):
self.MF.aSemiAmp = i
self.MF.updateMaskFromSettings()
cost = self.calculate_cost()
if cost < best_cost:
best_a.append(i)
best_cost = cost
self.MF.aSemiAmp = best_a[-1]
self.MF.updateMaskFromSettings()
best_b = [self.MF.bSemiAmp]
for i in range(5, 100):
self.MF.bSemiAmp = i
self.MF.updateMaskFromSettings()
cost = self.calculate_cost()
if cost < best_cost:
best_b.append(i)
best_cost = cost
self.MF.bSemiAmp = best_b[-1]
self.MF.updateMaskFromSettings()
self.MF.LSemiAmp = best_L[-1]
self.MF.aSemiAmp = best_a[-1]
self.MF.bSemiAmp = best_b[-1]
self.MF.ksize = best_mask_blur[-1]
self.MF.updateMaskFromSettings()
x_bounds = [(0, 150), (0, 150), (0, 150)]
x0 = np.array(
[self.MF.LSemiAmp, self.MF.aSemiAmp, self.MF.bSemiAmp])
# res = optimize.least_squares(self.costChangeParamsLAB,x0=x0,bounds=[(0,0,0),(150,150,150)],ftol=1000)
#res = optimize.minimize(self.costChangeParamsLAB, x0=x0, bounds=x_bounds,method="Powell")
res = optimize.minimize(self.costChangeParamsLAB,
x0=x0,
method="Powell")
self.MF.LSemiAmp = res.x[0]
self.MF.aSemiAmp = res.x[1]
self.MF.bSemiAmp = res.x[2]
self.MF.updateMaskFromSettings()
else:
best_bin = [self.MF.hist_filter.bins]
best_cost = self.calculate_cost()
for i in range(1, 200):
self.MF.hist_filter.set_bins(i)
self.MF.updateMaskFromSettings()
cost = self.calculate_cost()
if cost < best_cost:
best_bin.append(i)
best_cost = cost
self.MF.hist_filter.set_bins(best_bin[-1])
self.MF.updateMaskFromSettings()
best_mask_blur = [self.MF.hist_filter.mask_blur_size]
# best_cost = 0
for i in [1, 3, 5, 7, 9, 11, 13, 15, 17, 19]:
self.MF.hist_filter.set_mask_blur(i)
cost = self.calculate_cost()
if cost < best_cost:
best_mask_blur.append(i)
best_cost = cost
self.MF.hist_filter.set_mask_blur(best_mask_blur[-1])
best_kernel_blur = [self.MF.hist_filter.kernel_blur_size]
for i in [1, 3, 5, 7, 9, 11, 13, 15, 17, 19]:
self.MF.hist_filter.set_kernel_blur(i)
self.MF.updateMaskFromSettings()
cost = self.calculate_cost()
if cost < best_cost:
best_kernel_blur.append(i)
best_cost = cost
self.MF.hist_filter.set_kernel_blur(best_kernel_blur[-1])
self.MF.updateMaskFromSettings()
best_low_pth = [self.MF.hist_filter.low_pth]
self.MF.hist_filter.set_bins(best_bin[-1])
self.MF.hist_filter.set_mask_blur(best_mask_blur[-1])
self.MF.hist_filter.set_kernel_blur(best_kernel_blur[-1])
self.MF.hist_filter.set_low_pth(best_low_pth[-1])
self.MF.hist_filter.bins_opti = best_bin[-1]
self.MF.hist_filter.mask_blur_size_opti = best_mask_blur[-1]
self.MF.hist_filter.kernel_blur_size_opti = best_kernel_blur[-1]
self.MF.hist_filter.low_pth_opti = best_low_pth[-1]
# self.MF.hist_filter.set_low_pth(best_low_pth[-1])
self.MF.updateMaskFromSettings()
def colorKernelChange(self, bgr):
b = bgr[0]
g = bgr[1]
r = bgr[2]
def showSearchArea(self):
if self.SC.missAlgorithm == self.SC.missAlgorithmD["ST"]:
return True
else:
return False
for page in sorted(pageRanks[1]):
print '\t\t\t' + page,
for step, pageRank in sorted(pageRanks.iteritems()):
print
print 'Step ' + str(step) + '\t',
for page, rank in sorted(pageRank.iteritems()):
print '\t\t' + '{0:.4f}'.format(rank),
print
webGraph = frontier.getWebGraph(seedDocuments)
pageRank = pageRanker.getPageRank(webGraph)
pageContents = indexer.getPageToken(webGraph)
index = indexer.getIndex(pageContents)
searcher = Searcher(index, pageContents)
printWebGraph(webGraph)
printPageRanks(pageRank)
printPageContents(pageContents)
printIndex(index)
searcher.search('tokens')
searcher.searchAdvanced('tokens', pageRank)
print
searcher.search('index')
searcher.searchAdvanced('index', pageRank)
print
searcher.search('classification')
searcher.searchAdvanced('classification', pageRank)
print
from Authenticator import Authenticator from Searcher import Searcher auth = Authenticator() api = Authenticator.getapi(auth) Searcher(api)
def setUp(self):
self.reader = WordReader()
self.finder = Trie(self.reader)
self.searcher = Searcher(self.finder, '')
if egg in occupied:
return False
occupied.append((x,y))
return True
#M,N,K
puzzles = [ (5,5,2),
(6,6,2),
(8,8,1),
(10,10,3)]
Tmap = { puzzles[0]:(1,0.995),
puzzles[1]:(1,0.995),
puzzles[2]:(10,0.999),
puzzles[3]:(10,0.999)}
for puzzle in puzzles:
M,N,K = puzzle
board = Drawer([[0 for x in range(N)] for y in range(M)], (100,100))
initialSolution = Solution(puzzle)
solution = Searcher(initialSolution, M+2*(M+N-1), Tmap[puzzle][0], Tmap[puzzle][1])
carton = solution.search().state
for egg in carton:
board.drawDot(egg, 'blue')
#board.dump()
board.image.save('solutions/' + str(puzzle[0])+str(puzzle[1])+str(puzzle[2]) + '.png', 'PNG')
import pickle
if __name__ == "__main__":
print "Hello World";
lukija = WordReader(["../Material/Grimm's Fairy Tales.txt"])
# lukija.readWords()
punamusta = RedBlack(lukija)
trie = Trie(lukija)
# print 'Adding all words to Punamusta'
# punamusta.addFromReader()
searcher = Searcher(trie, '')
words = {}
checklist = [0]*11
while(True):
word = searcher.randomWord()
if len(word) > 10 or len(word) < 4:
continue
if not len(word) in words:
words[len(word)] = [word]
elif len(words[len(word)]) < 10:
words[len(word)].append(word)
else:
words[len(word)].append(word)
checklist[len(word)] = 1
print word, checklist
if sum(checklist[4:]) == 7:
class Main:
"""
Class Description :
Implements the Controller of the MVC model, runs the project.
"""
"""
Desctiption
This method is for initializing Main's properties
"""
def __init__(self):
self.main_path = ''
self.posting_path = ''
self.to_stem = False
self.indexer = None
self.reader = ReadFile()
self.languages = set()
self.searcher = None
self.queries_docs_results = []
self.avg_doc_length = 0
self.with_semantics = False
self.save_path = ''
"""
Description :
This method manages the program
"""
def start(self):
self.indexer = Indexer(self.posting_path)
if self.to_stem:
self.indexer.to_stem = True
dirs_list = os.listdir(self.main_path + '\\corpus')
# Create temp postings Multiprocessing
dirs_dict = ParallelMain.start(self.main_path, self.posting_path,
self.to_stem, dirs_list)
# Merging dictionaries that were created by the processes
docs = {}
files_names = []
post_files_lines = []
total_length = 0
for dir in dirs_dict.keys():
tmp_docs_dict = dirs_dict[dir][2]
for doc_id in tmp_docs_dict:
docs[doc_id] = tmp_docs_dict[doc_id]
total_length += docs[doc_id].length
for lang in dirs_dict[dir][3]:
self.languages.add(lang)
old_post_files_lines = dirs_dict[dir][0]
for i in range(0, len(old_post_files_lines)):
files_names.append(dir + "\\Posting" +
str(i) if not self.to_stem else dir +
"\\sPosting" + str(i))
post_files_lines.append(old_post_files_lines[i])
self.avg_doc_length = total_length / len(docs)
# Gets Cities that appear in the corpus
i = 0
while i < len(dirs_list):
self.reader.read_cities(self.main_path + '\\corpus', dirs_list[i])
i += 1
terms_dicts = [
dirs_dict["\\Postings1"][1], dirs_dict["\\Postings2"][1],
dirs_dict["\\Postings3"][1], dirs_dict["\\Postings4"][1]
]
terms_dict = Merge.start_merge(files_names, post_files_lines,
terms_dicts, self.posting_path,
self.to_stem)
self.indexer.docs_avg_length = self.avg_doc_length
self.indexer.terms_dict = terms_dict
self.indexer.docs_dict = docs
self.indexer.index_cities(self.reader.cities)
self.indexer.post_pointers(self.languages)
# self.searcher = Searcher(self.main_path, self.posting_path, self.indexer.terms_dict, self.indexer.cities_dict,
# self.indexer.docs_dict, self.avg_doc_length, self.to_stem, self.with_semantics)
# self.searcher.model = Word2Vec.load('model.bin')
# path = self.posting_path + '\FinalPost' + '\Final_Post'
# linecache.getline(path, 500000)
"""
Description :
This method calls the Indexer function for loading saved files to the programs main memory
"""
def load(self):
self.indexer = Indexer(self.posting_path)
if self.to_stem:
self.indexer.to_stem = True
self.languages = self.indexer.load()
self.avg_doc_length = self.indexer.docs_avg_length
self.searcher = Searcher(self.main_path, self.posting_path,
self.indexer.terms_dict,
self.indexer.cities_dict,
self.indexer.docs_dict, self.avg_doc_length,
self.to_stem, self.with_semantics)
self.searcher.model = Word2Vec.load(self.posting_path + '//model.bin')
"""
Description :
This method erases all of the files in the Posting path
"""
def reset(self):
shutil.rmtree(self.posting_path)
if not os.path.exists(self.posting_path):
os.makedirs(self.posting_path)
self.indexer = None
"""
Description :
This method returns the terms dictionary, used by GUI IndexView for showing the dictionary.
"""
def get_terms_dict(self):
return self.indexer.terms_dict
"""
Description :
This method returns the Languages of the corpus, used by GUI IndexView for showing the lagnuages.
"""
def get_languages(self):
# should return string with languages separated with '\n'
return self.languages
"""
Description :
This method gets the corpus path from the GUI
"""
def set_corpus_path(self, path):
self.main_path = path
"""
Description :
This method gets the posting path from the GUI
"""
def set_posting_path(self, path):
self.posting_path = path
"""
Description :
This method gets the stemming bool from the GUI
"""
def set_stemming_bool(self, to_stem):
self.to_stem = to_stem
def set_with_semantics(self, with_semantics):
self.with_semantics = with_semantics
self.searcher.with_semantics = with_semantics
def report(self):
num_count = 0
i = 0
freq = {}
for term in self.indexer.terms_dict.keys():
if Parse.isFloat(term):
num_count += 1
freq[term] = self.indexer.terms_dict[term][1]
freq_list = sorted(freq.items(), key=itemgetter(1))
with open('frequency.txt', 'wb') as f:
for n in freq_list:
f.write(str(n[0]) + ": " + str(n[1]) + '\n')
print "Num of terms which are nums: " + str(num_count)
print "Num of countries: " + str(len(self.indexer.countries))
print "Num of capitals: " + str(self.indexer.num_of_capitals)
def set_save_path(self, dir_path):
self.save_path = dir_path
def save(self):
file_name = ''
if self.to_stem:
file_name += 's'
if self.with_semantics:
file_name += 's'
file_name = '\\' + file_name + 'results.txt'
with open(self.save_path + file_name, 'a+') as f:
for query_result in self.queries_docs_results:
for doc in query_result[2]:
line = " {} 0 {} 1 42.38 {}\n".format(
query_result[0], doc[0], 'rg')
f.write(line)
def get_cities_list(self):
if self.indexer is None:
return None
return self.indexer.cities_dict.keys()
def start_query_search(self, query, chosen_cities):
return self.searcher.search(query, chosen_cities)
def start_file_search(self, queries_path_entry, chosen_cities):
queries_list = []
current_queries_results = []
with open(queries_path_entry, 'rb') as f:
lines = f.readlines()
id = 0
i = 0
query = ''
narr = ''
while i < len(lines):
if '<num>' in lines[i]:
id = lines[i].split(':')[1].replace('\n', '')
elif '<title>' in lines[i]:
query = lines[i].replace('<title>', '').replace('\n', '')
elif '<desc>' in lines[i]:
i += 1
while not '<narr>' in lines[i]:
query = '{} {}'.format(
query, lines[i].replace('<title>',
'').replace('\n', ''))
i += 1
queries_list.append((id, query))
i += 1
for query_tuple in queries_list:
docs_result = self.start_query_search(query_tuple[1],
chosen_cities)
tmp = (query_tuple[0], query_tuple[1], docs_result)
current_queries_results.append(tmp)
self.queries_docs_results.append(tmp)
return self.queries_docs_results
def get_doc_five_entities(self, doc_id):
return self.searcher.docs_dict[doc_id].five_entities
@author: tina
'''
from Parser import Parser
from Indexing import Indexing
from Searcher import Searcher
import whoosh.index as index
# main function
if __name__ == '__main__':
collection = dict()
parser = Parser()
indexer = Indexing()
path = "/home/katherine/COMP479/info-retrieval/trunk/src/outputs/"
# Parse the documents
collection = parser.parse(path)
# Index the documents
indexer.process(collection)
print "Finish Indexing."
# Creater Searcher object.
searcher = Searcher()
# Query the index
while True:
query = raw_input("Please enter a query:")
searcher.search(query)
loop_again = raw_input("Enter another query? (y/n)")
if loop_again == 'n':
break
import sys
import time
from Searcher import Searcher
if __name__ == '__main__':
print("Initializing...")
path_to_index = sys.argv[1]
searcher = Searcher(path_to_index)
print("Enter your query:\n")
while 1:
print("QUERY: ", end='')
query = input()
print()
start_time = time.time()
searcher.processAndSearchQuery(query)
print("\nRESPONSE TIME: %s seconds" % (time.time() - start_time))
print("====================")
def main(argv):
global results
parser = argparse.ArgumentParser()
parser_action = parser.add_mutually_exclusive_group(required=True)
parser_action.add_argument(
"--store",
action='store_const',
const=True,
help="Load all images in path and save them in the DB")
parser_action.add_argument(
"--search",
action='store_const',
const=True,
help="Give an image path and search for most similar images")
parser.add_argument("filename",
help="Path to directory where to get images from")
args = parser.parse_args(argv[1:])
# Initialize the color descriptor
cd = ColorDescriptor((8, 12, 3))
# Initialize the shape descriptors
sd = ShapeDescriptor(32)
try:
connection = pq.connect(user="******",
password="******",
host="127.0.0.1",
port="5432",
database="imagesdb",
sslmode="disable")
except (Exception, pq.Error) as error:
print("Error while connecting to PostgreSQL", error)
# Ensure DB structure is present
cursor = connection.cursor()
cursor.execute(
"SELECT 1 FROM information_schema.tables WHERE table_schema = %s AND table_name = %s",
('public', 'files'))
result = cursor.fetchall()
# If db is empty create table
if len(result) == 0:
create_table_query = """
CREATE TABLE files (
id serial primary key,
orig_filename text not null,
color_descriptor numeric[] not null,
sift numeric[] not null,
surf numeric[] not null,
kaze numeric[] not null,
orb numeric[] not null
)
"""
cursor.execute(create_table_query)
connection.commit()
# Run the command
if args.store:
# Reads all files in path into memory.
path = args.filename + "/*.jpg"
for fname in glob.glob(path):
print(fname)
f = open(fname, 'rb')
image = cv2.imread(fname)
# describe the image by using our descriptors
try:
color_features = cd.describe(image)
kaze_features, orb_features = sd.describe(image)
except ValueError:
continue
cursor.execute(
"INSERT INTO files(id, orig_filename, color_descriptor, kaze, orb)"
"VALUES (DEFAULT,%s,%s,%s,%s) RETURNING id",
(fname, color_features, kaze_features, orb_features))
returned_id = cursor.fetchone()[0]
f.close()
connection.commit()
print("Stored {0} into DB record {1}".format(
args.filename, returned_id))
elif args.search:
# Fetches the file from the DB into memory then writes it out.
# Same as for store, to avoid that use a large object.
print(args.filename)
image = cv2.imread(args.filename)
# Initialize the searcher
method = 'kaze'
distance = 'euclidean'
# Number of nearest neighbors
limit = 10
if method == "color":
cr = connection.cursor()
sql = 'SELECT orig_filename, color_descriptor FROM files;'
cr.execute(sql)
tmp = cr.fetchall()
df = sqlio.read_sql_query(sql, connection)
# sampled_df = df.sample(n=100, random_state=42)
color_features = cd.describe(image)
searcher = Searcher(color_features,
method=method,
distance=distance,
limit=limit,
dataframe=df)
results = searcher.search()
else:
if method == "kaze":
nsd = ShapeDescriptor(64)
kaze_features, _ = nsd.describe(image)
cr = connection.cursor()
sql = 'SELECT orig_filename, kaze FROM files;'
cr.execute(sql)
tmp = cr.fetchall()
df = sqlio.read_sql_query(sql, connection)
# sampled_df = df.sample(n=100, random_state=42)
searcher = Searcher(kaze_features,
method,
distance,
limit=limit,
dataframe=df)
results = searcher.search()
if method == "orb":
nsd = ShapeDescriptor(128)
_, orb_features = nsd.describe(image)
cr = connection.cursor()
sql = 'SELECT orig_filename, orb FROM files;'
cr.execute(sql)
tmp = cr.fetchall()
df = sqlio.read_sql_query(sql, connection)
searcher = Searcher(orb_features,
method,
distance,
limit=limit,
dataframe=df)
results = searcher.search()
# Print the results in console
print(results)
# Load the query image and display it
cv2.imshow("Query", image)
# Loop over the results
for (score, resultID) in results:
# Load the result image and display it
result = cv2.imread("static/" + resultID)
cv2.imshow("Result", result)
cv2.waitKey(0)
connection.close()
def setUp(self):
self.search = Searcher()
from java.io import StringReader
from org.apache.lucene.analysis import TokenStream
from org.apache.lucene.index import IndexWriterConfig
from org.apache.lucene.util import Version
from org.apache.lucene.search import Explanation
lucene.initVM()
analyzer = Analyzer(Version.LUCENE_CURRENT)
config = IndexWriterConfig(Version.LUCENE_CURRENT, analyzer)
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE)
indexer = Indexer(config, '/home/hnguyen/Projects/CLIFinder/cli.index')
indexer.index('/home/hnguyen/Projects/CLIFinder/cli')
searcher = Searcher(analyzer, '/home/hnguyen/Projects/CLIFinder/cli.index')
while True:
strQuery = raw_input("Query:")
if strQuery == '':
sys.exit(1)
docs, query = searcher.search(strQuery, 'content', 'name')
print '"%s" has %s result(s)' % (strQuery, len(docs))
for d in docs:
print 'Score: %s \nFile: %s \nDesc: %s \n' % (d.score, searcher.mIndexSearcher.doc(d.doc).get('name'), searcher.mIndexSearcher.doc(d.doc).get('content'))
print "<================================================>"
def search():
if request.method == "POST":
RESULTS_ARRAY = []
image_names = []
# get url
image_url = request.files['file_image']
method = request.form.get('method')
distance = request.form.get('distance')
number_of_neighbors = request.form.get('knn_slider')
try:
# load the query image and describe it
img = io.imread(image_url)
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
results = list()
if method == "color":
cr = cn.cursor()
sql = 'SELECT orig_filename, color_descriptor FROM files;'
cr.execute(sql)
tmp = cr.fetchall()
df = sqlio.read_sql_query(sql, cn)
#sampled_df = df.sample(n=100, random_state=42)
cd = ColorDescriptor((8, 12, 3))
color_features = cd.describe(img)
searcher = Searcher(color_features, method=method, distance=distance, limit=int(number_of_neighbors),
dataframe=df)
results = searcher.search()
else:
sd = ShapeDescriptor(32)
kaze_features, orb_features = sd.describe(img)
if method == "kaze":
cr = cn.cursor()
sql = 'SELECT orig_filename, kaze FROM files;'
cr.execute(sql)
tmp = cr.fetchall()
df = sqlio.read_sql_query(sql, cn)
#sampled_df = df.sample(n=100, random_state=42)
searcher = Searcher(kaze_features, method, distance, limit=int(number_of_neighbors),
dataframe=df)
results = searcher.search()
if method == "orb":
cr = cn.cursor()
sql = 'SELECT orig_filename, orb FROM files;'
cr.execute(sql)
tmp = cr.fetchall()
df = sqlio.read_sql_query(sql, cn)
#sampled_df = df.sample(n=100, random_state=42)
searcher = Searcher(orb_features, method, distance, limit=int(number_of_neighbors),
dataframe=df)
results = searcher.search()
# loop over the results, displaying the score and image name
for (score, resultID) in results:
RESULTS_ARRAY.append(
{"image": str(resultID), "score": str(score)})
image_names.append(resultID)
# return success
return render_template("results.html", image_names=image_names[:int(number_of_neighbors)])
except:
# return error
jsonify({"sorry": "Sorry, no results! Please try again."}), 500
from Searcher import Searcher
# ---------------
# Set airports and dates for your trip below
departure_airpts_togo = ['SAO', 'BSB']
arrival_airports_togo = ['SEL']
departure_airpts_back = ['SEL']
arrival_airports_back = ['SAO', 'BSB']
when_to_go = [
'2021-05-01', '2021-05-02', '2021-05-03', '2021-05-04', '2021-05-05'
]
when_to_return = [
'2021-05-22', '2021-05-23', '2021-05-24', '2021-05-25', '2021-05-26'
]
searcher = Searcher(departure_airpts_togo, arrival_airports_togo,
departure_airpts_back, arrival_airports_back, when_to_go,
when_to_return)
searcher.startSearching()
import cv2
from Searcher import Searcher
from hist import RGBHist
ap = argparse.ArgumentParser()
ap.add_argument("-q", "--query", required=True,
help="Path to the query image")
ap.add_argument("-d", "--dataset", required=True,
help="Path to the dataset")
ap.add_argument("-i", "--index", required=True,
help="Path to the index")
args = vars(ap.parse_args())
index = pickle.load(open(args["index"], 'rb'))
searcher = Searcher(index)
path = args["query"]
queryImage = cv2.imread(path)
cv2.imshow("Query", queryImage)
print("query: %s" % path)
desc = RGBHist([8,8,8])
queryFeatures = desc.describe(queryImage)
results = searcher.search(queryFeatures)
for j in range(10):
(score, imageName) = results[j]
print("\t%d. %s : %.3f" % (j + 1, imageName, score))
for page in sorted(pageRanks[1]):
print '\t\t\t' + page,
for step, pageRank in sorted(pageRanks.iteritems()):
print
print 'Step ' + str(step) + '\t',
for page, rank in sorted(pageRank.iteritems()):
print '\t\t' + '{0:.4f}'.format(rank),
print
webGraph = frontier.getWebGraph(seedDocuments)
pageRank = pageRanker.getPageRank(webGraph)
pageContents = indexer.getPageToken(webGraph)
index = indexer.getIndex(pageContents)
searcher = Searcher(index, pageContents)
printWebGraph(webGraph)
printPageRanks(pageRank)
printPageContents(pageContents)
printIndex(index)
searcher.search('tokens')
searcher.searchAdvanced('tokens', pageRank)
print
searcher.search('index')
searcher.searchAdvanced('index', pageRank)
print
searcher.search('classification')
searcher.searchAdvanced('classification', pageRank)
print
def images():
return render_main(request, 'images.html', 'images')
@app.route('/images/search', methods=['POST', 'GET'])
def images_search():
return render_results(request, 'images_results.html', 'images')
@app.route('/visual', methods=['POST', 'GET'])
def visual():
if request.method == "POST":
return search_handle(request, 'image')
error = request.args.get('error')
return render_template('visual.html', error=error)
@app.route('/visual/search', methods=['POST', 'GET'])
def visual_search():
return render_results(request, 'visual_results.html', 'visual')
searcher = None # To be initialized in __main__
db_cursor = None # To be initialized in __main__
if __name__ == '__main__':
searcher = Searcher('localhost')
db = sqlite3.connect('data/MSN_technology.db', check_same_thread = False)
db_cursor = db.cursor()
app.run(debug=True, port=8080)