def search(): global index query = request.args.get('q') doc_set = request.args.get('d') doc_set_names = DocSet(index_directory).get_doc_set_names() if query: if doc_set != 'all': actual_query = u"text:%s doc_set:%s" % ( query, doc_set) else: actual_query = query searcher = Searcher(index) results = searcher.find_by_full_text(actual_query) if results: message = None else: message = 'No documents found.' return render_template("index.html", results = results, message = message, query = query, selected_doc_set = doc_set, doc_set_names = doc_set_names) else: return render_template("index.html", doc_set_names = doc_set_names)
def search(request, search=None, cats=None, scats=None): page_number = request.GET.get('page', 1) searcher = Searcher(search, cats, scats) snps = Post.objects.order_by('-posted') snps = searcher.filter_queryset(snps) searcher.unfilter_categories() selector = QuerySelector(snps, dict( download=DownloadNzbAction(), )) if request.method == 'POST': action_response = selector.apply_action(request) if isinstance(action_response, HttpResponse): return action_response paginator = Paginator( selector, settings.POST_PER_PAGE, allow_empty_first_page=True, orphans=0, ) if page_number == "last": page = paginator.page(paginator.num_pages) else: try: page = paginator.page(page_number) except InvalidPage, EmptyPage: raise Http404
def scrape(self): self.scrape_results = defaultdict(lambda: defaultdict(list)) for category_of_document_path in self.qd_dict: for file in self.qd_dict[category_of_document_path]: s = Searcher(category_of_document_path, file, self.qd_dict) s.search() self.scrape_results[os.path.split(category_of_document_path)[1]][os.path.splitext(file)[0]] = s.results print len(self.scrape_results)
def search(self, query): """ Search in the index for answer to query and return a list of relevant urls. """ self._logger.info('query: ' + query) searcher = Searcher() return searcher.query(query)
def find_torrent(self): print 'Start capturing torrent' searcher = Searcher(self.driver) items = [] while(True): searcher.find_torrents(items) time.sleep(60) self.driver.refresh()
class SearcherTests(unittest.TestCase): def setUp(self): self.alist = [2,3,4,5,6,11,25] self.searcher = Searcher() def test_binarySearch(self): self.assertEqual(True, self.searcher.binary_search(self.alist, 2)) self.assertEqual(True, self.searcher.binary_search(self.alist, 25)) self.assertEqual(True, self.searcher.binary_search(self.alist, 5)) self.assertEqual(False, self.searcher.binary_search(self.alist, 7)) self.assertEqual(False, self.searcher.binary_search(self.alist, 26))
def get_route(self, start, *args): self.searcher = Searcher(self.matrix) routes = {"total_distance": 0, "path": [], "subsections": []} for dest in args: # Generate tree, and get route to dest route = self.searcher.generate_tree(start)[dest] routes["subsections"].append(route) routes["path"].extend(route["path"]) routes["total_distance"] += route["dist"] start = dest return routes
def test_plant_forest_1(self): ''' Tests that a planting a forest leads to the correct number of correct_assertions and these cover the right number of edges. ''' graph = Graph('test', idify=False, verbose=False) searcher = Searcher(graph) model = searcher.build_model(verbosity=0) rule = Rule(('1927286',), (('6293378', 'out', (('7241965',), ())),)) model.plant_forest(rule) assert(len(rule.correct_assertions) == 1) assert(len(rule.get_edges_covered()) == 6) assert(rule.get_labels_covered() == {('7241965', '36240'), ('7241965', '6555563'), ('7241965', '2415820'), ('7241965', '879961'), ('7241965', '6341376'), ('7241965', '6175574')})
def search(self, query): """ Executes a query over an existing index and returns the number of relevant docs and an ordered list of search results. Input: query - string. Output: A tuple containing the number of relevant search results, and a list of tweet_ids where the first element is the most relavant and the last is the least relevant result. """ searcher = Searcher(self._parser, self._indexer, model=self._model) return searcher.search(query)
def _results_generator(self): sep = "------------------------------------------------------\n" def _update_next(is_next: bool): if is_next: s.read_pos += URLS_DISPLAYED else: s.read_pos -= URLS_DISPLAYED new_content = s.results_gen() T.delete('1.0', END) for url, snippet in new_content: T.insert("end", url + '\n' + sep + snippet + '\n\n\n') btn = Button(self.subroot, text="Next Page", command=lambda: _update_next(True)) btn_p = Button(self.subroot, text="Previous Page", command=lambda: _update_next(False)) T.window_create(END, window=btn_p) T.window_create(END, window=btn) T.insert(END, "\n") if (self.query != None): cursor = self.session.cursor() s = Searcher(cursor, self.query) #displayed_content=[('a','b')] displayed_content = s.results_gen() S = Scrollbar(self.subroot) T = Text(self.subroot) btn = Button(self.subroot, text="Next Page", command=lambda: _update_next(True)) btn_p = Button(self.subroot, text="Previous Page", command=lambda: _update_next(False)) S.pack(side=RIGHT, fill=Y) T.pack(side=LEFT, fill=BOTH, expand=1) #btn.pack(side=BOTTOM) S.config(command=T.yview) T.config(yscrollcommand=S.set) T.config(state=NORMAL) T.insert(END, "Number of results found: " + str(len(s.scores)) + '\n') for url, snippet in displayed_content: T.insert(END, url + '\n' + sep + snippet + '\n\n\n') T.window_create(END, window=btn_p) T.window_create(END, window=btn)
def search_results(): """ Displays search results ordered by pagerank. """ graph.loadExisting(graph_data) pageRank = PageRank(graph) searcher = Searcher(pageRank, indexer) search_value = request.args['searchVal'] vals = searcher.search(indexdir, search_value) form = SearchForm() return render_template('search_results.html', search_value=search_value, return_values=vals, form=form)
def main( email, api_key=None, query=None, mongo_url=None, download_all=False, limit=None, **kwargs, ): """ Given an e-mail and API key, searches for Human RNA and outputs the result """ logging.basicConfig(level=logging.INFO) searcher = Searcher(email, api_key=api_key, mongo_url=mongo_url) if query and query != "human": searcher.search(query, limit=limit, **kwargs) else: logging.info("No query supplied, searching for Human RNA") searcher.search_human_rna(limit=limit) if download_all: assert searcher.cached, "Searcher is not cached." logging.info(f"Found {len(searcher)} results") logging.info("Downloading all results") searcher.download_all() else: for result in searcher: print(json.dumps(result, indent=4, sort_keys=True))
def main(): searcher = Searcher("data.json") while True: print response = "n" print "==================================" while response != "y": response = raw_input("Ready to say something? (y/n)") q, filters = query_speech() if not q: continue print "Filters:" print filters print searcher.process_query(q, filters)
def __init__(self, config=None, run_config=None): if not config: config = ConfigClass() if not run_config: run_config = RunConfigClass() self._run_config = run_config self._config = config self._parser = Parse(run_config) self._indexer = Indexer(run_config) self._model = None self.searcher = Searcher(self._parser, self._indexer, run_config, model=self._model)
def main(args): try: if len(args) != 3: Main.print_with_help("Invalid number of parameters.\n") return 1 dirs = args[:-1] for directory in dirs: if not os.path.isdir(directory): Main.print_with_help("'{}' is not directory.\n".format(directory)) return 1 elif not len(os.listdir(directory)): print("'{}' directory is empty.\n".format(directory)) return 1 search_type = args[-1] # using of strategy pattern if search_type == '-n': searcher = Searcher(SearchByName()) elif search_type == '-c': searcher = Searcher(SearchByContent()) else: Main.print_with_help("Invalid parameter.\n") return 1 searcher.duplicate_search(dirs) searcher.print_results() except Exception as exp: print("Exception: " + str(exp)) return -1
def search(self, query): """ Executes a query over an existing index and returns the number of relevant docs and an ordered list of search results. Input: query - string. Output: A tuple containing the number of relevant search results, and a list of tweet_ids where the first element is the most relavant and the last is the least relevant result. """ searcher = Searcher(parser=self._parser, indexer=self._indexer, correction=True) n_relevant, ranked_doc_ids = searcher.search(query) return n_relevant, [id for (id, rank) in ranked_doc_ids]
class TestSearcher(TestCase): def setUp(self): self.searcher = Searcher() def test_search_valid(self): self.assertIsInstance(self.searcher.search('13/04/2018', 'BCN', 'DUB'), str) def test_search_bad_date_format(self): with self.assertRaises(UnexpectedError): self.searcher.search('13.04.2018', 'BCN', 'DUB') def test_search_non_existing_flight(self): with self.assertRaises(FlightNotFoundError): self.searcher.search('13/04/2015', 'AAA', 'BBB')
def search_and_rank_query(query, inverted_index, k, stemming, output_path): p = Parse(stemming) query_as_list = [term.text.lower() for term in p.parse_sentence(query)] searcher = Searcher(inverted_index, os.path.join(output_path, PostingFile)) w_of_term_in_query = searcher.CalculateW(query_as_list) relevant_docs = searcher.relevant_docs_from_posting(list(w_of_term_in_query.keys())) ranked_docs = searcher.ranker.rank_relevant_doc(relevant_docs, w_of_term_in_query) output = searcher.ranker.retrieve_top_k(ranked_docs, k) return output
def search(self, query): """ Executes a query over an existing index and returns the number of relevant docs and an ordered list of search results. Input: query - string. Output: A tuple containing the number of relevant search results, and a list of tweet_ids where the first element is the most relavant and the last is the least relevant result. """ searcher = Searcher(self._parser, self._indexer, model=self._model) # spell checker query_as_list = self._parser.parse_sentence(query) inverted_idx = self.indexer.inverted_idx spell = SpellChecker() misspelled = spell.unknown(query_as_list) assist = [x.lower() for x in query_as_list] # all the query terms in lower case for word in misspelled: if word.upper() in inverted_idx.keys() or word.lower( ) in inverted_idx.keys() or ' ' in word: continue # if the word is in the inverted index- no correction need word_idx = assist.index(word) corrections = spell.edit_distance_1( word ) # list of all the suggested corrections with distance value 1 corrections_dict = {} # check if the suggested corrections is in inverted index and collect the frequency of each correction for correction in corrections: if correction.upper() in inverted_idx.keys(): corrections_dict[correction] = inverted_idx[ correction.upper()] if correction.lower() in inverted_idx.keys(): corrections_dict[correction] = inverted_idx[ correction.lower()] if corrections_dict: query_as_list[word_idx] = max( corrections_dict, key=corrections_dict.get ) # choose the most common correction else: query_as_list[word_idx] = spell.correction(word) new_query = ' '.join(query_as_list) relevant_docs = searcher.search(new_query) return relevant_docs
def search_and_rank_query(query, inverted_index, k, config=None): """ This function search for relevant docs according to the query and rank them :param query: :param inverted_index: :param k: :param config: :return: """ p = Parse(config.toStem) query_as_list = p.parse_sentence(query) searcher = Searcher(inverted_index, config) relevant_docs = searcher.relevant_docs_from_posting(query_as_list) ranked_docs = searcher.ranker.rank_relevant_doc(relevant_docs) return searcher.ranker.retrieve_top_k(ranked_docs, k)
def search(keywords): keyword_list = keywords.split(" ") collector = Collector() if collector.is_meiju_info_file_exist(): collector.read_all_meiju_info_from_file() else: collector.save_all_meiju_info() collector.write_all_meiju_info_to_file() searcher = Searcher() meiju_ename_list = searcher.search_meiju_list_by_english_name_keyword(collector, keyword_list) click.echo("Total %d Meiju is found. Following are the lists:" % len(meiju_ename_list)) for meiju_ename in meiju_ename_list: click.echo("%s" % meiju_ename) return
def search(self, query): """ Executes a query over an existing index and returns the number of relevant docs and an ordered list of search results. Input: query - string. Output: A tuple containing the number of relevant search results, and a list of tweet_ids where the first element is the most relavant and the last is the least relevant result. """ query_as_list = self._parser.parse_sentence(query, 0) original_query_list = query.split(" ") stop_words = stopwords.words('english') original_query_list = [w for w in original_query_list if w not in stop_words] # find long terms and upper case words counter = 0 while counter < len(original_query_list): len_term = 1 word = original_query_list[counter] if word.isupper(): # NBA if word.find("\n") != -1: word = word[:-1] if word.find(".") != -1: word = word[:-1] query_as_list.append(word) elif len(word) > 1 and re.search('[a-zA-Z]', word) and word[0].isupper(): # upper first char term = word if original_query_list.index(word) + 1 < len(original_query_list): index = original_query_list.index(word) + 1 while index < len(original_query_list): # find all term if len(original_query_list[index]) > 1 and re.search('[a-zA-Z]', original_query_list[index]) and \ original_query_list[index][0].isupper(): new_word2 = original_query_list[index][0] + original_query_list[index][ 1:].lower() # Donald Trump term += " " + new_word2 index += 1 len_term += 1 else: break if len_term > 1: query_as_list.append(term) counter += len_term wordNet = WordNet_ranker(query_as_list) wordNet_query = wordNet.extend_query() searcher = Searcher(self._parser, self._indexer, model=self._model) return searcher.search(wordNet_query) # TODO: add K results
class Server: # # Create the server, loading local resources including the index. # def __init__(self, app: Flask): self.app = app self.searcher = Searcher(DATABASE_INDEX_PATH, CITY_ZIPS_PATH) CORS(self.app) # # Start the server. # def run(self): self.app.run(debug=True) # # Central get request handler for the application. # def get(self) -> Response: parameters = self.get_parameters() hikes = self.searcher.search(parameters) return jsonify(hikes) # # Parses search parameters from the current request's query parameters and returns them. # def get_parameters(self) -> SearchParameters: return SearchParameters(name=get_string_parameter("name"), origin=get_string_parameter("origin"), distance=get_integer_parameter("distance"), rating=get_integer_parameter("rating"), min_length=get_integer_parameter("min_length"), max_length=get_integer_parameter("max_length"))
def Match(queryname,k): folder="static" queryImage = cv2.imread(os.path.join(folder,queryname)) desc = RGBHistogram([8, 8, 8]) # load the index perform the search #index = cPickle.loads(open("index.txt").read()) d = min(division(queryImage),7) folder='indexIMG' index = shelve.open(os.path.join(folder,'f%d.shelve'%d)) searcher = Searcher(index) queryFeatures = desc.describe(queryImage) results = searcher.search(queryFeatures) l=len(results) lastRes=results[:min(k,l)] return lastRes
def flask_search(image): cd = HogDescriptor() # load the query image and describe it print(image) image = cv2.imread(image) image = (255 - image) image = cv2.resize(image, (256, 256)) query = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) features = cd.describe_hog(query) # perform the search # args['index'] is not defined searcher = Searcher('hog_index.csv') results = searcher.search(features) print(results) return results
def server(config): """ Function to initialize the server using the config file @param config: Configuration file """ app.searcher = Searcher(config) return app
def search(self, query): """ Executes a query over an existing index and returns the number of relevant docs and an ordered list of search results. Input: query - string. Output: A tuple containing the number of relevant search results, and a list of tweet_ids where the first element is the most relavant and the last is the least relevant result. """ if self._indexer.inverted_idx == None: print("can't run query without inverted index been loaded") return searcher = Searcher(self._parser, self._indexer, model=self._model) return searcher.search(query)
def evaluate_encoder(encoder: nn.Module, test_loader: torch.utils.data.DataLoader, loss_fn: nn.TripletMarginLoss = None, writer: SummaryWriter = None, epoch: int = -1): with torch.no_grad(): encoder.eval() searcher = Searcher.get_simple_index(encoder.embedding_dim) embeddings_x = [] embeddings_y = [] epoch_losses = [] for step, (x, y_pos, y_neg) in enumerate(test_loader): x, y_pos, y_neg = x.cuda(), y_pos.cuda(), y_neg.cuda() x_enc = encoder(x) y_pos_enc = encoder(y_pos) y_neg_enc = encoder(y_neg) if loss_fn: loss_val = loss_fn(x_enc, y_pos_enc, y_neg_enc) epoch_losses.append(loss_val.item()) embeddings_x.append(x_enc) embeddings_y.append(y_pos_enc) print(' Test batch {} of {}'.format(step + 1, len(test_loader)), file=sys.stderr) embeddings_x = torch.cat(embeddings_x, dim=0) embeddings_y = torch.cat(embeddings_y, dim=0) searcher.add(embeddings_x) lookup = searcher.search(embeddings_y, 100) correct_100 = sum(y in x for y, x in enumerate(lookup[1])) / len(lookup[1]) correct_50 = sum(y in x[:50] for y, x in enumerate(lookup[1])) / len(lookup[1]) correct_10 = sum(y in x[:10] for y, x in enumerate(lookup[1])) / len(lookup[1]) correct_1 = sum(y == x[0] for y, x in enumerate(lookup[1])) / len(lookup[1]) print(f'Test loss: {np.mean(epoch_losses):.4f}') print( 'Test accuracy:\n top1 {}\n top10 {}\n top50 {}\n top100 {}' .format(correct_1, correct_10, correct_50, correct_100)) if writer: writer.add_scalars('Accuracy', { 'top1': correct_1, 'top10': correct_10, 'top50': correct_50, 'top100': correct_100, }, global_step=epoch) writer.add_scalar('Loss/test', np.mean(epoch_losses), global_step=epoch) if epoch == -1 or epoch % 5 == 1: mat = torch.cat([embeddings_x[:1000], embeddings_y[:1000]], dim=0) labels = list(range(1000)) + list(range(1000)) writer.add_embedding(mat, labels, tag='Embeddings', global_step=epoch) return correct_1 * 100, correct_100 * 100, lookup, embeddings_x, embeddings_y
class RuntimePredict: """Input term, return predict scores.""" def __init__(self): # Mongodb client = MongoClient("mongodb://localhost:27017/") db = client.invertedindex # mongodb collections self._char_index = db.char_index self._doc_index = db.doc_index self._searcher = Searcher() def predict(self, term): docIdGen = self._searcher.searchIdGenerator(term) scoresList = [] for docId in docIdGen: doc = self._doc_index.find_one({"_id": ObjectId(docId)}) scoresList.append(doc["score"]) doc_nb = len(scoresList) scoreDict = {} # Sum label value in from all doc scores. for scores in scoresList: for score in scores: if score[0] not in scoreDict: scoreDict[score[0]] = 0 scoreDict[score[0]] += score[1] # Calculate average. for label in scoreDict: scoreDict[label] = scoreDict[label] / doc_nb return scoreDict
def __init__(self, parent = None): super(MainWindow, self).__init__(parent) self.setupUi(self) self.setWindowTitle("Search My Workspace") # List to display search results self.searchResultsModel = SearchResultsModel() self.listView_result.setModel(self.searchResultsModel) searchResultsDelegate = SearchResultsDelegate(self.listView_result) self.listView_result.setItemDelegate(searchResultsDelegate); # Tree to display facets self.facetModel = FacetModel() self.treeView_facet.setModel(self.facetModel) self.facetModel.modelReset.connect(self.treeView_facet.expandAll) # searcher self.searcher = Searcher() self.searcher.searchDone.connect(self.searchDone) # signal slots self.createConnections()
def search_cut(self): # load the query image and describe it query = cv2.imread("out.jpg") features = extract_features(query) # perform the search searcher = Searcher("index6.csv") results = searcher.search(features) # loop over the results for (score, resultID) in results: # load the result image and display it result = cv2.imread("dataset" + "/" + resultID) cv2.imshow("Result", result) cv2.waitKey(0)
def get_candidates( samples: torch.Tensor, small_encoder: SmallEncoder, large_encoder: LargeEncoder, searcher: Searcher, database: sqlite3.Connection ) -> Tuple[List[List[Tuple[float, int]]], Dict[int, Tuple[str, int, np.ndarray, np.ndarray]]]: samples_s = transform_melspectrogram(samples, large=False).cuda() samples_l = transform_melspectrogram(samples, large=True).cuda() enc_s = small_encoder(samples_s).cpu() enc_l = large_encoder(samples_l).cpu() lookup = searcher.search(enc_s, k=100)[1] candidates_ids = np.unique(lookup) candidates = database.execute( 'select id, name, offset, s_hash, l_hash from samples where id in ({});' .format(','.join(str(x) for x in candidates_ids))).fetchall() candidates = { idx: (title, offset, np.frombuffer(hash_s, dtype='float32'), np.frombuffer(hash_l, dtype='float32')) for idx, title, offset, hash_s, hash_l in candidates } best_matches = [] for knn, e_l in zip(lookup, enc_l.numpy()): dists_l = [(((candidates[v][3] - e_l)**2).mean(), v) for v in knn] dists_l = sorted(dists_l, key=itemgetter(0)) best_matches.append(dists_l) return best_matches, candidates
def gensimProcess(stemming): from six import iteritems dictionary = corpora.Dictionary(line.lower().split() for line in open('/tmp/gensim_docs.txt')) stop_list = stopList(stemming) print( "\tFilter tokens matching with a stop list and tokens present in all documents" ) stop_ids = [ dictionary.token2id[stop_w] for stop_w in stop_list if stop_w in dictionary.token2id ] once_ids = [ tokenid for tokenid, docfreq in iteritems(dictionary.dfs) if docfreq == 1 ] #Filter dictionnary from stop word and token uses only one. dictionary.filter_tokens(stop_ids + once_ids) if args.memory_care: print('filter extremes') dictionary.filter_extremes(no_below=8, no_above=0.4, keep_n=80000) dictionary.compactify() corpus_memory_friendly = MyCorpus(dictionary) corpus = list() print("\tVectorize with tfidf model") tfidf = models.TfidfModel(corpus_memory_friendly) for vector in corpus_memory_friendly: corpus.append(tfidf[vector]) #if you want to save corpus #corpora.BleiCorpus.serialize('/tmp/corpus.lda-c', corpus) #corpus = corpora.BleiCorpus('/tmp/corpus.lda-c') if args.fit_params: print('\tInstantiate searcher') #params_alpha=[0.01, 0.1, 0.25] nums_topics = [20, 50, 100, 175, 300, 400, 500] params_alpha = [0.01, 0.1, 0.25, 0.5, 0.75, 1.00] searcher = Searcher(nums_topics, params_alpha, corpus, dictionary) searcher.search() else: lda = models.LdaMulticore(corpus, id2word=dictionary, num_topics=500, iterations=50, passes=25, alpha=1, eta=0.01) lda.save('lda.model')
def search(self, query, k=None): """ Executes a query over an existing index and returns the number of relevant docs and an ordered list of search results. Input: query - string. Output: A tuple containing the number of relevant search results, and a list of tweet_ids where the first element is the most relavant and the last is the least relevant result. """ query_as_tuple = self._parser.parse_sentence(query) query_as_list = query_as_tuple[0] + query_as_tuple[1] searcher = Searcher(self._parser, self._indexer, model=self._model) extended_query = self.expand_query(query_as_list) return searcher.search_with_extension(query_as_list, extended_query, k)
def setUp(self): """ Setup search engine that will be subjected to the tests. """ self.twitter = Twitter(CUR_DIR + "/test_crossfit.tweets", CUR_DIR + "/test_stop_words.txt") self.twitter.load_tweets_and_build_index() self.searcher = Searcher(self.twitter.tweets, self.twitter.stop_words)
def __init__(self, config=None): self._config = config # self._parser = Parse() self._parser = Parse(self._config) self._indexer = Indexer(self._config) self._ranker = Ranker() self._model = None self._searcher = Searcher(self._parser, self._indexer)
def __init__(self, access_course, access_note, access_user, access_enrollment): """ to initialize a submit note instance """ self.access_course = access_course self.access_note = access_note self.access_user = access_user self.access_enrollment = access_enrollment self.searcher = Searcher(self.access_course, self.access_note, self.access_user) self.userControl = userControl = UserControl(self.access_user, self.access_enrollment)
def search(img): # initialize the image descriptor cd = ColorDescriptor((8, 12, 3)) # load the query image and describe it #query = cv2.imread(args["query"]) #query = cv2.imread("queries/2.png") features = cd.describe(img) # perform the search searcher = Searcher("feature.csv") results = searcher.search(features) return results
class Manager: def __init__(self): self.client = docker.from_env(timeout=86400) self.preparer = Preparer() self.searcher = Searcher() self.trainer = Trainer() self.interactor = Interactor() self.generate_save_tag = lambda tag, save_id: hashlib.sha256( (tag + save_id).encode()).hexdigest() def set_preparer_config(self, preparer_config): self.preparer.set_config(preparer_config) def set_searcher_config(self, searcher_config): self.searcher.set_config(searcher_config) def set_trainer_config(self, trainer_config): self.trainer.set_config(trainer_config) def set_interactor_config(self, interactor_config): self.interactor.set_config(interactor_config) def prepare(self, preparer_config=None): if preparer_config: self.set_preparer_config(preparer_config) self.preparer.prepare(self.client, COLLECTION_PATH_GUEST, self.generate_save_tag) def search(self, searcher_config=None): if searcher_config: self.set_searcher_config(searcher_config) self.searcher.search(self.client, OUTPUT_PATH_GUEST, TOPIC_PATH_HOST, TOPIC_PATH_GUEST, TEST_SPLIT_PATH_GUEST, self.generate_save_tag) def train(self, trainer_config=None): if trainer_config: self.set_trainer_config(trainer_config) self.trainer.train(self.client, TOPIC_PATH_GUEST, TEST_SPLIT_PATH_GUEST, VALIDATION_SPLIT_PATH_GUEST, self.generate_save_tag) def interact(self, interactor_config=None): if interactor_config: self.set_interactor_config(interactor_config) self.interactor.interact(self.client, self.generate_save_tag)
def test_searcher_returns_dictionary(searcher: Searcher, mocker): """ The searcher must return a json formatted SRA resultset """ mocker.patch("Bio.Entrez.esearch") Entrez.esearch.return_value = io.StringIO("{}") result = searcher.search("Human", max_results=3) assert isinstance(result, dict)
def plotAllFreq(X_train, y_train, X_test, y_test, min_n=2, max_n=15): ''' Plot Precision, Recall / frequency for all algo in algos.py/algoProd :param X_train: embedding for build indices/build graph. Vector (n,m) :param y_train: corresponding class for X_train. Vector (n, ) :param X_test: embedding for test. Vector (k,m) :param y_test: corresponding class for X_test. Vector (k, ) :param min_n: min number of nearest neighbor :param max_n: max number of nearest neighbor :return: nothing ''' fig_precision, ax_precision = plt.subplots() ax_precision.set(ylabel='Item per second', xlabel='Precision', title='Precision / Item per second(1/sec)') ax_precision.grid() fig_recall, ax_recall = plt.subplots() ax_recall.set(ylabel='Item per second', xlabel='Recall', title='Recall / Item per second (1/sec)') ax_recall.grid() algos = [] for algo in algosProd: try: algos.append(Searcher(X_train, y_train, algo)) except: print('Algo init failed :(', algo) pass for searcher in algos: print('Start algo test:', searcher.name) (precision, recall, freq) = getNeighborFreq(searcher, X_test, y_test, min_n, max_n) ax_precision.plot(precision, freq, '-', alpha=0.6, label=searcher.name, linewidth=2, ms=5, mew=1, lw=2) ax_recall.plot(recall, freq, '-', alpha=0.6, label=searcher.name, linewidth=2, ms=5, mew=1, lw=2) ax_recall.legend(loc='best') ax_precision.legend(loc='best') ax_precision.grid(b=True, which='major', color='0.65', linestyle='-') ax_recall.grid(b=True, which='major', color='0.65', linestyle='-') fig_precision.savefig('precision_time.png', dpi=600) fig_recall.savefig('recall_time.png', dpi=600)
class SearcherTests(unittest.TestCase): """ Test case for SearchEngine class. """ def setUp(self): """ Setup search engine that will be subjected to the tests. """ self.twitter = Twitter(CUR_DIR + "/test_crossfit.tweets", CUR_DIR + "/test_stop_words.txt") self.twitter.load_tweets_and_build_index() self.searcher = Searcher(self.twitter.tweets, self.twitter.stop_words) def test_indexed_doc_count(self): self.assertEqual(self.searcher.count(), 10) def test_existent_term_search(self): """ Test if search is correctly performed. """ results = self.searcher.search("coach") expected_results = 3 self.assertEqual(results[0].indexable.docid, expected_results) def test_non_existent_term_search(self): """ Test if search is correctly performed. """ expected_results = [] results = self.searcher.search("asdasdasdas") self.assertListEqual(results, expected_results) def test_search_result_limit(self): """ Test if search results can be limited. """ results = self.searcher.search("crossfit", 1) expected_results = 6 self.assertEqual(results[0].indexable.docid, expected_results)
def __init__(self, my_audio_album): wx.Frame.__init__(self, None) self.audio_album = my_audio_album splitter = wx.SplitterWindow(self) p = wx.Panel(splitter) self.nb = wx.Notebook(p) self.player = PlayerTab(splitter, my_audio_album) self.song_tab = SongTab(self.nb, my_audio_album.songs, self.player) self.song_tab.SetSize(600, 600) self.artist_tab = ArtistTab(self.nb, self.player, my_audio_album.artists) self.albums_tab = HalfSplittedTab(self.nb, my_audio_album.albums, self.player, "Choose an album to listen") self.genres_tab = HalfSplittedTab(self.nb, my_audio_album.genres, self.player, "Choose a genre to listen") self.playlist_tab = PlaylistTab(self.nb, my_audio_album.playlists, self.player, "Choose a playlist to listen", my_audio_album) self.nb.AddPage(self.song_tab, "Songs") self.nb.AddPage(self.artist_tab, "Artists") self.nb.AddPage(self.albums_tab, "Albums") self.nb.AddPage(self.genres_tab, "Genres") self.nb.AddPage(self.playlist_tab, "Playlists") splitter.SplitHorizontally(p, self.player) splitter.SetSashGravity(0.8) sizer = wx.BoxSizer() sizer.Add(self.nb, 1, wx.EXPAND) p.SetSizer(sizer) tb = wx.ToolBar(self, -1) self.ToolBar = tb self.searcher = Searcher(my_audio_album) self.search = wx.SearchCtrl(tb, style=wx.TE_PROCESS_ENTER) tb.Bind(wx.EVT_TEXT_ENTER, self.on_search) tb.AddControl(self.search) img = wx.Bitmap(os.path.join(bitmapDir, "update.png")) tb.AddTool(23, 'update', img) tb.Bind(wx.EVT_TOOL, self.update) tb.Realize()
def load(self): if os.path.exists(self.file_path): f = open(self.file_path, "r") conn_map = pickle.load(f) f.close() return conn_map conn_maps = {} for name, conn in self.connections.items(): conn_map = {} searcher = Searcher(conn) searcher.search() conn_map["tables"] = searcher.tables conn_map["connection"] = searcher.connection conn_maps[name] = conn_map f = open(self.file_path, "w") pickle.dump(conn_maps, f) f.close() return conn_maps
class MainWindow(QtGui.QMainWindow, Ui_MainWindow): def __init__(self, parent = None): super(MainWindow, self).__init__(parent) self.setupUi(self) self.setWindowTitle("Search My Workspace") # List to display search results self.searchResultsModel = SearchResultsModel() self.listView_result.setModel(self.searchResultsModel) searchResultsDelegate = SearchResultsDelegate(self.listView_result) self.listView_result.setItemDelegate(searchResultsDelegate); # Tree to display facets self.facetModel = FacetModel() self.treeView_facet.setModel(self.facetModel) self.facetModel.modelReset.connect(self.treeView_facet.expandAll) # searcher self.searcher = Searcher() self.searcher.searchDone.connect(self.searchDone) # signal slots self.createConnections() ''' connect signal/slot pairs ''' def createConnections(self): self.pushButton_search.clicked.connect(self.searchResultsModel.clearMyModel) self.pushButton_search.clicked.connect(self.facetModel.clearMyModel) self.pushButton_search.clicked.connect(self.search) def search(self): query = self.lineEdit_search.text() self.searcher.search(query) def searchDone(self): self.searchResultsModel.handleSearchResults(self.searcher.getDocs()) self.facetModel.handleSearchResults(self.searcher.getFacets()) self.label_searchResult.setText('About {0} search results for [{1}]'.format(self.searcher.getHits(), self.searcher.getQuery()))
class MainWindow(QtGui.QMainWindow, Ui_MainWindow): def __init__(self, parent = None): super(MainWindow, self).__init__(parent) self.setupUi(self) self.searchResultsView = SearchResultsView(self.listView_result) self.searchResultsView.viewDetailedContent.connect(self.showDetailedContent) self.facetView = FacetView(self.treeView_facet) self.facetView.facetOptionChanged.connect(self.search) # searcher self.searcher = Searcher() self.searcher.searchDone.connect(self.searchDone) self.createConnections() def createConnections(self): self.pushButton_search.clicked.connect(self.searchResultsView.clear) self.pushButton_search.clicked.connect(self.facetView.clear) self.pushButton_search.clicked.connect(self.search) def search(self): query = self.lineEdit_search.text() options = self.facetView.getFacetSearchOptions() self.searcher.search(query, options) def showDetailedContent(self, fileLocation): myviewer = viewer.viewerSimpleFactory(fileLocation, self) myviewer.showDetailedContent() @QtCore.Slot(int) def searchDone(self, searchResultCount): if searchResultCount > 0: self.searchResultsView.handleSearchResults(self.searcher.getDocs(), self.searcher.getHighlighting()) self.facetView.handleSearchResults(self.searcher.getFacets()) self.label_searchResult.setText('About {0} search results for [{1}]'.format(searchResultCount, self.searcher.getQuery()))
def setUp(self): self.searcher = Searcher() self.mikeObj = Person("Mike", 15) self.mikeObj2 = Person("Mike", 99) self.mikeObj3 = Person("Mike", 15, "Michael") self.jouhan = Person("Jouhan", 22) self.jouhan1 = Person("Jouhan", 74) self.alex = Person("Alex", 15) self.tim = Person("Tim", 100) self.cat = Animal("Cat", "Red", True) self.fish1 = Animal("Fish", "Blue", False) self.fish2 = Animal("Fish", "Red", False) self.dog = Animal("Dog", "Black", True) self.human = Animal("Human", "White", True)
def main_search(self): cd = ColorDescriptor((8, 12, 3)) # load the query image and describe it query = cv2.imread('test_dataset/'+self.name) features = cd.describe(query) # perform the search searcher = Searcher(self.file_path) results = searcher.search(features) # display the query cv2.imshow("Query",query) # loop over the results for (score, resultID) in results: # load the result image and display it result = cv2.imread(self.data_path + "/" + resultID) r=300.0/result.shape[1] dim=(300,int(result.shape[0]*r)) resized = cv2.resize(result, dim, interpolation = cv2.INTER_AREA) cv2.imshow("RESULT", resized) cv2.waitKey(0) cv2.destroyAllWindows()
class SubmitNote: """The following can be used to submit a note""" def __init__(self, access_course, access_note, access_user, access_enrollment): """ to initialize a submit note instance """ self.access_course = access_course self.access_note = access_note self.access_user = access_user self.access_enrollment = access_enrollment self.searcher = Searcher(self.access_course, self.access_note, self.access_user) self.userControl = userControl = UserControl(self.access_user, self.access_enrollment) """in later iterations we will add a userID, and a file directly fileID, for now they are the same as a note""" def submit_note(self, start_date, end_date, notes, user_id, department, number, section=None): possible_courses = self.searcher.search_courses( CourseSearchParams(department=department, number=number, section=section) ) user = self.access_user.get_user(user_id) if len(possible_courses) == 1 and ( user.role == User.ROLE_ADMIN or ( user.role == User.ROLE_SUBMITTER and self.userControl.is_user_enrolled(user_id, possible_courses[0].id) == True ) ): notesUploadFile = self.access_note.store_notes_file(notes.file, notes.filename) note = Note(start_date, end_date, notesUploadFile, possible_courses[0].id, user_id, datetime.now()) """with our note object created, submit to our db.""" """our note is submitted""" return self.access_note.insert_note(note) else: # was not able to submit note return None def update_note_file(self, note_id, user_id, notes): user = self.access_user.get_user(user_id) note = self.access_note.get_note(note_id) note.notes = self.access_note.store_notes_file(notes.file, notes.filename) note.modified_by = user_id note.modified_on = datetime.now() return self.access_note.update_note(note) def remove_note(self, note_id): return self.access_note.delete_note(note_id)
def __init__(self, parent = None): super(MainWindow, self).__init__(parent) self.setupUi(self) self.searchResultsView = SearchResultsView(self.listView_result) self.searchResultsView.viewDetailedContent.connect(self.showDetailedContent) self.facetView = FacetView(self.treeView_facet) self.facetView.facetOptionChanged.connect(self.search) # searcher self.searcher = Searcher() self.searcher.searchDone.connect(self.searchDone) self.createConnections()
def main(): if POST_FILE.startswith("@"): print "Please set the POST_FILE variable" return if not POLL_FREQ: print "Please set the poll frequency. (should be atleast 45)" return s = Searcher(POLL_FREQ) s.add_subreddit("forhire") s.add_subreddit("jobbit") s.add_subreddit("jobs4Bitcoins") pynotify.init("Job Searcher") while True: new_posts = s.get_new_posts() if(new_posts): for subreddit in new_posts: if(len(new_posts[subreddit]) < 5): for post in new_posts[subreddit]: n = pynotify.Notification(subreddit.display_name, post) n.set_urgency(CRIT) n.show() print subreddit.display_name + " : " + post else: fp = open(POST_FILE, "a") fp.write(asctime()) for post in new_posts[subreddit]: fp.write(subreddit.display_name + " : " + post + "\n") fp.close() n = pynotify.Notification("New Posts!", "Too many new posts! New posts have been saved to ~/.new_posts.txt") n.set_urgency(CRIT) n.show() sleep(s.poll_int)
def __init__(self, eventmanager): self._eventmanager = eventmanager self._world = world = eventmanager._world self.burst_deferred_maker = self._world.burst_deferred_maker # shortcuts to BurstDeferred factory. NOTE: don't get wrapped by BehaviorActions - should they? self.make = self.burst_deferred_maker.make self.wrap = self.burst_deferred_maker.wrap self.succeed = self.burst_deferred_maker.succeed self._motion = world.getMotionProxy() self._speech = world.getSpeechProxy() #self._video = world.getALVideoDeviceProxy() self._imops = world.getImopsProxy() self._current_camera = None # This is based on our commands only - we are the only ones changing it self._current_fps = burst_consts.INITIAL_FRAME_PER_SECOND self._joint_names = self._world.jointnames self._journey = Journey(self) self._movecoordinator = self._world._movecoordinator self.currentCamera = CAMERA_WHICH_BOTTOM_CAMERA self._camera_switch_time = world.time self.tracker = Tracker(self) # Please remember to stop # Todo - Locking self.centerer = Centerer(self) # Please remember to stop self.searcher = Searcher(self) # all of these behaviors self.localizer = Localizer(self) # when you stop the InitialBehavior. Thank you. self.headControllers = [self.tracker, self.centerer, self.searcher] # we keep track of the last head bd self._current_head_bd = self.succeed(self) self._current_motion_bd = self.succeed(self) # slight changes between 1.3.8 and 1.2.0 if is_120: self.setWalkConfig = self.setWalkConfig_120 else: self.setWalkConfig = self.setWalkConfig_138
fooderList={} for data in dataList: fooderList[str(data[0])+'.jpg']=data; IDList =[] for obj in dataList: IDList.append(str(obj[0])) # perform the search searcher = Searcher(args["featureFile"],IDList) gS.close() results = searcher.search(features) # output index answer file with open(args["answerPath"]+"\\"+args["ID"]+".json",'wb') as f: # loop over the results # should make more beautiful json format ansIndex={} i=0; ansIndex["status"]='true'; ansIndex["numberOfImgs"]=len(results); for (score, imageName) in results:
#!/usr/bin/python print('Content-type: text/html\r\n\r') import cgi, json import cgitb; cgitb.enable() import datetime from searcher import Searcher cgitb.enable() # for troubleshooting #the cgi library gets vars from html data = cgi.FieldStorage() mySearcher = Searcher() html = "" query = data.getvalue('query') a = datetime.datetime.now() results = mySearcher.getSearchResults(query) b = datetime.datetime.now() c = b - a n = 0 # r[1][0] = rank # r[1][1] = url # r[1][2] = modified # r[1][3] = size # r[1][4] = dict{keyword: freq} # r[1][5] = array{parents} # r[1][6] = array{children} # set up data structure to hold json skeleton
def setUp(self): self.alist = [2,3,4,5,6,11,25] self.searcher = Searcher()
def matcher(Dataset,Query): dataset='uploads' query_image=Query index='index.csv' res=0 res1=0 cd=descriptor((8,12,3)) query=cv2.imread(query_image) gray=cv2.cvtColor(query,cv2.COLOR_BGR2GRAY) cv2.imwrite('grey.jpg',gray) query_grey=cv2.imread('grey.jpg') features=cd.describe(query) features_grey=cd.describe(query_grey) searcher=Searcher(index) results=searcher.search(features) results_grey=searcher.search(features_grey) #cv2.imshow("Query",query) for (score,resultID) in results: result=cv2.imread(dataset+resultID) #m=match(query_image,dataset+resultID) #cv2.imshow("Result",result) print(resultID) if score < 7.95: print(score) t=math.floor(score) res=100-t print(100-t) else: print(score) t=math.floor(score) res=t print(t+t) #cv2.waitKey(0) print("---------------grey---------------") for (score,resultID) in results_grey: result=cv2.imread(dataset+resultID) #cv2.imshow("Result",result) print(resultID) if score < 7.95: print(score) t=math.floor(score) res1=100-t print(100-t) else: print(score) t=math.floor(score) res1=t print(t+t) #cv2.waitKey(0) return (res,res1)
import argparse import cv2 import time # construct the argument parser and parse the arguments ap = argparse.ArgumentParser() ap.add_argument("-i", "--index", default='index.csv', help="Path to where the computed index will be stored") ap.add_argument("-l", "--limit", default=10, help="Query result limit") ap.add_argument("-d", "--dist", default='euclidean', help="Distance model") ap.add_argument("-q", "--query", required=True, help="Path to the query image") args = vars(ap.parse_args()) # initialize the image descriptor cd = ColorDescriptor((8, 12, 3)) # load the query image and describe it query = cv2.imread(args["query"]) features = cd.describe(query) # perform the search searcher = Searcher(args["index"]) results = searcher.search(features, int(args["limit"]), args["dist"]) # loop over the results for (score, resultID) in results: # load the result image and display it print resultID
args_parser = argparse.ArgumentParser(description='Search all files in a ' 'directory (including subdirectories) ' 'for a regex pattern') args_parser.add_argument('-p', '--regex-pattern', help='regex pattern to ' 'search for in the file(s) in the directory', required=True) args_parser.add_argument('-o', '--output-file', nargs='?', help='path to the ' 'output file where all matching lines will be written' ' to', default='matches.txt', type=str) args_parser.add_argument('-d', '--directory', nargs='?', type=str, help='directory of file(s) which will be scanned', default='./') args = args_parser.parse_args() searcher = Searcher() start_time = time.time() count = 0 result = searcher.search_for_pattern(args.directory, args.regex_pattern) with open(args.output_file, 'w+b') as file: for res in result: count += len(res) file.writelines(res) end_time = time.time() print('time elapsed:') utils.print_elapsed_time(start_time=start_time, end_time=end_time) print('#### found {0!s} matching line(s) ####'.format(count)) print('done!')
ap.add_argument("-r", "--result-path", required = True, help = "Path to the result path") ap.add_argument("-m", "--mask", required = True, help = "Path to image mask (same size as images) to break into a grid") ap.add_argument("-g", "--gridsize", required = True, help = "Dimension for a single width/height for the square grid mask") args = vars(ap.parse_args()) # initialize the image descriptor cd = ColorDescriptor((8, 12, 3),args["mask"],args["gridsize"]) # load the query image and describe it query = cv2.imread(args["query"]) features = cd.describe(query) # perform the search searcher = Searcher(args["index"]) scores,image_names = searcher.search(features) # display the query cv2.imshow("Query", query) # loop over the results for x in range(0,len(scores)): resultID = image_names[x] print "Similar image %s is %s, score: %s" %(x,resultID,scores[x]) # load the result image and display it result = cv2.imread(args["result_path"] + "/" + resultID) cv2.imshow("Result", result) cv2.waitKey(0)