def main():
parser = argparse.ArgumentParser()
subparser = parser.add_subparsers(dest='subparser')
lexicon_argparser = subparser.add_parser("generate_lexicon")
lexicon_argparser.add_argument('--b_range', type=str, help="Batches numbers range start and end creating/updating lexicon from. For example 1,3")
lexicon_argparser.add_argument('--d', type=int, default=0, help="Print demo results.")
forward_index_argparser = subparser.add_parser("generate_forward_index")
forward_index_argparser.add_argument('--b_range', type=str, help="Batches numbers range start and end creating/updating forward index from. For example 1,3")
forward_index_argparser.add_argument('--d', type=int, default=0, help="Print demo results.")
inverted_index_argparser = subparser.add_parser("generate_inverted_index")
inverted_index_argparser.add_argument('--b', type=str, help="Forward Index Batches to create inverted_index from. Comma Separated.")
inverted_index_argparser.add_argument('--d', type=int, default=0, help="Print demo results.")
search = subparser.add_parser("search")
search.add_argument("--q", type=str, help="Search Query.")
args = parser.parse_args()
if args.subparser == 'generate_lexicon':
batche_range = list(map(int, args.b_range.split(",")))
generate_lexicon.main(*batche_range, demo=args.d)
elif args.subparser == 'generate_forward_index':
batche_range = list(map(int, args.b_range.split(",")))
generate_forward_index.main(*batche_range, demo=args.d)
elif args.subparser == 'generate_inverted_index':
batches = args.b.split(',')
generate_inverted_index.main(batches, demo=args.d)
elif args.subparser == 'search':
lexicon = Lexicon(config.LEXICON_PATH)
inverted_index = InvertedIndex(config.INVERTED_INDEX_BARRELS_PATH, config.INVERTED_INDEX_BARRELS_TEMP_PATH, len(lexicon), config.INVERTED_INDEX_BARREL_SIZE)
search = Search(lexicon, inverted_index)
print(search.search(args.q))
def search_results(request):
''' I search for substitute to the product requested. '''
form = RequestForm(request.GET)
if form.is_valid():
query = form.cleaned_data['user_request']
else:
return render(request, 'search/results.html', {'product': "None"})
search = Search()
result_infos = []
search_product = search.find_product(query)
if not search_product:
return render(request, 'search/results.html', {'product': "None"})
else:
product = search.product_infos(search_product)
for categorie in product['categories']:
result_info = search.find_substitute(search_product)
result_infos.extend(search.result_infos(result_info))
result_infos = [
i for n, i in enumerate(result_infos)
if i not in result_infos[n + 1:]
]
return render(request, 'search/results.html', {
'product': product,
'results': result_infos
})
def conceal(tweet_file, config, endword_index=False):
dicts = dictionaries.load_dictionaries(config)
print 'keywords (x) = ', config.x
print 'Essence len = ', config.essence_len
distillery = Distillery(config.essence_len, dicts.keywords)
search_engine = Search()
raw_data_words = open(tweets_path + tweet_file).read().split()
data_words = [
keyword for word in raw_data_words
for keyword in dicts.english["".join(
c for c in word.lower()
if c not in ('!', '.', ':', ',', '?', '"', '-'))]
]
if endword_index:
words = [dicts.keywords[endword_index]] * config.w
else:
words = [dicts.keywords[config.x - 1]] * config.w
collected_words = [(words, '', '')]
stats = WordsStats(config, tweet_file, collected_words)
try:
while True:
# Avoid inserting 3rd link word in data
# iteration_type = len(collected_words) % 10
iteration_type = 1
if iteration_type == 0:
insert_link_word_in_d = True
choose_new_link_word = False
elif iteration_type == 1:
first_link_word = 'This string is ignored'
insert_link_word_in_d = False
choose_new_link_word = True
else:
insert_link_word_in_d = False
choose_new_link_word = False
words, link, first_link_word = conceal_step(
data_words, words, first_link_word, insert_link_word_in_d,
choose_new_link_word, search_engine, distillery, dicts, stats)
collected_words.append((
words,
link,
first_link_word,
))
if not data_words:
break
except Exception:
print(traceback.format_exc())
t, v, tb = sys.exc_info()
# distillery.browser.close()
raise t, v, tb
print "collected words are: %s" % collected_words
return collected_words
def search(request):
query = request.GET.get('q') # default to an empty string if q not present
search = Search()
results = search.get_results(query=query, model=Line)
return render(request,
template_name='search.html',
context={
'query': query,
'results': results
})
async def search(
q: str = Query(None, max_length=280),
page: Optional[int] = Query(
None,
ge=eval(cfg["search"]["pg_range"])["ge"],
le=eval(cfg["search"]["pg_range"])["le"],
),
):
return Search()._query(q, page)
# Re-queries and populates database at scheduled time
# Use cron expression to set refresh rate
@aiocron.crontab(cfg["CRAWLER"]["refresh_rate"])
def list(request):
query = request.GET.get('q', None)
anonymous = request.GET.get('anonymous', False)
owner = None
if not anonymous:
owner = Account.get_by_user(request.user)
try:
search = Search(query, owner)
search.process()
except SearchException:
pass
notify_count = Notification.objects.filter(owner__user=request.user, viewed=False).count()
return render(request, 'search/list.html', {'search': search, 'notify_count': notify_count})
'''Entry point for Search class'''
from search.search import Search
from utils.utils import Utils
if __name__ == '__main__':
UTILS = Utils()
CONFIG = UTILS.get_config_file('config.ini')
SEARCH_KEYWORD = CONFIG.get('Search', 'search_keyword')
UTILS.user_path()
SEARCH = Search()
SEARCH.search(search_keyword=SEARCH_KEYWORD)
# -*- coding: utf-8 -*-
# __author__ = "zok"
# Date: 2019/2/28 Python: 3.7
from search.search import Search
from config import KEY
if __name__ == '__main__':
"""start"""
se = Search(KEY)
se.start()
from indexing.lexicon import Lexicon
from indexing.inverted_index import InvertedIndex
from search.search import Search
# flask app & Api
app = Flask(__name__)
api = Api(app)
cors = CORS(app)
app.config['CORS_HEADERS'] = 'Content-Type'
# Indexes
lexicon = Lexicon(config.LEXICON_PATH)
inverted_index = InvertedIndex(config.INVERTED_INDEX_BARRELS_PATH,
config.INVERTED_INDEX_BARRELS_TEMP_PATH,
len(lexicon), config.INVERTED_INDEX_BARREL_SIZE)
search = Search(lexicon, inverted_index)
# for handling searches
class Setup(Resource):
@cross_origin()
def get(self):
return render_template('index.html')
class Document(Resource):
def get(self, doc_id):
doc_id = int(doc_id[-7:])
batch = doc_id // 64 + 1