def test_idf(self):
self.assertEqual(self.index_.idf("dog"), 1.0986122886681098)
self.parameters.params["repo_dir"] = '/scratch/index/indri_5_7/ap8889'
self.index_ = Index(self.parameters)
self.assertEqual(self.index_.document_count("first"), 0)
def test_document_count(self):
self.assertEqual(self.index_.document_count("dog"), 1)
self.parameters.params["repo_dir"] = '/scratch/index/indri_5_7/ap8889'
self.index_ = Index(self.parameters)
self.assertEqual(self.index_.document_count("emotional"), 2973)
def test_index_with_xml_libraries(self):
xml_libs = os.path.join(
env.RESOURCES_DIR,
'library'
)
db_dir_with_xml = os.path.join(
env.RESULTS_DIR,
'db_dir_with_xml')
scanner = Scanner(xml_libs)
scanner.scan(
self.suite_dir,
'robot',
db_dir_with_xml
)
index = Index(db_dir_with_xml, self.index_dir, self.xml_libs)
index.index_consturctor(self.resource_a_table_name)
files = os.listdir(self.index_dir)
self.assertEqual(len(files), 1)
with open(os.path.join(self.index_dir, files[0])) as f:
data = json.load(f)
self.assertTrue(
any(kw[2] == 'SwingLibrary' for kw in data['keywords'])
)
self.assertTrue(
any(kw[0] == 'Add Table Cell Selection' for kw in data['keywords'])
)
self.assertTrue(
any(kw[0] == 'Select From Popup Menu' for kw in data['keywords'])
)
def test_term_count(self):
self.assertEqual(self.index_.term_count("dog"), 2)
self.parameters.params["repo_dir"] = '/scratch/index/indri_5_7/ap8889'
self.index_ = Index(self.parameters)
self.assertEqual(self.index_.term_count("emotional"), 3515)
def info(self, fi='', pagenum=100):
info = FileUtil.readfile(fi)
keywords = PreDeal.seg(info)
# 1. 关键词提取
keys = jieba.analyse.textrank(info,
topK=10,
withWeight=False,
allowPOS=('ns', 'n', 'vn', 'v'))
# 2. 调用搜索引擎爬取相关网页
# 2.1 抓取链接
spider_link = SpiderLink(keys, self.root)
spider_link.crawl(pagenum)
# 2.2 抓取内容
filename = '_'.join(keys) + '.html'
spider_to = SpiderTo(filename)
spider_to.crawl()
# 3. 文本预处理,去重,去停用词,分词,保留url和关键词集合
p = PreDeal()
filepath = os.path.join(config.spidertext, '_'.join(keys))
prepath = os.path.join(config.prepapath, '_'.join(keys))
p.savetexts(filepath=filepath, prepath=prepath)
# 4. 构建索引, 并检索,得到包含关键词信息的网页
# 4.1 索引构建
indexpath = os.path.join(config.indexpath, '_'.join(keys))
idx = Index()
idx.build(datapath=prepath, indexpath=indexpath)
search = Search1(filename=fi, pindexp=indexpath)
# 4.2 搜索并保存
info_k = keywords[:]
num = search.retrieve(keywords=info_k)
return keywords, num
def __init__(self, parameters):
self.stop_words = set(nltk.corpus.stopwords.words('english'))
self.tokenizer = nltk.tokenize.RegexpTokenizer(r'\w+')
self.parameters = parameters
self.index_ = Index(self.parameters)
self.store_collection_if_not_exists()
input_ = self.parameters.params["lda"]["file_name"]
super().__init__(input_)
def test_check_if_exists_in_index(self):
self.parameters.params["repo_dir"] = '/scratch/index/indri_5_7/ap8889'
self.index_ = Index(self.parameters)
self.assertTrue(self.index_.check_if_exists_in_index("emotional"))
self.assertFalse(self.index_.check_if_exists_in_index("first"))
self.assertFalse(self.index_.check_if_exists_in_index("included"))
self.assertTrue(self.index_.check_if_exists_in_index("includes"))
def index_single(db_path, db_table, index_path, module_search_path,
libs_in_xml):
for path_ in module_search_path:
sys.path.append(path_)
if not path.exists(index_path):
makedirs(index_path)
index = Index(db_path=db_path,
index_path=index_path,
xml_libraries=libs_in_xml)
index.index_consturctor(table=db_table)
def test_tf(self):
doc_words = SimpleDocument(self.parameters).get_words(
"../configs/others/pride_and_prejudice_wiki.txt")
self.assertEqual(self.index_.tf("dog", doc_words), 0.5)
self.parameters.params["repo_dir"] = '/scratch/index/indri_5_7/ap8889'
self.index_ = Index(self.parameters)
self.assertEqual(self.index_.tf("emotional", doc_words), 0.5)
def setUp(self):
self.index_dir = os.path.join(
env.RESULTS_DIR,
'index_dir',
)
if os.path.exists(self.index_dir):
while os.path.exists(self.index_dir):
shutil.rmtree(self.index_dir)
sleep(0.1)
os.makedirs(self.index_dir)
self.index = Index(self.db_dir, self.index_dir)
def test_tfidf(self):
self.parameters.params["repo_dir"] = '/scratch/index/indri_5_7/ap8889'
self.index_ = Index(self.parameters)
doc_words = SimpleDocument(self.parameters).get_words(
"../configs/others/pride_and_prejudice_wiki.txt")
tfidf_1 = self.index_.tfidf('emotional', doc_words)
print(tfidf_1, file=sys.stderr)
self.assertEqual(tfidf_1, 2.0455597255490345)
with self.assertRaises(Exception) as context:
self.index_.tfidf('is', doc_words)
self.assertTrue(
'unigram "is" not exist. Probably was a stopword in indexing.' in
str(context.exception))
def test_obtain_term_ids_of_a_document(self):
self.parameters.params["repo_dir"] = '/scratch/index/indri_5_7/ap8889'
self.index_ = Index(self.parameters)
res = self.index_.obtain_term_ids_of_a_document(1)
self.assertEqual(
res,
('AP881107-0001',
(147, 771, 0, 78064, 26, 2828, 1283, 92, 126, 147, 175009, 159395,
771, 55, 0, 0, 2362, 26, 2828, 919, 0, 0, 115, 8, 461, 1624,
1826, 0, 35, 693, 1198, 0, 195412, 0, 724, 430, 621, 340, 0, 771,
0, 1502, 20649, 4327, 1620, 9, 247, 0, 0, 866, 0, 643, 0, 2828,
415, 101374, 1289, 2015, 276, 1246, 0, 24, 29, 586, 0, 272, 0,
856, 0, 101374, 1826, 2153, 0, 174, 693, 0, 195412, 0, 0, 158999,
1037, 0, 117013, 137162, 123, 157, 0, 4415, 159395, 0, 0, 2262,
0, 0, 56, 0, 101374, 251, 0, 0, 189, 101374, 0, 569, 0, 332,
3095, 1873, 0, 2974, 0, 0, 13, 63630, 0, 485, 461, 91464, 0, 0,
91, 0, 50405, 0, 0, 156, 159395, 0, 690, 0, 347, 0, 24, 4049, 0,
101374, 0, 0, 0, 771, 0, 0, 92, 126, 690, 131907, 609, 0, 56, 0,
88, 2222, 0, 0, 1624, 0, 160974, 0, 9, 0, 0, 436, 0, 2362, 273,
0, 774, 1620, 13, 0, 263, 0, 887, 339, 176, 0, 0, 0, 91, 0,
50405, 0, 1620, 0, 0, 0, 289, 0, 1202, 101374, 0, 254, 0, 4543,
8, 193, 91, 0, 979, 3597, 0, 3095, 0, 791, 0, 2768, 937, 0, 0,
264, 91, 0, 0, 461, 2464, 0, 9, 0, 0, 1333, 2198, 45622, 0, 4433,
0, 1624, 0, 678, 0, 0, 0, 0, 3790, 0, 40, 0, 0, 118, 135313,
1620, 727, 136295, 0, 0, 0, 3408, 0, 2362, 6, 0, 245, 0, 0, 494,
0, 415, 101374, 0, 2042, 435, 0, 0, 0, 0, 0, 586, 0, 347, 1461,
0, 0, 116, 0, 0, 354, 0, 0, 9, 145130, 0, 48310, 120515, 51, 0,
0, 1, 956, 540, 430, 32, 32, 0, 0, 0, 221, 340, 0, 9, 771,
101374, 0, 1431, 897, 123, 0, 248, 0, 0, 91, 0, 50405, 0, 919, 0,
114, 667, 0, 70387, 0, 62918, 0, 0, 101374, 261, 0, 0, 311, 212,
0, 0, 61, 0, 4322, 767, 144897, 0, 62, 0, 0, 91, 0, 50405, 0,
375, 8, 546, 0, 0, 26, 3527, 543, 3657, 3224, 0, 0, 0, 2223, 0,
533, 0, 1019, 367, 0, 674, 0, 165, 101374, 0, 0, 436, 0, 4322,
767, 144897, 0, 62, 0, 91, 0, 0, 445, 8, 0, 0, 550, 0, 0, 26, 91,
3533, 1512, 285, 0, 0, 543, 0, 0, 0, 0, 0, 192, 0, 0, 1798,
101374, 0, 328, 0, 26, 0, 2362, 45369, 1390, 0, 156, 167, 0, 0,
91, 0, 50405, 0, 791, 0, 28, 0, 0, 264, 4415, 1977, 0, 187, 15,
0, 0, 550, 1483, 273, 0, 247, 101374, 887, 0, 91, 0, 50405, 168,
0, 303, 0, 515, 34, 77704, 0, 156, 23, 701, 273, 114, 1056, 0, 0,
409, 490, 101374, 771, 897, 899, 0, 248, 0, 77704, 0, 0, 0, 0,
1502, 599, 463, 147, 415, 4327, 2512, 0, 0, 70387, 0, 91, 0,
50405, 221, 62, 0, 17, 41, 8, 1548, 0, 0, 3657, 3224, 550, 0, 0,
171, 0, 2597, 4327, 62, 4, 0, 512, 0, 0, 1, 73, 532, 7, 0, 0, 0,
192, 0, 3657, 3224, 0, 0, 0, 1548, 101374, 626, 918, 455, 0, 114,
219, 0, 0, 245, 3095, 0, 0, 333, 0)))
def test_obtain_text_of_a_document(self):
self.parameters.params["repo_dir"] = '/scratch/index/indri_5_7/ap8889'
self.index_ = Index(self.parameters)
res = self.index_.obtain_text_of_a_document(1)
self.assertEqual(
res, """
Public Order Minister Tassos Sehiotis
resigned Monday after a Greek-American banker indicted in a $30
million financial scandal fled the country, apparently aboard a
yacht.
The conservative opposition immediately demanded the resignation
of Premier Andreas Papandreou's socialist government, claiming it
was staging a cover-up.
Banker George Koskotas, 34, disappeared Saturday afternoon. A
police officer, speaking on condition of anonymity, said Koskotas
fled abroad on Sunday, apparently by yacht from the seaside village
of Megalo Pefko, 20 miles from Athens.
Sehiotis said a warrant had been issued for Koskotas' arrest. One
week ago, Koskotas was banned from leaving Greece pending the
outcome of an official enquiry into alleged financial irregularities
at the Bank of Crete, which he controls.
Sehiotis, whose ministry was responsible for police surveillance
of Koskotas, said he was resigning ``since such (public order
ministry) omissions ... create an issue of political sensitivity.''
The scandal has shaken the government because of accusations in
Greek newspapers that senior socialist officials were involved in
illegal deals set up by the Bank of Crete.
The socialists also have been criticized for permitting Koskotas
to build a multi-million dollar banking and media empire in Greece
since 1984 without adequate checks by the central bank on his
financial background.
The government last week pledged ``absolute clarity'' in
uncovering the scandal and warned there will be ``no pardons' for
members of the ruling Panhellenic Socialist Movement (PASOK) who may
be implicated.
``The Greek people are left with the conviction that George
Koskotas was spirited away so that he would not speak. The
responsibility goes all the way to the top of the government
pyramid,'' Constantine Mitsotakis, leader of the New Democracy main
opposition party party, said in a statement demanding the government
resign.
Koskotas was suspended Oct. 20 as chairman of the Bank of Crete
and indicted on five counts of forgery and embezzlement.
Last week Koskotas appeared before a district attorney on a
charge of forging documents purporting to show that the Bank of
Crete had $13 million invested with the American brokerage firm
Merrill Lynch.
He was not detained but given until Nov. 14 to prepare his
defense.
Koskotas also is accused of forging documents purporting to show
his bank had another $17 million in an account with an American
bank, Irving Trust Corp. Both U.S. firms have said they had no
record of the deposits.
Koskotas, who holds both American and Greek citizenship, bought a
controlling interest in the Bank of Crete in 1984 after working in
its central Athens branch for six years as an accountant.
Rival newspapers have claimed Koskotas illegally used Bank of
Crete money to fund his publishing group Grammi, which controls
three daily newspapers, five magazines and a radio station.
Koskotas resigned Oct. 29 as chairman of Grammi, the day after
the premier's son, Education Minister George Papandreou, denounced
as a forgery a Bank of Crete statement showing a $2.3 million
transfer to a Merrill Lynch account in his name.
The younger Papandreou showed reporters a letter from a New York
lawyer saying there was no record at Merrill Lynch of such a
transfer.
Koskotas' parents, brother, wife and five children all have left
Greece during the past week.
""")
def __init__(self, word2vec_, parameters):
self.word2vec_model = word2vec_.model
self.single_document = SimpleDocument(parameters)
self.index_ = Index(parameters)
self.stop_words = set(nltk.corpus.stopwords.words('english'))
def __init__(self, parameters):
self.parameters = parameters
self.enchant_dict = enchant.Dict("en_US")
self.stopwords = stopwords.words('english')
self.index_ = Index(self.parameters)
import sys
from datetime import datetime
from index.document import IndexDocument
from index.index import Index
from server.server import fields
from index.utils import kendal_tau
from metrics.utils import avg_sd
index = Index()
index.load()
def validate_queries(queries):
for i, query in enumerate(queries):
if not query[0] in fields:
print('{} is not a valid field (in query #{}). Try one of: {}'.format(query[0], i+1, str(fields)))
sys.exit(1)
def ranking_correlation(queries):
validate_queries(queries)
for query in queries:
ranking_tfidf = index.get_documents_for_query(query[0], query[1], query[2], True)
ranking_raw = index.get_documents_for_query(query[0], query[1], query[2], False)
correlation = kendal_tau(ranking_tfidf, ranking_raw)
print('For query [{}] on field [{}]: {} Kendal Tau correlation. Rankings with {} documents.'.format(query[1], query[0], correlation, query[2]))
def query_response_time(queries):
validate_queries(queries)
def __init__(self, parameters):
self.index_ = Index(parameters)
self.stop_words = set(nltk.corpus.stopwords.words('english'))
def setUp(self):
self.parameters = Parameters()
self.parameters.params["repo_dir"] = '../index/test_files/index'
self.index_ = Index(self.parameters)
def test_obtain_terms_of_a_document(self):
self.parameters.params["repo_dir"] = '/scratch/index/indri_5_7/ap8889'
self.index_ = Index(self.parameters)
res = self.index_.obtain_terms_of_a_document(1)
print(res, file=sys.stderr)
self.assertEqual(res, ('AP881107-0001', (
'minist', 'resign', '', 'gree', 'american', 'banker', 'escap',
'public', 'order', 'minist', 'tasso', 'sehioti', 'resign',
'mondai', '', '', 'greek', 'american', 'banker', 'indict', '', '',
'30', 'million', 'financi', 'scandal',
'fled', '', 'countri', 'appar', 'aboard', '', 'yacht', '',
'conserv', 'opposit', 'immedi', 'demand', '', 'resign', '',
'premier', 'andrea', 'papandr', 'socialist', 'govern', 'claim', '',
'', 'stage', '', 'cover', '', 'banker', 'georg', 'koskota', '34',
'disappear', 'saturdai', 'afternoon', '', 'polic', 'offic',
'speak', '', 'condit', '', 'anonym', '', 'koskota', 'fled',
'abroad', '', 'sundai', 'appar', '', 'yacht', '', '', 'seasid',
'villag', '', 'megalo', 'pefko', '20', 'mile', '', 'athen',
'sehioti', '', '', 'warrant', '', '', 'issu', '', 'koskota',
'arrest', '', '', 'ago', 'koskota', '', 'ban', '', 'leav', 'greec',
'pend', '', 'outcom', '', '', 'offici', 'enquiri', '', 'alleg',
'financi', 'irregular', '', '', 'bank', '', 'crete', '', '',
'control', 'sehioti', '', 'ministri', '', 'respons', '', 'polic',
'surveil', '', 'koskota', '', '', '', 'resign', '', '', 'public',
'order', 'ministri', 'omiss', 'creat', '', 'issu', '', 'polit',
'sensit', '', '', 'scandal', '', 'shaken', '', 'govern', '', '',
'accus', '', 'greek', 'newspap', '', 'senior', 'socialist',
'offici', '', 'involv', '', 'illeg', 'deal', 'set', '', '', '',
'bank', '', 'crete', '', 'socialist', '', '', '', 'critic', '',
'permit', 'koskota', '', 'build', '', 'multi', 'million', 'dollar',
'bank', '', 'media', 'empir', '', 'greec', '', '1984', '', 'adequ',
'check', '', '', 'central', 'bank', '', '', 'financi',
'background', '', 'govern', '', '', 'pledg', 'absolut', 'clariti',
'', 'uncov', '', 'scandal', '', 'warn', '', '', '', '', 'pardon',
'', 'member', '', '', 'rule', 'panhellen', 'socialist', 'movement',
'pasok', '', '', '', 'implic', '', 'greek', 'peopl', '', 'left',
'', '', 'convict', '', 'georg', 'koskota', '', 'spirit', 'awai',
'', '', '', '', '', 'speak', '', 'respons', 'goe', '', '', 'wai',
'', '', 'top', '', '', 'govern', 'pyramid', '', 'constantin',
'mitsotaki', 'leader', '', '', 'new', 'democraci', 'main',
'opposit', 'parti', 'parti', '', '', '', 'statement', 'demand', '',
'govern', 'resign', 'koskota', '', 'suspend', 'oct', '20', '',
'chairman', '', '', 'bank', '', 'crete', '', 'indict', '', 'five',
'count', '', 'forgeri', '', 'embezzl', '', '', 'koskota', 'appear',
'', '', 'district', 'attornei', '', '', 'charg', '', 'forg',
'document', 'purport', '', 'show', '', '', 'bank', '', 'crete', '',
'13', 'million', 'invest', '', '', 'american', 'brokerag', 'firm',
'merril', 'lynch', '', '', '', 'detain', '', 'given', '', 'nov',
'14', '', 'prepar', '', 'defens', 'koskota', '', '', 'accus', '',
'forg', 'document', 'purport', '', 'show', '', 'bank', '', '',
'17', 'million', '', '', 'account', '', '', 'american', 'bank',
'irv', 'trust', 'corp', '', '', 'firm', '', '', '', '', '',
'record', '', '', 'deposit', 'koskota', '', 'hold', '', 'american',
'', 'greek', 'citizenship', 'bought', '', 'control', 'interest',
'', '', 'bank', '', 'crete', '', '1984', '', 'work', '', '',
'central', 'athen', 'branch', '', 'six', 'year', '', '', 'account',
'rival', 'newspap', '', 'claim', 'koskota', 'illeg', '', 'bank',
'', 'crete', 'monei', '', 'fund', '', 'publish', 'group', 'grammi',
'', 'control', 'three', 'daili', 'newspap', 'five', 'magazin', '',
'', 'radio', 'station', 'koskota', 'resign', 'oct', '29', '',
'chairman', '', 'grammi', '', '', '', '', 'premier', 'son', 'educ',
'minist', 'georg', 'papandr', 'denounc', '', '', 'forgeri', '',
'bank', '', 'crete', 'statement', 'show', '', '2', '3', 'million',
'transfer', '', '', 'merril', 'lynch', 'account', '', '', 'name',
'', 'younger', 'papandr', 'show', 'report', '', 'letter', '', '',
'new', 'york', 'lawyer', 'sai', '', '', '', 'record', '', 'merril',
'lynch', '', '', '', 'transfer', 'koskota', 'parent', 'brother',
'wife', '', 'five', 'children', '', '', 'left', 'greec', '', '',
'past', '')))
def test_term(self):
self.parameters.params["repo_dir"] = '/scratch/index/indri_5_7/ap8889'
self.index_ = Index(self.parameters)
res = self.index_.term(147)
print(res, file=sys.stderr)
self.assertEqual(res, 'minist')