class HiddendataTestCase(unittest.TestCase):
def setUp(self):
result_dir = os.path.abspath(os.path.dirname(__file__) + os.path.sep + "../../") + '/example/dblp_result'
self.hiddendata = HiddenData(result_dir, "row['info']['key']", ["row['info']['title']"])
def tearDown(self):
self.hiddendata = None
def test_proResult(self):
result_raw = [{u'@score': u'2', u'info': {
u'title': u'Radio Access Technology Selection in Heterogeneous Wireless Networks. (S\xe9lection de technologie d'acc\xe8s radio dans les r\xe9seaux sans-fil h\xe9t\xe9rog\xe8nes).',
u'url': u'http://dblp.org/rec/phd/hal/ElHelou14', u'authors': {u'author': u'Melhem El Helou'},
u'key': u'phd/hal/ElHelou14', u'year': u'2014', u'type': u'Books and Theses'}, u'@id': u'709440',
u'url': u'URL#709440'}]
result_er = [(
[u'radio', u'access', u'technology', u'selection', u'in', u'heterogeneous', u'wireless', u'networks', u's',
u'lection', u'de', u'technologie', u'd', u'apos', u'acc', u's', u'radio', u'dans', u'les', u'r', u'seaux',
u'sans', u'fil', u'h', u't', u'rog', u'nes'], u'phd/hal/ElHelou14')]
assert self.hiddendata.proResult(result_raw) == result_er
result_file.csv
match_file.pkl
match_file.csv
enriched_file.pkl
enriched_file.csv
"""
sample_file = 'dblp_sample.csv'
localdata_file = 'dblp_3881.csv'
result_dir = 'dblp_result'
sampledata = SampleData(sample_ratio=0.5,
samplepath=sample_file,
filetype='csv',
uniqueid="key",
querylist=["title"])
localdata = LocalData(localpath=localdata_file,
filetype='csv',
uniqueid="ID",
querylist=['title'],
matchlist=['title'])
hiddendata = HiddenData(result_dir=result_dir,
uniqueid="info.key",
matchlist=["info.title"])
budget = 20
smartcrawl.smartCrawl(budget, dblp, sampledata, localdata, hiddendata)
"""
pool_thre = 2
jaccard_thre = 0.85
threads = 4
smartcrawl.smartCrawl(budget, dblp, sampledata, localdata, hiddendata, pool_thre, jaccard_thre, threads)
"""
yelp = SearchApi(client_id=client_id, client_secret=client_secret, top_k=300, delay=5, search_term=search_term,
**parameters)
"""
\yelp_sample_AZ.pkl
yelp_3000_AZ.csv
yelp_result\\result_file.pkl
result_file.csv
match_file.pkl
match_file.csv
enriched_file.pkl
enriched_file.csv
"""
sample_file = 'yelp_sample_AZ.pkl'
localdata_file = 'yelp_3000_AZ.csv'
result_dir = 'yelp_result'
sampledata = SampleData(sample_ratio=0.5, samplepath=sample_file, filetype='pkl', uniqueid="business_id", querylist=["name"])
localdata = LocalData(localpath=localdata_file, filetype='csv', uniqueid="business_id",
querylist=["name"],
matchlist=["name", "full_address"])
hiddendata = HiddenData(result_dir=result_dir, uniqueid="id",
matchlist=["name", "location.display_address.*"])
budget = 20
smartcrawl.smartCrawl(budget, yelp, sampledata, localdata, hiddendata)
"""
pool_thre = 2
jaccard_thre = 0.85
threads = 4
smartcrawl.smartCrawl(budget, yelp, sampledata, localdata, hiddendata, pool_thre, jaccard_thre, threads)
"""
format The result format of the search. Recognized values are "xml", xml ...?q=test&format=json
"json", and "jsonp".
h Maximum number of search results (hits) to return. For bandwidth 30 ...?q=test&h=100
reasons, this number is capped at 1000.
f The first hit in the numbered sequence of search results
(starting with 0) to return. In combination with the h parameter, 0 ...?q=test&h=100&f=300
this parameter can be used for pagination of search results.
c Maximum number of completion terms (see below) to return. For 10 ...?q=test&c=0
bandwidth reasons, this number is capped at 1000.
"""
search_term = 'q'
parameters = {'h': 1000}
dblp = PublApi(delay=5, search_term=search_term, **parameters)
"""
\dblp_sample.pkl
dblp_10000.pkl
dblp_result\\result_file.pkl
result_file.csv
match_file.pkl
match_file.csv
"""
sample_file = 'dblp_sample.pkl'
localdata_file = 'dblp_10000.pkl'
result_dir = 'dblp_result'
sampledata = SampleData(samplepath=sample_file, filetype='pkl', uniqueid="row['key']", querylist=["row['title']"])
localdata = LocalData(localpath=localdata_file, filetype='pkl', uniqueid="row['key']", querylist=["row['title']"],
matchlist=["row['title']"])
hiddendata = HiddenData(result_dir=result_dir, uniqueid="row['info']['key']", matchlist=["row['info']['title']"])
smartcrawl.smartCrawl(top_k, count, pool_thre, jaccard_thre, threads, budget, dblp, sampledata, localdata, hiddendata)
yelp_result\\result_file.pkl
result_file.csv
match_file.pkl
match_file.csv
"""
sample_file = 'yelp_sample.pkl'
localdata_file = 'yelp_3000.pkl'
result_dir = 'yelp_result'
sampledata = SampleData(samplepath=sample_file,
filetype='pkl',
uniqueid="row['id']",
querylist=["row['name']"])
"""
localdata = LocalData(localpath=localdata_file, filetype='pkl', uniqueid="row['business_id']",
querylist=["row['name']"],
matchlist=["row['name']", "row['full_address']"])
hiddendata = HiddenData(result_dir=result_dir, uniqueid="row['id']",
matchlist=["row['name']", "' '.join(row['location']['display_address'])"])
"""
localdata = LocalData(localpath='yelp_5882.csv',
filetype='csv',
uniqueid='ID',
querylist=['NAME'],
matchlist=['NAME', 'ADDRESS'])
hiddendata = HiddenData(
result_dir=result_dir,
uniqueid="row['id']",
matchlist=["row['name']", "' '.join(row['location']['display_address'])"])
smartcrawl.smartCrawl(top_k, count, pool_thre, jaccard_thre, threads, budget,
yelp, sampledata, localdata, hiddendata)
def setUp(self):
result_dir = os.path.abspath(
os.path.dirname(__file__) + os.path.sep +
"../../") + '/csv_example/dblp_result'
self.hiddendata = HiddenData(result_dir, "info.key", ["info.title"])
google = TextSearchApi(location='in+Toronto', top_k=60, delay=5, search_term=search_term, **parameters)
"""
\yelp_sample_Toronto.pkl
yelp_3000_Toronto.csv
google_result\\result_file.pkl
result_file.csv
match_file.pkl
match_file.csv
enriched_file.pkl
enriched_file.csv
"""
sample_file = 'yelp_sample_Toronto.pkl'
localdata_file = 'yelp_3000_Toronto.csv'
result_dir = 'google_result'
sampledata = SampleData(sample_ratio=0.5, samplepath=sample_file, filetype='pkl', uniqueid="business_id",
querylist=["name"])
localdata = LocalData(localpath=localdata_file, filetype='csv', uniqueid="business_id",
querylist=["name"],
matchlist=["name", "full_address"])
hiddendata = HiddenData(result_dir=result_dir, uniqueid="place_id",
matchlist=["name", "formatted_address"])
budget = 20
smartcrawl.smartCrawl(budget, google, sampledata, localdata, hiddendata)
"""
pool_thre = 2
jaccard_thre = 0.85
threads = 4
smartcrawl.smartCrawl(budget, google, sampledata, localdata, hiddendata, pool_thre, jaccard_thre, threads)
"""
def setUp(self):
result_dir = os.path.abspath(os.path.dirname(__file__) + os.path.sep + "../../") + '/example/dblp_result'
self.hiddendata = HiddenData(result_dir, "row['info']['key']", ["row['info']['title']"])