class HiddendataTestCase(unittest.TestCase): def setUp(self): result_dir = os.path.abspath(os.path.dirname(__file__) + os.path.sep + "../../") + '/example/dblp_result' self.hiddendata = HiddenData(result_dir, "row['info']['key']", ["row['info']['title']"]) def tearDown(self): self.hiddendata = None def test_proResult(self): result_raw = [{u'@score': u'2', u'info': { u'title': u'Radio Access Technology Selection in Heterogeneous Wireless Networks. (S\xe9lection de technologie d'acc\xe8s radio dans les r\xe9seaux sans-fil h\xe9t\xe9rog\xe8nes).', u'url': u'http://dblp.org/rec/phd/hal/ElHelou14', u'authors': {u'author': u'Melhem El Helou'}, u'key': u'phd/hal/ElHelou14', u'year': u'2014', u'type': u'Books and Theses'}, u'@id': u'709440', u'url': u'URL#709440'}] result_er = [( [u'radio', u'access', u'technology', u'selection', u'in', u'heterogeneous', u'wireless', u'networks', u's', u'lection', u'de', u'technologie', u'd', u'apos', u'acc', u's', u'radio', u'dans', u'les', u'r', u'seaux', u'sans', u'fil', u'h', u't', u'rog', u'nes'], u'phd/hal/ElHelou14')] assert self.hiddendata.proResult(result_raw) == result_er
result_file.csv match_file.pkl match_file.csv enriched_file.pkl enriched_file.csv """ sample_file = 'dblp_sample.csv' localdata_file = 'dblp_3881.csv' result_dir = 'dblp_result' sampledata = SampleData(sample_ratio=0.5, samplepath=sample_file, filetype='csv', uniqueid="key", querylist=["title"]) localdata = LocalData(localpath=localdata_file, filetype='csv', uniqueid="ID", querylist=['title'], matchlist=['title']) hiddendata = HiddenData(result_dir=result_dir, uniqueid="info.key", matchlist=["info.title"]) budget = 20 smartcrawl.smartCrawl(budget, dblp, sampledata, localdata, hiddendata) """ pool_thre = 2 jaccard_thre = 0.85 threads = 4 smartcrawl.smartCrawl(budget, dblp, sampledata, localdata, hiddendata, pool_thre, jaccard_thre, threads) """
yelp = SearchApi(client_id=client_id, client_secret=client_secret, top_k=300, delay=5, search_term=search_term, **parameters) """ \yelp_sample_AZ.pkl yelp_3000_AZ.csv yelp_result\\result_file.pkl result_file.csv match_file.pkl match_file.csv enriched_file.pkl enriched_file.csv """ sample_file = 'yelp_sample_AZ.pkl' localdata_file = 'yelp_3000_AZ.csv' result_dir = 'yelp_result' sampledata = SampleData(sample_ratio=0.5, samplepath=sample_file, filetype='pkl', uniqueid="business_id", querylist=["name"]) localdata = LocalData(localpath=localdata_file, filetype='csv', uniqueid="business_id", querylist=["name"], matchlist=["name", "full_address"]) hiddendata = HiddenData(result_dir=result_dir, uniqueid="id", matchlist=["name", "location.display_address.*"]) budget = 20 smartcrawl.smartCrawl(budget, yelp, sampledata, localdata, hiddendata) """ pool_thre = 2 jaccard_thre = 0.85 threads = 4 smartcrawl.smartCrawl(budget, yelp, sampledata, localdata, hiddendata, pool_thre, jaccard_thre, threads) """
format The result format of the search. Recognized values are "xml", xml ...?q=test&format=json "json", and "jsonp". h Maximum number of search results (hits) to return. For bandwidth 30 ...?q=test&h=100 reasons, this number is capped at 1000. f The first hit in the numbered sequence of search results (starting with 0) to return. In combination with the h parameter, 0 ...?q=test&h=100&f=300 this parameter can be used for pagination of search results. c Maximum number of completion terms (see below) to return. For 10 ...?q=test&c=0 bandwidth reasons, this number is capped at 1000. """ search_term = 'q' parameters = {'h': 1000} dblp = PublApi(delay=5, search_term=search_term, **parameters) """ \dblp_sample.pkl dblp_10000.pkl dblp_result\\result_file.pkl result_file.csv match_file.pkl match_file.csv """ sample_file = 'dblp_sample.pkl' localdata_file = 'dblp_10000.pkl' result_dir = 'dblp_result' sampledata = SampleData(samplepath=sample_file, filetype='pkl', uniqueid="row['key']", querylist=["row['title']"]) localdata = LocalData(localpath=localdata_file, filetype='pkl', uniqueid="row['key']", querylist=["row['title']"], matchlist=["row['title']"]) hiddendata = HiddenData(result_dir=result_dir, uniqueid="row['info']['key']", matchlist=["row['info']['title']"]) smartcrawl.smartCrawl(top_k, count, pool_thre, jaccard_thre, threads, budget, dblp, sampledata, localdata, hiddendata)
yelp_result\\result_file.pkl result_file.csv match_file.pkl match_file.csv """ sample_file = 'yelp_sample.pkl' localdata_file = 'yelp_3000.pkl' result_dir = 'yelp_result' sampledata = SampleData(samplepath=sample_file, filetype='pkl', uniqueid="row['id']", querylist=["row['name']"]) """ localdata = LocalData(localpath=localdata_file, filetype='pkl', uniqueid="row['business_id']", querylist=["row['name']"], matchlist=["row['name']", "row['full_address']"]) hiddendata = HiddenData(result_dir=result_dir, uniqueid="row['id']", matchlist=["row['name']", "' '.join(row['location']['display_address'])"]) """ localdata = LocalData(localpath='yelp_5882.csv', filetype='csv', uniqueid='ID', querylist=['NAME'], matchlist=['NAME', 'ADDRESS']) hiddendata = HiddenData( result_dir=result_dir, uniqueid="row['id']", matchlist=["row['name']", "' '.join(row['location']['display_address'])"]) smartcrawl.smartCrawl(top_k, count, pool_thre, jaccard_thre, threads, budget, yelp, sampledata, localdata, hiddendata)
def setUp(self): result_dir = os.path.abspath( os.path.dirname(__file__) + os.path.sep + "../../") + '/csv_example/dblp_result' self.hiddendata = HiddenData(result_dir, "info.key", ["info.title"])
google = TextSearchApi(location='in+Toronto', top_k=60, delay=5, search_term=search_term, **parameters) """ \yelp_sample_Toronto.pkl yelp_3000_Toronto.csv google_result\\result_file.pkl result_file.csv match_file.pkl match_file.csv enriched_file.pkl enriched_file.csv """ sample_file = 'yelp_sample_Toronto.pkl' localdata_file = 'yelp_3000_Toronto.csv' result_dir = 'google_result' sampledata = SampleData(sample_ratio=0.5, samplepath=sample_file, filetype='pkl', uniqueid="business_id", querylist=["name"]) localdata = LocalData(localpath=localdata_file, filetype='csv', uniqueid="business_id", querylist=["name"], matchlist=["name", "full_address"]) hiddendata = HiddenData(result_dir=result_dir, uniqueid="place_id", matchlist=["name", "formatted_address"]) budget = 20 smartcrawl.smartCrawl(budget, google, sampledata, localdata, hiddendata) """ pool_thre = 2 jaccard_thre = 0.85 threads = 4 smartcrawl.smartCrawl(budget, google, sampledata, localdata, hiddendata, pool_thre, jaccard_thre, threads) """
def setUp(self): result_dir = os.path.abspath(os.path.dirname(__file__) + os.path.sep + "../../") + '/example/dblp_result' self.hiddendata = HiddenData(result_dir, "row['info']['key']", ["row['info']['title']"])