def test_persistent():
try:
drop_db()
#Crawl local test.html, and save information to mongodb
master.main("test_url.txt", 0)
#Crawl local test.html and save info in memory
bot = cs.crawler(None, "test_url.txt")
bot.crawl(depth=0)
#compare in mem data with persistent data
_id = bot._word_id_cache['xsfd']
assert get_word_id('xsfd') == bot._word_id_cache['xsfd']
_doc_ids = bot._inverted_index[_id]
assert get_doc_ids(_id) == list(bot._inverted_index[_id])
except:
logging.exception("")
return False
return True
def hello():
main(request.args.get('location'))
return "Model Trained"
#!/usr/bin/env python
# encoding: utf-8
import master
import sys
if __name__ == "__main__":
intf = sys.argv[1]
master.main(intf)
def main():
fileName = sys.argv[1]
mrm = master.main(fileName)
server.main(mrm, fileName)
client.main(mrm, fileName)
#!/usr/bin/env python
# encoding: utf-8
import argparse
import sys
import master
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('intf', type=str, help="Network interface")
args = parser.parse_args()
master.main(args.intf)
