示例#1
0
 def __init__(self):
     config = ConfigFiles.config()
     self.client = MongoUtils.create_client_from_config(config)
     self.db = self.client.scrape
     self.bulk = self.db.words.initialize_ordered_bulk_op()
示例#2
0
 def __init__(self):
     config = ConfigFiles.config()
     self.client = MongoUtils.create_client_from_config(config)
     self.db = self.client.scrape
     self.bulk = self.db.articles.initialize_ordered_bulk_op()
示例#3
0
 def GET(self):
     client = MongoUtils.create_client_from_config(self.config)
     db = client.scrape
     words = MongoUtils.find_all_words(db)
     client.close()
     return words
示例#4
0
from scrapymasters.common.MongoUtils import MongoUtils
from string import punctuation




config = {
            'url': 'localhost:27017',
            'username': "",
            'password': "",
            'dbname': "scrape",
            'scrapeUrl': "http://www.bbc.com/"
        }


client = MongoUtils.create_client_from_config(config)
db = client.scrape
bulk = db.words.initialize_ordered_bulk_op()
word = "sometimes"
word_query = { "word": word }
url = "http://www.bbc.com/news/world-latin-america-41168117"
url_to_insert = {"$addToSet": {"urls": url }}\

print(db.words.find(word_query).count())
print("=======")

bulk.find(word_query).update(url_to_insert)

result = bulk.execute()
print("Index write results:")
print(result)
示例#5
0
 def GET(self, word):
     client = MongoUtils.create_client_from_config(self.config)
     db = client.scrape
     articles = MongoUtils.find_article_by_word(db, word)
     client.close()
     return articles
示例#6
0
 def GET(self):
     client = MongoUtils.create_client_from_config(self.config)
     db = client.scrape
     articles = MongoUtils.find_all_articles(db)
     client.close()
     return articles
示例#7
0
 def GET(self):
     client = MongoUtils.create_client_from_config(self.config)
     db = client.scrape
     words = MongoUtils.find_all_words(db)
     client.close()
     return words
示例#8
0
 def GET(self, word):
     client = MongoUtils.create_client_from_config(self.config)
     db = client.scrape
     articles = MongoUtils.find_article_by_word(db, word)
     client.close()
     return articles
示例#9
0
 def GET(self):
     client = MongoUtils.create_client_from_config(self.config)
     db = client.scrape
     articles = MongoUtils.find_all_articles(db)
     client.close()
     return articles