def __init__(self): config = ConfigFiles.config() self.client = MongoUtils.create_client_from_config(config) self.db = self.client.scrape self.bulk = self.db.words.initialize_ordered_bulk_op()
def __init__(self): config = ConfigFiles.config() self.client = MongoUtils.create_client_from_config(config) self.db = self.client.scrape self.bulk = self.db.articles.initialize_ordered_bulk_op()
def GET(self): client = MongoUtils.create_client_from_config(self.config) db = client.scrape words = MongoUtils.find_all_words(db) client.close() return words
from scrapymasters.common.MongoUtils import MongoUtils from string import punctuation config = { 'url': 'localhost:27017', 'username': "", 'password': "", 'dbname': "scrape", 'scrapeUrl': "http://www.bbc.com/" } client = MongoUtils.create_client_from_config(config) db = client.scrape bulk = db.words.initialize_ordered_bulk_op() word = "sometimes" word_query = { "word": word } url = "http://www.bbc.com/news/world-latin-america-41168117" url_to_insert = {"$addToSet": {"urls": url }}\ print(db.words.find(word_query).count()) print("=======") bulk.find(word_query).update(url_to_insert) result = bulk.execute() print("Index write results:") print(result)
def GET(self, word): client = MongoUtils.create_client_from_config(self.config) db = client.scrape articles = MongoUtils.find_article_by_word(db, word) client.close() return articles
def GET(self): client = MongoUtils.create_client_from_config(self.config) db = client.scrape articles = MongoUtils.find_all_articles(db) client.close() return articles
def GET(self): client = MongoUtils.create_client_from_config(self.config) db = client.scrape words = MongoUtils.find_all_words(db) client.close() return words
def GET(self, word): client = MongoUtils.create_client_from_config(self.config) db = client.scrape articles = MongoUtils.find_article_by_word(db, word) client.close() return articles
def GET(self): client = MongoUtils.create_client_from_config(self.config) db = client.scrape articles = MongoUtils.find_all_articles(db) client.close() return articles