def test_find_article_by_url_should_return_list_of_results_from_cursor(self): # Given mock_cursor = ["someitem"] mock_db = MagicMock() mock_db.articles = MagicMock() mock_db.articles.find = MagicMock(return_value=mock_cursor) some_url = "http://www.someurl.com" # When result = MongoUtils.find_article_by_url(mock_db, some_url) # Then self.assertEqual(result, ["someitem"]) mock_db.articles.find.assert_called_with({"url": some_url})
def __init__(self): config = ConfigFiles.config() self.client = MongoUtils.create_client_from_config(config) self.db = self.client.scrape self.bulk = self.db.articles.initialize_ordered_bulk_op()
def __init__(self): config = ConfigFiles.config() self.client = MongoUtils.create_client_from_config(config) self.db = self.client.scrape self.bulk = self.db.words.initialize_ordered_bulk_op()
def GET(self): client = MongoUtils.create_client_from_config(self.config) db = client.scrape words = MongoUtils.find_all_words(db) client.close() return words
from scrapymasters.common.MongoUtils import MongoUtils from string import punctuation config = { 'url': 'localhost:27017', 'username': "", 'password': "", 'dbname': "scrape", 'scrapeUrl': "http://www.bbc.com/" } client = MongoUtils.create_client_from_config(config) db = client.scrape bulk = db.words.initialize_ordered_bulk_op() word = "sometimes" word_query = { "word": word } url = "http://www.bbc.com/news/world-latin-america-41168117" url_to_insert = {"$addToSet": {"urls": url }}\ print(db.words.find(word_query).count()) print("=======") bulk.find(word_query).update(url_to_insert) result = bulk.execute() print("Index write results:") print(result)
def GET(self, word): client = MongoUtils.create_client_from_config(self.config) db = client.scrape articles = MongoUtils.find_article_by_word(db, word) client.close() return articles
def GET(self): client = MongoUtils.create_client_from_config(self.config) db = client.scrape articles = MongoUtils.find_all_articles(db) client.close() return articles