def test_find_article_by_url_should_return_list_of_results_from_cursor(self):
# Given
mock_cursor = ["someitem"]
mock_db = MagicMock()
mock_db.articles = MagicMock()
mock_db.articles.find = MagicMock(return_value=mock_cursor)
some_url = "http://www.someurl.com"
# When
result = MongoUtils.find_article_by_url(mock_db, some_url)
# Then
self.assertEqual(result, ["someitem"])
mock_db.articles.find.assert_called_with({"url": some_url})
def __init__(self):
config = ConfigFiles.config()
self.client = MongoUtils.create_client_from_config(config)
self.db = self.client.scrape
self.bulk = self.db.articles.initialize_ordered_bulk_op()
def __init__(self):
config = ConfigFiles.config()
self.client = MongoUtils.create_client_from_config(config)
self.db = self.client.scrape
self.bulk = self.db.words.initialize_ordered_bulk_op()
def GET(self):
client = MongoUtils.create_client_from_config(self.config)
db = client.scrape
words = MongoUtils.find_all_words(db)
client.close()
return words
from scrapymasters.common.MongoUtils import MongoUtils
from string import punctuation
config = {
'url': 'localhost:27017',
'username': "",
'password': "",
'dbname': "scrape",
'scrapeUrl': "http://www.bbc.com/"
}
client = MongoUtils.create_client_from_config(config)
db = client.scrape
bulk = db.words.initialize_ordered_bulk_op()
word = "sometimes"
word_query = { "word": word }
url = "http://www.bbc.com/news/world-latin-america-41168117"
url_to_insert = {"$addToSet": {"urls": url }}\
print(db.words.find(word_query).count())
print("=======")
bulk.find(word_query).update(url_to_insert)
result = bulk.execute()
print("Index write results:")
print(result)
def GET(self, word):
client = MongoUtils.create_client_from_config(self.config)
db = client.scrape
articles = MongoUtils.find_article_by_word(db, word)
client.close()
return articles
def GET(self):
client = MongoUtils.create_client_from_config(self.config)
db = client.scrape
articles = MongoUtils.find_all_articles(db)
client.close()
return articles
def GET(self):
client = MongoUtils.create_client_from_config(self.config)
db = client.scrape
words = MongoUtils.find_all_words(db)
client.close()
return words
def GET(self, word):
client = MongoUtils.create_client_from_config(self.config)
db = client.scrape
articles = MongoUtils.find_article_by_word(db, word)
client.close()
return articles
def GET(self):
client = MongoUtils.create_client_from_config(self.config)
db = client.scrape
articles = MongoUtils.find_all_articles(db)
client.close()
return articles