class TestPlanetaLudicoScrap(unittest.TestCase):
def setUp(self):
self.mock_collection = mock()
self.scrap = PlanetaLudicoScrap(self.mock_collection)
def _read_entries(self, _):
return MockKimonoPlanetaLudicoAPI.json
def test_when_scrapping_saves_one_element_per_entry_in_db(self):
#mock_scrap = mock(self.scrap)
#when(mock_scrap)._read_entries(any()).thenReturn(MockKimonoPlanetaLudicoAPI.json)
self.scrap._read_entries = self._read_entries
result = self.scrap.scrapListOfURL([URL("http://www.kimonolabs.com/api/6j0yuuni?apikey=4bac761d58acc84da9ccadf9e1ff2d8f", "Planeta Ludico")])
verify(self.mock_collection, 2).save(any())
#self.assertEqual(len(result), 2)
def test_when_read_an_entry_title_is_title_of_emtry(self):
result = self.scrap._build_entry(MockKimonoPlanetaLudicoAPI.entry_json)
self.assertEqual(result.title,
MockKimonoPlanetaLudicoAPI.entry_json['title']['text'])
def test_when_read_an_entry_date_is_date_of_emtry(self):
result = self.scrap._build_entry(MockKimonoPlanetaLudicoAPI.entry_json)
self.assertEqual(result.date,
MockKimonoPlanetaLudicoAPI.entry_json['date'])
def test_when_read_an_entry_link_is_link_of_emtry(self):
result = self.scrap._build_entry(MockKimonoPlanetaLudicoAPI.entry_json)
self.assertEqual(result.link,
MockKimonoPlanetaLudicoAPI.entry_json['title']['href'])
def test_when_read_an_entry_source_is_the_blog(self):
result = self.scrap._build_entry(MockKimonoPlanetaLudicoAPI.entry_json)
self.assertEqual(result.json()['location'],
MockKimonoPlanetaLudicoAPI.entry_json['source']['text'])
def setUp(self):
self.mock_collection = mock()
self.scrap = PlanetaLudicoScrap(self.mock_collection)
threads.setMsgPageLimit(250) # Nunca bajes este valor o perderas mensajes, al menos mantenlo igual
threads.scrapListOfURL(labsk_urls)
delta = datetime.now() - starttime
print "----------------------------------------------"
print "Total time: ", delta
print "Page limit ", threads.pagelimit, " Msg page limit ", threads.msgpagelimit
print str(listener)
mr = db.merge('link')
print str(mr)
print "Scrapping planeta ludico"
blogs = PlanetaLudicoScrap(db.blogs_collection())
listener = BlogListener()
blogs.setListener(listener)
blogs.scrapListOfURL(planetaludico_urls)
print "----------------------------------------------"
print str(listener)
#------------------------------------------------
# Build reports
print "Building reports"
def write(name, html_text):
path = '../webgui/templates/'
