def test_get_website_document_with_error(self, mock_elastich):
mock_elastich.side_effect = mock.Mock(
side_effect=ElasticsearchException()) # noqa
with self.assertRaises(ElasticsearchException):
webiste_nosql = WebsiteNoSql()
webiste_nosql.get(1)
self.assertEqual(mock_elastich.call_count, 1)
def _save_website_links(self, website, list_urls):
website_nosql = WebsiteNoSql()
body = {'urls': list_urls}
check_website_exist_done = self._check_if_website_exist_done(
website.url) # noqa
if check_website_exist_done is False:
self._put_website_links_on_quee(list_urls)
website_nosql.create(website.id, body)
else:
website_done = Website.query\
.filter(Website.url == website.url)\
.filter(Website.status == Website.Status.DONE).first()
website_nosql_db = website_nosql.get(website_done.id)
if website_nosql_db:
if website_nosql_db['_source'] != body:
self._put_website_links_on_quee(list_urls)
website_nosql.create(website.id, body)
website_nosql.update(website_done.id, body)
else:
website_nosql.create(website.id, body)
else:
logger.error(
"Website NoSql not found: on website_id {0}".format(
website_done.id)) # noqa
raise WebsiteBackendException("Website NoSql not found")
def test_get_website_document(self):
webiste_nosql = WebsiteNoSql()
body = {'urls': ['http://ibm.com.br']}
webiste_nosql.create(3, body)
bd_web_site = webiste_nosql.get(3)
body = {'urls': ['http://ibm.com.br']}
self.assertEquals(bd_web_site['_source'], body)
webiste_nosql.delete(3)
def test_update_website_document(self):
webiste_nosql = WebsiteNoSql()
body = {'urls': ['http://ibm.com.br']}
webiste_nosql.create(4, body)
body = {'urls': ['http://vtrmantovani.com.br']}
webiste_nosql.update(4, body)
bd_web_site = webiste_nosql.get(4)
self.assertEquals(bd_web_site['_source'], body)
webiste_nosql.delete(4)
def test_save_website_links_without_website_done(self):
db.session.add(self.website)
db.session.commit()
list_urls = ['http://ibm.com.br']
website_backend = WebsiteBackend()
website_backend._save_website_links(self.website, list_urls)
webiste_nosql = WebsiteNoSql()
self.assertEqual(Website.query.count(), 2)
self.assertTrue(webiste_nosql.get(1))
webiste_nosql.delete(1)
def test_save_website_links_without_website_nosql_differents_urls(self):
self.website.status = Website.Status.DONE
db.session.add(self.website)
db.session.commit()
webiste_nosql = WebsiteNoSql()
list_urls = ['http://vtrmantovani.com.br']
body = {'urls': list_urls}
webiste_nosql.create(1, body)
website_backend = WebsiteBackend()
website_backend._save_website_links(self.website, list_urls)
self.assertEqual(Website.query.count(), 1)
self.assertTrue(webiste_nosql.get(1))
webiste_nosql.delete(1)
def get(p_website_id):
try:
website_db = WebsiteNoSql()
website = website_db.get(p_website_id)
if not website:
raise NotFound("Website id not found")
urls = website['_source']
if not urls:
logger.error("Source not found on website_id={}".format(
p_website_id)) # noqa
raise BadRequestGeneric(description="Source not found")
except ElasticsearchException as e:
logger.error(
"ElasticsearchException error in get website_id:{0}. Error:{1}".
format(p_website_id, e)) # noqa
raise BadRequestGeneric(description="Some problems in bd")
return jsonify(website['_source'])
def search():
r = request.json
status = r['status']
limit = r['limit']
offset = r['offset']
list_websites = []
try:
website_list = Website.query.filter(Website.status == status)\
.limit(limit).offset(offset).all()
total_itens = Website.query.filter(Website.status == status).count()
list_urls = []
for website in website_list:
if website.status == Website.Status.DONE:
website_db = WebsiteNoSql()
website_no_sql = website_db.get(website.id)
if not website_no_sql:
logger.error(
"Website not found in search website_id={}".format(
website.id)) # noqa
raise NotFound("Website id not found")
urls = website_no_sql['_source']
list_urls = urls['urls']
list_websites.append({
'id': website.id,
'website': website.url,
'urls': list_urls
})
except ElasticsearchException as e:
logger.error(
"ElasticsearchException error in search website, Error: {0}".
format(e)) # noqa
raise BadRequestGeneric(description="Some problems in bd")
return jsonify({'websites': list_websites, 'total_itens': total_itens})
def test_get_website_document_not_found(self):
webiste_nosql = WebsiteNoSql()
bd_web_site = webiste_nosql.get(1)
self.assertEquals(bd_web_site, None)