def test_get_elastic_items_filter(self):
"""Test whether the elastic method works properly with filter"""
perceval_backend = Git('/tmp/perceval_mc84igfc/gittest', '/tmp/foo')
elastic = ElasticSearch(self.es_con, self.target_index, GitOcean.mapping)
# Load items
items = json.loads(read_file('data/git.json'))
ocean = GitOcean(perceval_backend)
ocean.elastic = elastic
ocean.feed_items(items)
filter = {
"name": "uuid",
"value": [
"43f217b2f678a5691fdbc5c6c5302243e79e5a90",
"00ee6902e34b309cd05706c26e3e195a62492f60"
]
}
eitems = ElasticItems(perceval_backend)
eitems.elastic = elastic
r_json = eitems.get_elastic_items(_filter=filter)
hits = r_json['hits']['hits']
self.assertEqual(len(hits), 2)
self.assertEqual(hits[0]['_source']['uuid'], "43f217b2f678a5691fdbc5c6c5302243e79e5a90")
self.assertEqual(hits[1]['_source']['uuid'], "00ee6902e34b309cd05706c26e3e195a62492f60")
def test_perceval_params(self):
"""Test the extraction of perceval params from an URL"""
url = "https://github.com/grimoirelab/perceval"
expected_params = ['https://github.com/grimoirelab/perceval']
self.assertListEqual(GitOcean.get_perceval_params_from_url(url),
expected_params)
url = "https://github.com/grimoirelab/perceval /tmp/perceval-repo"
expected_params = ['https://github.com/grimoirelab/perceval']
self.assertListEqual(GitOcean.get_perceval_params_from_url(url),
expected_params)
def test_fetch_filter_raw(self):
"""Test whether the fetch with filter raw properly works"""
perceval_backend = Git('/tmp/perceval_mc84igfc/gittest', '/tmp/foo')
elastic = ElasticSearch(self.es_con, self.target_index, GitOcean.mapping)
# Load items
items = json.loads(read_file('data/git.json'))
ocean = GitOcean(perceval_backend)
ocean.elastic = elastic
ocean.feed_items(items)
eitems = ElasticItems(perceval_backend)
eitems.set_filter_raw("data.commit:87783129c3f00d2c81a3a8e585eb86a47e39891a")
eitems.elastic = elastic
items = [ei for ei in eitems.fetch()]
self.assertEqual(len(items), 1)
def test_get_connector_name(self):
"""Test whether the connector name is correctly returned"""
eitems = ElasticItems(self.perceval_backend)
self.assertIsNone(eitems.get_connector_name())
eitems = GitOcean(self.perceval_backend)
self.assertEqual(eitems.get_connector_name(), 'git')
def test_arthur_params(self):
"""Test the extraction of arthur params from an URL"""
url = "https://github.com/grimoirelab/perceval"
expected_params = {
'uri': 'https://github.com/grimoirelab/perceval',
'url': 'https://github.com/grimoirelab/perceval'
}
self.assertDictEqual(GitOcean.get_arthur_params_from_url(url), expected_params)
def test_fetch(self):
"""Test whether the fetch method properly works"""
perceval_backend = Git('/tmp/perceval_mc84igfc/gittest', '/tmp/foo')
elastic = ElasticSearch(self.es_con, self.target_index, GitOcean.mapping)
# Load items
items = json.loads(read_file('data/git.json'))
ocean = GitOcean(perceval_backend)
ocean.elastic = elastic
ocean.feed_items(items)
eitems = ElasticItems(perceval_backend)
eitems.scroll_size = 2
eitems.elastic = elastic
items = [ei for ei in eitems.fetch()]
self.assertEqual(len(items), 9)
def test_get_elastic_items(self):
"""Test whether the elastic method works properly"""
perceval_backend = Git('/tmp/perceval_mc84igfc/gittest', '/tmp/foo')
elastic = ElasticSearch(self.es_con, self.target_index, GitOcean.mapping)
# Load items
items = json.loads(read_file('data/git.json'))
ocean = GitOcean(perceval_backend)
ocean.elastic = elastic
ocean.feed_items(items)
eitems = ElasticItems(perceval_backend)
eitems.elastic = elastic
r_json = eitems.get_elastic_items()
total = r_json['hits']['total']
total = total['value'] if isinstance(total, dict) else total
self.assertEqual(total, 9)
def test_arthur_params(self):
"""Test the extraction of arthur params from an URL"""
with open("data/projects-release.json") as projects_filename:
url = json.load(projects_filename)['grimoire']['git'][0]
arthur_params = {
'uri': 'https://github.com/grimoirelab/perceval',
'url': 'https://github.com/grimoirelab/perceval'
}
self.assertDictEqual(arthur_params,
GitOcean.get_arthur_params_from_url(url))
def test_fetch_no_results(self):
"""Test whether a message is logged when no results are found"""
perceval_backend = Git('/tmp/perceval_mc84igfc/gittest-not_found', '/tmp/foo')
elastic = ElasticSearch(self.es_con, self.target_index, GitOcean.mapping)
# Load items
items = json.loads(read_file('data/git.json'))
ocean = GitOcean(perceval_backend)
ocean.elastic = elastic
ocean.feed_items(items)
eitems = ElasticItems(perceval_backend)
eitems.elastic = elastic
with self.assertLogs(logger, level='DEBUG') as cm:
items = [ei for ei in eitems.fetch()]
self.assertEqual(len(items), 0)
self.assertRegex(cm.output[-2], 'DEBUG:grimoire_elk.elastic_items:No results found.*')
self.assertRegex(cm.output[-1], 'DEBUG:grimoire_elk.elastic_items:Releasing scroll_id=*')
def test_fetch_from_date(self):
"""Test whether the fetch method with from_date properly works"""
perceval_backend = Git('/tmp/perceval_mc84igfc/gittest', '/tmp/foo')
elastic = ElasticSearch(self.es_con, self.target_index, GitOcean.mapping)
# Load items
items = json.loads(read_file('data/git.json'))
ocean = GitOcean(perceval_backend)
ocean.elastic = elastic
ocean.feed_items(items)
# Fetch total items
eitems = ElasticItems(perceval_backend)
eitems.elastic = elastic
items = [ei for ei in eitems.fetch()]
self.assertEqual(len(items), 9)
# Fetch with from date
from_date = str_to_datetime("2018-02-09T08:33:22.699+00:00")
eitems = ElasticItems(perceval_backend, from_date=from_date)
eitems.elastic = elastic
items = [ei for ei in eitems.fetch()]
self.assertEqual(len(items), 2)