def get_backend_id(backend_name, backend_params):
if backend_name not in get_connectors():
raise RuntimeError("Unknown backend %s" % backend_name)
connector = get_connectors()[backend_name]
klass = connector[3] # BackendCmd for the connector
backend_cmd = klass(*backend_params)
backend = backend_cmd.backend
return backend.unique_id
def get_ocean_backend(backend_cmd, enrich_backend, no_incremental):
""" Get the ocean backend configured to start from the last enriched date """
if no_incremental:
last_enrich = None
else:
last_enrich = get_last_enrich(backend_cmd, enrich_backend)
logger.debug("Last enrichment: %s", last_enrich)
backend = None
connector = get_connectors()[enrich_backend.get_connector_name()]
if backend_cmd:
backend = backend_cmd.backend
signature = inspect.signature(backend.fetch)
if 'from_date' in signature.parameters:
ocean_backend = connector[1](backend, from_date=last_enrich)
elif 'offset' in signature.parameters:
ocean_backend = connector[1](backend, offset=last_enrich)
else:
ocean_backend = connector[1](backend)
else:
if last_enrich:
ocean_backend = connector[1](backend, from_date=last_enrich)
else:
ocean_backend = connector[1](backend)
return ocean_backend
def test_enrich(self, sortinghat=False, projects=False):
"""Test enrich all sources"""
config = configparser.ConfigParser()
config.read(CONFIG_FILE)
es_con = dict(config.items('ElasticSearch'))['url']
logging.info("Enriching data in: %s", es_con)
connectors = get_connectors()
for con in sorted(connectors.keys()):
perceval_backend = None
ocean_index = "test_" + con
enrich_index = "test_" + con + "_enrich"
clean = False
ocean_backend = connectors[con][1](perceval_backend)
elastic_ocean = get_elastic(es_con, ocean_index, clean,
ocean_backend)
ocean_backend.set_elastic(elastic_ocean)
clean = True
if not sortinghat and not projects:
enrich_backend = connectors[con][2](perceval_backend)
elif sortinghat and not projects:
enrich_backend = connectors[con][2](
perceval_backend, db_sortinghat=DB_SORTINGHAT)
elif not sortinghat and projects:
enrich_backend = connectors[con][2](
perceval_backend, db_projects_map=DB_PROJECTS)
elastic_enrich = get_elastic(es_con, enrich_index, clean,
enrich_backend)
enrich_backend.set_elastic(elastic_enrich)
enrich_count = self.__enrich_items(ocean_backend, enrich_backend)
logging.info("Total items enriched %i ", enrich_count)
def test_enrich(self, sortinghat=False, projects=False):
"""Test enrich all sources"""
config = configparser.ConfigParser()
config.read(CONFIG_FILE)
es_con = dict(config.items('ElasticSearch'))['url']
logging.info("Enriching data in: %s", es_con)
connectors = get_connectors()
for con in sorted(connectors.keys()):
perceval_backend = None
ocean_index = "test_"+con
enrich_index = "test_"+con+"_enrich"
clean = False
ocean_backend = connectors[con][1](perceval_backend)
elastic_ocean = get_elastic(es_con, ocean_index, clean, ocean_backend)
ocean_backend.set_elastic(elastic_ocean)
clean = True
if not sortinghat and not projects:
enrich_backend = connectors[con][2]()
elif sortinghat and not projects:
enrich_backend = connectors[con][2](db_sortinghat=DB_SORTINGHAT)
elif not sortinghat and projects:
enrich_backend = connectors[con][2](db_projects_map=DB_PROJECTS)
elastic_enrich = get_elastic(es_con, enrich_index, clean, enrich_backend)
enrich_backend.set_elastic(elastic_enrich)
enrich_count = self.__enrich_items(ocean_backend, enrich_backend)
logging.info("Total items enriched %i ", enrich_count)
def test_read_data(self):
"""Test load all sources JSON"""
config = configparser.ConfigParser()
config.read(CONFIG_FILE)
connectors = get_connectors()
# Check we have data for all the data sources
for con in sorted(connectors.keys()):
with open(os.path.join("data", con + ".json")) as f:
json.load(f)
def test_read_data(self):
"""Test load all sources JSON"""
config = configparser.ConfigParser()
config.read(CONFIG_FILE)
connectors = get_connectors()
# Check we have data for all the data sources
for con in sorted(connectors.keys()):
with open(os.path.join("data", con + ".json")) as f:
json.load(f)
def test_refresh_project(self):
"""Test refresh project field for all sources"""
# self.test_enrich_sh() # Load the identities in ES
config = configparser.ConfigParser()
config.read(CONFIG_FILE)
es_con = dict(config.items('ElasticSearch'))['url']
logging.info("Refreshing data in: %s", es_con)
connectors = get_connectors()
for con in sorted(connectors.keys()):
enrich_index = "test_"+con+"_enrich"
enrich_backend = connectors[con][2](db_projects_map=DB_PROJECTS)
clean = False
elastic_enrich = get_elastic(es_con, enrich_index, clean, enrich_backend)
enrich_backend.set_elastic(elastic_enrich)
logging.info("Refreshing projects fields in enriched index %s", elastic_enrich.index_url)
self.__refresh_projects(enrich_backend)
def test_data_load(self):
"""Test load all sources JSON data into ES"""
config = configparser.ConfigParser()
config.read(CONFIG_FILE)
es_con = dict(config.items('ElasticSearch'))['url']
logging.info("Loading data in: %s", es_con)
connectors = get_connectors()
for con in sorted(connectors.keys()):
with open(os.path.join("data", con + ".json")) as f:
items = json.load(f)
es_index = "test_"+con
clean = True
perceval_backend = None
ocean_backend = connectors[con][1](perceval_backend)
elastic_ocean = get_elastic(es_con, es_index, clean, ocean_backend)
ocean_backend.set_elastic(elastic_ocean)
self.__data2es(items, ocean_backend)
def test_data_load(self):
"""Test load all sources JSON data into ES"""
config = configparser.ConfigParser()
config.read(CONFIG_FILE)
es_con = dict(config.items('ElasticSearch'))['url']
logging.info("Loading data in: %s", es_con)
connectors = get_connectors()
for con in sorted(connectors.keys()):
with open(os.path.join("data", con + ".json")) as f:
items = json.load(f)
es_index = "test_"+con
clean = True
perceval_backend = None
ocean_backend = connectors[con][1](perceval_backend)
elastic_ocean = get_elastic(es_con, es_index, clean, ocean_backend)
ocean_backend.set_elastic(elastic_ocean)
self.__data2es(items, ocean_backend)
def test_init(self):
"""Test whether the backends can be loaded """
self.assertEqual(len(get_connectors()), NUMBER_BACKENDS)
def test_init(self):
"""Test whether the backends can be loaded """
self.assertEqual(len(get_connectors()), NUMBER_BACKENDS)
