def get_backend_id(backend_name, backend_params): if backend_name not in get_connectors(): raise RuntimeError("Unknown backend %s" % backend_name) connector = get_connectors()[backend_name] klass = connector[3] # BackendCmd for the connector backend_cmd = klass(*backend_params) backend = backend_cmd.backend return backend.unique_id
def get_ocean_backend(backend_cmd, enrich_backend, no_incremental): """ Get the ocean backend configured to start from the last enriched date """ if no_incremental: last_enrich = None else: last_enrich = get_last_enrich(backend_cmd, enrich_backend) logger.debug("Last enrichment: %s", last_enrich) backend = None connector = get_connectors()[enrich_backend.get_connector_name()] if backend_cmd: backend = backend_cmd.backend signature = inspect.signature(backend.fetch) if 'from_date' in signature.parameters: ocean_backend = connector[1](backend, from_date=last_enrich) elif 'offset' in signature.parameters: ocean_backend = connector[1](backend, offset=last_enrich) else: ocean_backend = connector[1](backend) else: if last_enrich: ocean_backend = connector[1](backend, from_date=last_enrich) else: ocean_backend = connector[1](backend) return ocean_backend
def test_enrich(self, sortinghat=False, projects=False): """Test enrich all sources""" config = configparser.ConfigParser() config.read(CONFIG_FILE) es_con = dict(config.items('ElasticSearch'))['url'] logging.info("Enriching data in: %s", es_con) connectors = get_connectors() for con in sorted(connectors.keys()): perceval_backend = None ocean_index = "test_" + con enrich_index = "test_" + con + "_enrich" clean = False ocean_backend = connectors[con][1](perceval_backend) elastic_ocean = get_elastic(es_con, ocean_index, clean, ocean_backend) ocean_backend.set_elastic(elastic_ocean) clean = True if not sortinghat and not projects: enrich_backend = connectors[con][2](perceval_backend) elif sortinghat and not projects: enrich_backend = connectors[con][2]( perceval_backend, db_sortinghat=DB_SORTINGHAT) elif not sortinghat and projects: enrich_backend = connectors[con][2]( perceval_backend, db_projects_map=DB_PROJECTS) elastic_enrich = get_elastic(es_con, enrich_index, clean, enrich_backend) enrich_backend.set_elastic(elastic_enrich) enrich_count = self.__enrich_items(ocean_backend, enrich_backend) logging.info("Total items enriched %i ", enrich_count)
def test_enrich(self, sortinghat=False, projects=False): """Test enrich all sources""" config = configparser.ConfigParser() config.read(CONFIG_FILE) es_con = dict(config.items('ElasticSearch'))['url'] logging.info("Enriching data in: %s", es_con) connectors = get_connectors() for con in sorted(connectors.keys()): perceval_backend = None ocean_index = "test_"+con enrich_index = "test_"+con+"_enrich" clean = False ocean_backend = connectors[con][1](perceval_backend) elastic_ocean = get_elastic(es_con, ocean_index, clean, ocean_backend) ocean_backend.set_elastic(elastic_ocean) clean = True if not sortinghat and not projects: enrich_backend = connectors[con][2]() elif sortinghat and not projects: enrich_backend = connectors[con][2](db_sortinghat=DB_SORTINGHAT) elif not sortinghat and projects: enrich_backend = connectors[con][2](db_projects_map=DB_PROJECTS) elastic_enrich = get_elastic(es_con, enrich_index, clean, enrich_backend) enrich_backend.set_elastic(elastic_enrich) enrich_count = self.__enrich_items(ocean_backend, enrich_backend) logging.info("Total items enriched %i ", enrich_count)
def test_read_data(self): """Test load all sources JSON""" config = configparser.ConfigParser() config.read(CONFIG_FILE) connectors = get_connectors() # Check we have data for all the data sources for con in sorted(connectors.keys()): with open(os.path.join("data", con + ".json")) as f: json.load(f)
def test_refresh_project(self): """Test refresh project field for all sources""" # self.test_enrich_sh() # Load the identities in ES config = configparser.ConfigParser() config.read(CONFIG_FILE) es_con = dict(config.items('ElasticSearch'))['url'] logging.info("Refreshing data in: %s", es_con) connectors = get_connectors() for con in sorted(connectors.keys()): enrich_index = "test_"+con+"_enrich" enrich_backend = connectors[con][2](db_projects_map=DB_PROJECTS) clean = False elastic_enrich = get_elastic(es_con, enrich_index, clean, enrich_backend) enrich_backend.set_elastic(elastic_enrich) logging.info("Refreshing projects fields in enriched index %s", elastic_enrich.index_url) self.__refresh_projects(enrich_backend)
def test_data_load(self): """Test load all sources JSON data into ES""" config = configparser.ConfigParser() config.read(CONFIG_FILE) es_con = dict(config.items('ElasticSearch'))['url'] logging.info("Loading data in: %s", es_con) connectors = get_connectors() for con in sorted(connectors.keys()): with open(os.path.join("data", con + ".json")) as f: items = json.load(f) es_index = "test_"+con clean = True perceval_backend = None ocean_backend = connectors[con][1](perceval_backend) elastic_ocean = get_elastic(es_con, es_index, clean, ocean_backend) ocean_backend.set_elastic(elastic_ocean) self.__data2es(items, ocean_backend)
def test_init(self): """Test whether the backends can be loaded """ self.assertEqual(len(get_connectors()), NUMBER_BACKENDS)