def test_get_repos_by_backend_sections_unknown(self): """Test whether the repos of each section are properly loaded when the unknown section is present""" config = Config(CONF_FILE_UNKNOWN) task = TaskProjects(config) self.assertEqual(task.execute(), None) # repos not in unknown expected_list = ["https://github.com/chaoss/grimoirelab-perceval"] repos = task.get_repos_by_backend_section("git") self.assertListEqual(repos, expected_list) repos = task.get_repos_by_backend_section("git", raw=False) self.assertListEqual(repos, expected_list) # repos only in unknown expected_list = ["https://bugzilla.mozilla.org"] repos = task.get_repos_by_backend_section("bugzillarest") self.assertListEqual(repos, expected_list) repos = task.get_repos_by_backend_section("bugzillarest", raw=False) self.assertListEqual(repos, expected_list) # repos in unknown and other section expected_list = ["gerrit.onosproject.org"] repos = task.get_repos_by_backend_section("gerrit:onos") self.assertListEqual(repos, expected_list) expected_list = [ "gerrit.onosproject.org --filter-raw=data.project:OnosSystemTest", "gerrit.onosproject.org --filter-raw=data.project:OnosSystemTestJenkins", "gerrit.onosproject.org --filter-raw=data.project:cord-openwrt", "gerrit.onosproject.org --filter-raw=data.project:fabric-control", "gerrit.onosproject.org --filter-raw=data.project:manifest" ] repos = task.get_repos_by_backend_section("gerrit:onos", raw=False) repos.sort() expected_list.sort() self.assertListEqual(repos, expected_list)
def test_get_repos_by_backend_section(self): """Test whether the repos of each section are properly loaded""" config = Config(CONF_FILE) task = TaskProjects(config) self.assertEqual(task.execute(), None) config.conf.keys() backend_sections = list(set([sect for sect in config.conf.keys() for backend_section in Config.get_backend_sections() if sect and sect.startswith(backend_section)])) backend_sections.sort() backend = backend_sections[0] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'askbot') self.assertEqual(repos, ['https://ask.puppet.com']) backend = backend_sections[1] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'bugzilla') self.assertEqual(repos, ['https://bugs.eclipse.org/bugs/']) backend = backend_sections[2] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'bugzillarest') self.assertEqual(repos, ['https://bugzilla.mozilla.org']) backend = backend_sections[3] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'confluence') self.assertEqual(repos, ['https://wiki.open-o.org/']) backend = backend_sections[4] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'discourse') self.assertEqual(repos, ['https://foro.mozilla-hispano.org/']) backend = backend_sections[5] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'dockerhub') self.assertEqual(repos, ['bitergia kibiter']) backend = backend_sections[6] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'functest') self.assertEqual(repos, ['http://testresults.opnfv.org/test/']) backend = backend_sections[7] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'gerrit') self.assertEqual(repos, ['review.openstack.org']) backend = backend_sections[8] repos = task.get_repos_by_backend_section(backend) repos.sort() expected_list = [ "https://github.com/VizGrimoire/GrimoireLib " "--filter-raw-prefix=data.files.file:grimoirelib_alch,data.files.file:README.md", "https://github.com/MetricsGrimoire/CMetrics"] expected_list.sort() self.assertEqual(backend, 'git') self.assertEqual(repos, expected_list) backend = backend_sections[9] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'github') self.assertEqual(repos, ['https://github.com/grimoirelab/perceval']) backend = backend_sections[10] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'github:pull') self.assertEqual(repos, ['https://github.com/grimoirelab/perceval']) backend = backend_sections[11] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'gitlab') self.assertEqual(repos, ['https://gitlab.com/inkscape/inkscape-web']) backend = backend_sections[12] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'google_hits') self.assertEqual(repos, ['bitergia grimoirelab']) backend = backend_sections[13] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'hyperkitty') self.assertEqual(repos, ['https://lists.mailman3.org/archives/list/[email protected]']) backend = backend_sections[14] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'jenkins') self.assertEqual(repos, ['https://build.opnfv.org/ci']) backend = backend_sections[15] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'jira') self.assertEqual(repos, ['https://jira.opnfv.org']) backend = backend_sections[16] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'mattermost') self.assertEqual(repos, ['https://chat.openshift.io 8j366ft5affy3p36987pcugaoa']) backend = backend_sections[17] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'mattermost:group1') self.assertEqual(repos, ['https://chat.openshift.io 8j366ft5affy3p36987cip']) backend = backend_sections[18] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'mattermost:group2') self.assertEqual(repos, ['https://chat.openshift.io 8j366ft5affy3p36987ciop']) backend = backend_sections[19] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'mbox') self.assertEqual(repos, ['metrics-grimoire ~/.perceval/mbox']) backend = backend_sections[20] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'mediawiki') self.assertEqual(repos, ['https://wiki.mozilla.org']) backend = backend_sections[21] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'meetup') self.assertEqual(repos, ['South-East-Puppet-User-Group']) backend = backend_sections[22] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'mozillaclub') self.assertEqual(repos, ['https://spreadsheets.google.com/feeds/cells/' '1QHl2bjBhMslyFzR5XXPzMLdzzx7oeSKTbgR5PM8qp64/ohaibtm/public/values?alt=json']) backend = backend_sections[23] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'nntp') self.assertEqual(repos, ['news.mozilla.org mozilla.dev.project-link']) backend = backend_sections[24] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'phabricator') self.assertEqual(repos, ['https://phabricator.wikimedia.org']) backend = backend_sections[25] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'pipermail') self.assertEqual(repos, ['https://mail.gnome.org/archives/libart-hackers/']) backend = backend_sections[26] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'puppetforge') self.assertEqual(repos, ['']) backend = backend_sections[27] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'redmine') self.assertEqual(repos, ['http://tracker.ceph.com/']) backend = backend_sections[28] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'remo') self.assertEqual(repos, ['https://reps.mozilla.org']) backend = backend_sections[29] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'remo:activities') self.assertEqual(repos, ['https://reps.mozilla.org']) backend = backend_sections[30] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'rss') self.assertEqual(repos, ['https://blog.bitergia.com/feed/']) backend = backend_sections[31] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'slack') self.assertEqual(repos, ['C7LSGB0AU']) backend = backend_sections[32] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'stackexchange') repos.sort() expected_list = [ "https://stackoverflow.com/questions/tagged/ovirt", "https://stackoverflow.com/questions/tagged/rdo", "https://stackoverflow.com/questions/tagged/kibana" ] expected_list.sort() self.assertEqual(repos, expected_list) backend = backend_sections[33] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'supybot') self.assertEqual(repos, ['openshift ~/.perceval/irc/percevalbot/logs/ChannelLogger/freenode/#openshift/']) backend = backend_sections[34] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'telegram') self.assertEqual(repos, ['Mozilla_analytics']) backend = backend_sections[35] repos = task.get_repos_by_backend_section(backend) self.assertEqual(backend, 'twitter') self.assertEqual(repos, ['bitergia'])
def execute(self): def check_arthur_task(repo, backend_args): """ Check if a task exists in arthur and if not, create it """ arthur_repo_json = self.__create_arthur_json(repo, backend_args) logger.debug('JSON config for arthur %s', json.dumps(arthur_repo_json, indent=True)) # First check is the task already exists try: r = requests.post(self.arthur_url + "/tasks") except requests.exceptions.ConnectionError as ex: logging.error("Can not connect to %s", self.arthur_url) raise RuntimeError("Can not connect to " + self.arthur_url) task_ids = [task['task_id'] for task in r.json()['tasks']] new_task_ids = [ task['task_id'] for task in arthur_repo_json['tasks'] ] # TODO: if a tasks already exists maybe we should delete and readd it already_tasks = list(set(task_ids).intersection(set(new_task_ids))) if len(already_tasks) > 0: logger.warning( "Tasks not added to arthur because there are already existing tasks %s", already_tasks) else: r = requests.post(self.arthur_url + "/add", json=arthur_repo_json) r.raise_for_status() logger.info('[%s] collection configured in arthur for %s', self.backend_section, repo) def collect_arthur_items(repo): aitems = self.__feed_backend_arthur(repo) if not aitems: return connector = get_connector_from_name(self.backend_section) klass = connector[1] # Ocean backend for the connector ocean_backend = klass(None) es_col_url = self._get_collection_url() es_index = self.conf[self.backend_section]['raw_index'] clean = False elastic_ocean = get_elastic(es_col_url, es_index, clean, ocean_backend) ocean_backend.set_elastic(elastic_ocean) ocean_backend.feed(arthur_items=aitems) cfg = self.config.get_conf() if 'collect' in cfg[self.backend_section] and not cfg[ self.backend_section]['collect']: logging.info('%s collect disabled', self.backend_section) return if 'scroll_size' in cfg['general']: ElasticItems.scroll_size = cfg['general']['scroll_size'] if 'bulk_size' in cfg['general']: ElasticSearch.max_items_bulk = cfg['general']['bulk_size'] logger.info('Programming arthur for [%s] raw data collection', self.backend_section) clean = False fetch_archive = False if 'fetch-archive' in self.conf[self.backend_section] and self.conf[ self.backend_section]['fetch-archive']: fetch_archive = True # repos could change between executions because changes in projects repos = TaskProjects.get_repos_by_backend_section(self.backend_section) if not repos: logger.warning("No collect repositories for %s", self.backend_section) for repo in repos: # If the repo already exists don't try to add it to arthur tag = self.backend_tag(repo) if tag not in self.arthur_items: self.arthur_items[tag] = [] repo, repo_labels = self._extract_repo_labels( self.backend_section, repo) p2o_args = self._compose_p2o_params(self.backend_section, repo) filter_raw = p2o_args[ 'filter-raw'] if 'filter-raw' in p2o_args else None if filter_raw: # If filter-raw exists it means that there is an equivalent URL # in the `unknown` section of the projects.json. Thus the URL with # filter-raw is ignored in the collection phase, while the URL # in `unknown` is considered in this phase. logging.warning("Not collecting filter raw repository: %s", repo) continue backend_args = self._compose_perceval_params( self.backend_section, repo) logger.debug(backend_args) check_arthur_task(repo, backend_args) collect_arthur_items(repo)
def execute(self): errors = [] cfg = self.config.get_conf() if 'scroll_size' in cfg['general']: ElasticItems.scroll_size = cfg['general']['scroll_size'] if 'bulk_size' in cfg['general']: ElasticSearch.max_items_bulk = cfg['general']['bulk_size'] if 'collect' in cfg[self.backend_section] and not cfg[ self.backend_section]['collect']: logging.info('%s collect disabled', self.backend_section) return errors t2 = time.time() logger.info('[%s] collection phase starts', self.backend_section) print("Collection for {}: starting...".format(self.backend_section)) clean = False fetch_archive = False if 'fetch-archive' in cfg[self.backend_section] and cfg[ self.backend_section]['fetch-archive']: fetch_archive = True # repos could change between executions because changes in projects repos = TaskProjects.get_repos_by_backend_section(self.backend_section) if not repos: logger.warning("No collect repositories for %s", self.backend_section) for repo in repos: repo, repo_labels = self._extract_repo_labels( self.backend_section, repo) p2o_args = self._compose_p2o_params(self.backend_section, repo) filter_raw = p2o_args[ 'filter-raw'] if 'filter-raw' in p2o_args else None if filter_raw: # If filter-raw exists it means that there is an equivalent URL # in the `unknown` section of the projects.json. Thus the URL with # filter-raw is ignored in the collection phase, while the URL # in `unknown` is considered in this phase. logging.warning("Not collecting filter raw repository: %s", repo) continue url = p2o_args['url'] backend_args = self._compose_perceval_params( self.backend_section, repo) logger.debug(backend_args) logger.info('[%s] collection starts for %s', self.backend_section, repo) es_col_url = self._get_collection_url() ds = self.backend_section backend = self.get_backend(self.backend_section) project = None # just used for github in cauldron es_aliases = self.select_aliases(cfg, self.backend_section) try: error_msg = feed_backend(es_col_url, clean, fetch_archive, backend, backend_args, cfg[ds]['raw_index'], cfg[ds]['enriched_index'], project, es_aliases=es_aliases, projects_json_repo=repo, repo_labels=repo_labels) error = {'backend': backend, 'repo': repo, 'error': error_msg} errors.append(error) except Exception: logger.error( "Something went wrong collecting data from this %s repo: %s . " "Using the backend_args: %s " % (ds, url, str(backend_args))) traceback.print_exc() raise DataCollectionError('Failed to collect data from %s' % url) logger.info('[%s] collection finished for %s', self.backend_section, repo) t3 = time.time() spent_time = time.strftime("%H:%M:%S", time.gmtime(t3 - t2)) logger.info('[%s] collection phase finished in %s', self.backend_section, spent_time) print("Collection for {}: finished after {} hours".format( self.backend_section, spent_time)) self.retain_data(cfg['general']['retention_time'], self.conf['es_collection']['url'], self.conf[self.backend_section]['raw_index']) return errors
def __enrich_items(self): time_start = datetime.now() logger.info('[%s] enrichment phase starts', self.backend_section) cfg = self.config.get_conf() if 'scroll_size' in cfg['general']: ElasticItems.scroll_size = cfg['general']['scroll_size'] if 'bulk_size' in cfg['general']: ElasticSearch.max_items_bulk = cfg['general']['bulk_size'] no_incremental = False # not used due to https://github.com/chaoss/grimoirelab-elk/pull/773 github_token = None pair_programming = False node_regex = None if 'git' in cfg and 'pair-programming' in cfg['git']: pair_programming = cfg['git']['pair-programming'] if 'jenkins' in cfg and 'node_regex' in cfg['jenkins']: node_regex = cfg['jenkins']['node_regex'] only_studies = False only_identities = False # repos could change between executions because changes in projects repos = TaskProjects.get_repos_by_backend_section(self.backend_section, raw=False) if not repos: logger.warning("No enrich repositories for %s", self.backend_section) # Get the metadata__timestamp value of the last item inserted in the enriched index before # looping over the repos which data is stored in the same index. This is needed to make sure # that the incremental enrichment works for data sources that are collected globally but only # partially enriched. elastic_enrich = get_elastic( cfg['es_enrichment']['url'], cfg[self.backend_section]['enriched_index']) last_enrich_date = elastic_enrich.get_last_item_field( "metadata__timestamp") if last_enrich_date: last_enrich_date = last_enrich_date.replace(tzinfo=None) for repo in repos: repo, repo_labels = self._extract_repo_labels( self.backend_section, repo) p2o_args = self._compose_p2o_params(self.backend_section, repo) filter_raw = p2o_args[ 'filter-raw'] if 'filter-raw' in p2o_args else None jenkins_rename_file = p2o_args[ 'jenkins-rename-file'] if 'jenkins-rename-file' in p2o_args else None url = p2o_args['url'] # Second process perceval params from repo backend_args = self._compose_perceval_params( self.backend_section, url) studies_args = None backend = self.get_backend(self.backend_section) if 'studies' in self.conf[self.backend_section] and \ self.conf[self.backend_section]['studies']: studies_args = self.__load_studies() logger.info('[%s] enrichment starts for %s', self.backend_section, self.anonymize_url(repo)) es_enrich_aliases = self.select_aliases(cfg, self.backend_section) try: es_col_url = self._get_collection_url() enrich_backend( es_col_url, self.clean, backend, backend_args, self.backend_section, cfg[self.backend_section]['raw_index'], cfg[self.backend_section]['enriched_index'], None, # projects_db is deprecated cfg['projects']['projects_file'], self.db_sh, no_incremental, only_identities, github_token, False, # studies are executed in its own Task only_studies, cfg['es_enrichment']['url'], None, # args.events_enrich self.db_user, self.db_password, self.db_host, None, # args.refresh_projects, None, # args.refresh_identities, author_id=None, author_uuid=None, filter_raw=filter_raw, jenkins_rename_file=jenkins_rename_file, unaffiliated_group=self.db_unaffiliate_group, pair_programming=pair_programming, node_regex=node_regex, studies_args=studies_args, es_enrich_aliases=es_enrich_aliases, last_enrich_date=last_enrich_date, projects_json_repo=repo, repo_labels=repo_labels) except Exception as ex: logger.error( "Something went wrong producing enriched data for %s . " "Using the backend_args: %s ", self.backend_section, str(backend_args)) logger.error("Exception: %s", ex) raise DataEnrichmentError( 'Failed to produce enriched data for ' + self.backend_section) logger.info('[%s] enrichment finished for %s', self.backend_section, self.anonymize_url(repo)) spent_time = str(datetime.now() - time_start).split('.')[0] logger.info('[%s] enrichment phase finished in %s', self.backend_section, spent_time)
def __enrich_items(self): time_start = time.time() # logger.info('%s starts for %s ', 'enrichment', self.backend_section) logger.info('[%s] enrichment starts', self.backend_section) print("Enrichment for {}: starting...".format(self.backend_section)) cfg = self.config.get_conf() if 'scroll_size' in cfg['general']: ElasticItems.scroll_size = cfg['general']['scroll_size'] if 'bulk_size' in cfg['general']: ElasticSearch.max_items_bulk = cfg['general']['bulk_size'] no_incremental = False github_token = None pair_programming = False if 'github' in cfg and 'backend_token' in cfg['github']: github_token = cfg['github']['backend_token'] if 'git' in cfg and 'pair-programming' in cfg['git']: pair_programming = cfg['git']['pair-programming'] only_studies = False only_identities = False # repos could change between executions because changes in projects repos = TaskProjects.get_repos_by_backend_section(self.backend_section) if not repos: logger.warning("No enrich repositories for %s", self.backend_section) for repo in repos: # First process p2o params from repo p2o_args = self._compose_p2o_params(self.backend_section, repo) filter_raw = p2o_args[ 'filter-raw'] if 'filter-raw' in p2o_args else None filters_raw_prefix = p2o_args[ 'filters-raw-prefix'] if 'filters-raw-prefix' in p2o_args else None jenkins_rename_file = p2o_args[ 'jenkins-rename-file'] if 'jenkins-rename-file' in p2o_args else None url = p2o_args['url'] # Second process perceval params from repo backend_args = self._compose_perceval_params( self.backend_section, url) studies_args = None if 'studies' in self.conf[self.backend_section] and \ self.conf[self.backend_section]['studies']: studies_args = self.__load_studies() try: es_col_url = self._get_collection_url() logger.debug('[%s] enrichment starts for %s', self.backend_section, repo) backend = self.get_backend(self.backend_section) enrich_backend( es_col_url, self.clean, backend, backend_args, cfg[self.backend_section]['raw_index'], cfg[self.backend_section]['enriched_index'], None, # projects_db is deprecated cfg['projects']['projects_file'], cfg['sortinghat']['database'], no_incremental, only_identities, github_token, False, # studies are executed in its own Task only_studies, cfg['es_enrichment']['url'], None, # args.events_enrich cfg['sortinghat']['user'], cfg['sortinghat']['password'], cfg['sortinghat']['host'], None, # args.refresh_projects, None, # args.refresh_identities, author_id=None, author_uuid=None, filter_raw=filter_raw, filters_raw_prefix=filters_raw_prefix, jenkins_rename_file=jenkins_rename_file, unaffiliated_group=cfg['sortinghat']['unaffiliated_group'], pair_programming=pair_programming, studies_args=studies_args) except Exception as ex: logger.error( "Something went wrong producing enriched data for %s . " "Using the backend_args: %s ", self.backend_section, str(backend_args)) logger.error("Exception: %s", ex) raise DataEnrichmentError( 'Failed to produce enriched data for ' + self.backend_section) # Let's try to create the aliases for the enriched index if not self.enrich_aliases: logger.debug("Creating aliases after enrich") task_aliases = TaskPanelsAliases(self.config) task_aliases.set_backend_section(self.backend_section) task_aliases.execute() logger.debug("Done creating aliases after enrich") self.enrich_aliases = True spent_time = time.strftime("%H:%M:%S", time.gmtime(time.time() - time_start)) logger.info('[%s] enrichment finished in %s', self.backend_section, spent_time) print("Enrichment for {}: finished after {} hours".format( self.backend_section, spent_time))
def execute(self): cfg = self.config.get_conf() if 'scroll_size' in cfg['general']: ElasticItems.scroll_size = cfg['general']['scroll_size'] if 'bulk_size' in cfg['general']: ElasticSearch.max_items_bulk = cfg['general']['bulk_size'] if ('collect' in cfg[self.backend_section] and not cfg[self.backend_section]['collect']): logging.info('%s collect disabled', self.backend_section) return t2 = time.time() logger.info('[%s] raw data collection starts', self.backend_section) print("Collection for {}: starting...".format(self.backend_section)) clean = False fetch_archive = False if ('fetch-archive' in cfg[self.backend_section] and cfg[self.backend_section]['fetch-archive']): fetch_archive = True # repos could change between executions because changes in projects repos = TaskProjects.get_repos_by_backend_section(self.backend_section) if not repos: logger.warning("No collect repositories for %s", self.backend_section) for repo in repos: p2o_args = self._compose_p2o_params(self.backend_section, repo) filter_raw = p2o_args[ 'filter-raw'] if 'filter-raw' in p2o_args else None if filter_raw: # If filter-raw exists the goal is to enrich already collected # data, so don't collect anything logging.warning("Not collecting filter raw repository: %s", repo) continue url = p2o_args['url'] backend_args = self._compose_perceval_params( self.backend_section, repo) logger.debug(backend_args) logger.debug('[%s] collection starts for %s', self.backend_section, repo) es_col_url = self._get_collection_url() ds = self.backend_section backend = self.get_backend(self.backend_section) project = None # just used for github in cauldron try: feed_backend(es_col_url, clean, fetch_archive, backend, backend_args, cfg[ds]['raw_index'], cfg[ds]['enriched_index'], project) except Exception: logger.error( "Something went wrong collecting data from this %s repo: %s . " "Using the backend_args: %s " % (ds, url, str(backend_args))) traceback.print_exc() raise DataCollectionError('Failed to collect data from %s' % url) t3 = time.time() spent_time = time.strftime("%H:%M:%S", time.gmtime(t3 - t2)) logger.info('[%s] Data collection finished in %s', self.backend_section, spent_time) print("Collection for {}: finished after {} hours".format( self.backend_section, spent_time))
def execute(self): cfg = self.config.get_conf() if 'gerrit' not in cfg or 'git' not in cfg: logger.error("gerrit and git are needed for track items.") return # We need to track the items in all git repositories from OPNFV git_repos = [] repos_raw = TaskProjects.get_repos_by_backend_section("git") # git://git.opnfv.org/apex -> https://git.opnfv.org/apex/plain/UPSTREAM for repo in repos_raw: repo = repo.replace("git://", "https://") repo += "/plain/UPSTREAM" git_repos.append(repo) project = cfg['track_items']['project'] elastic_url_enrich = cfg['es_enrichment']['url'] # The raw data comes from upstream project elastic_url_raw = cfg['track_items']['upstream_raw_es_url'] index_gerrit_raw = cfg['track_items']['raw_index_gerrit'] index_git_raw = cfg['track_items']['raw_index_git'] index_gerrit_enrich = cfg['gerrit']['enriched_index'] index_git_enrich = cfg['git']['enriched_index'] db_config = { "database": cfg['sortinghat']['database'], "user": cfg['sortinghat']['user'], "password": cfg['sortinghat']['password'], "host": cfg['sortinghat']['host'] } logger.debug("Importing track items from %s ", git_repos) # # Gerrit Reviews # gerrit_uris = [] for git_repo in git_repos: gerrit_uris += fetch_track_items(git_repo, self.ITEMS_DATA_SOURCE) gerrit_numbers = get_gerrit_numbers(gerrit_uris) logger.info("Total gerrit track items to be imported: %i", len(gerrit_numbers)) enriched_items = enrich_gerrit_items(elastic_url_raw, index_gerrit_raw, gerrit_numbers, project, db_config) logger.info("Total gerrit track items enriched: %i", len(enriched_items)) elastic = ElasticSearch(elastic_url_enrich, index_gerrit_enrich) total = elastic.bulk_upload(enriched_items, "uuid") # # Git Commits # commits_sha = get_commits_from_gerrit(elastic_url_raw, index_gerrit_raw, gerrit_numbers) logger.info("Total git track items to be checked: %i", len(commits_sha)) enriched_items = enrich_git_items(elastic_url_raw, index_git_raw, commits_sha, project, db_config) logger.info("Total git track items enriched: %i", len(enriched_items)) elastic = ElasticSearch(elastic_url_enrich, index_git_enrich) total = elastic.bulk_upload(enriched_items, "uuid")
def __enrich_items(self): time_start = time.time() # logger.info('%s starts for %s ', 'enrichment', self.backend_section) logger.info('[%s] enrichment phase starts', self.backend_section) print("Enrichment for {}: starting...".format(self.backend_section)) cfg = self.config.get_conf() if 'scroll_size' in cfg['general']: ElasticItems.scroll_size = cfg['general']['scroll_size'] if 'bulk_size' in cfg['general']: ElasticSearch.max_items_bulk = cfg['general']['bulk_size'] no_incremental = False github_token = None pair_programming = False node_regex = None if 'github' in cfg and 'backend_token' in cfg['github']: github_token = cfg['github']['backend_token'] if 'git' in cfg and 'pair-programming' in cfg['git']: pair_programming = cfg['git']['pair-programming'] if 'jenkins' in cfg and 'node_regex' in cfg['jenkins']: node_regex = cfg['jenkins']['node_regex'] only_studies = False only_identities = False # repos could change between executions because changes in projects repos = TaskProjects.get_repos_by_backend_section(self.backend_section, raw=False) if not repos: logger.warning("No enrich repositories for %s", self.backend_section) # Get the metadata__timestamp value of the last item inserted in the enriched index before # looping over the repos which data is stored in the same index. This is needed to make sure # that the incremental enrichment works for data sources that are collected globally but only # partially enriched. elastic_enrich = get_elastic(cfg['es_enrichment']['url'], cfg[self.backend_section]['enriched_index']) last_enrich_date = elastic_enrich.get_last_item_field("metadata__timestamp") if last_enrich_date: last_enrich_date = last_enrich_date.replace(second=0, microsecond=0, tzinfo=None) for repo in repos: # First process p2o params from repo p2o_args = self._compose_p2o_params(self.backend_section, repo) filter_raw = p2o_args['filter-raw'] if 'filter-raw' in p2o_args else None filters_raw_prefix = p2o_args['filter-raw-prefix'] if 'filter-raw-prefix' in p2o_args else None jenkins_rename_file = p2o_args['jenkins-rename-file'] if 'jenkins-rename-file' in p2o_args else None url = p2o_args['url'] # Second process perceval params from repo backend_args = self._compose_perceval_params(self.backend_section, url) studies_args = None backend = self.get_backend(self.backend_section) if 'studies' in self.conf[self.backend_section] and \ self.conf[self.backend_section]['studies']: studies_args = self.__load_studies() logger.info('[%s] enrichment starts for %s', self.backend_section, repo) es_enrich_aliases = self.select_aliases(cfg, self.backend_section) try: es_col_url = self._get_collection_url() enrich_backend(es_col_url, self.clean, backend, backend_args, self.backend_section, cfg[self.backend_section]['raw_index'], cfg[self.backend_section]['enriched_index'], None, # projects_db is deprecated cfg['projects']['projects_file'], cfg['sortinghat']['database'], no_incremental, only_identities, github_token, False, # studies are executed in its own Task only_studies, cfg['es_enrichment']['url'], None, # args.events_enrich cfg['sortinghat']['user'], cfg['sortinghat']['password'], cfg['sortinghat']['host'], None, # args.refresh_projects, None, # args.refresh_identities, author_id=None, author_uuid=None, filter_raw=filter_raw, filters_raw_prefix=filters_raw_prefix, jenkins_rename_file=jenkins_rename_file, unaffiliated_group=cfg['sortinghat']['unaffiliated_group'], pair_programming=pair_programming, node_regex=node_regex, studies_args=studies_args, es_enrich_aliases=es_enrich_aliases, last_enrich_date=last_enrich_date) except Exception as ex: logger.error("Something went wrong producing enriched data for %s . " "Using the backend_args: %s ", self.backend_section, str(backend_args)) logger.error("Exception: %s", ex) raise DataEnrichmentError('Failed to produce enriched data for ' + self.backend_section) logger.info('[%s] enrichment finished for %s', self.backend_section, repo) spent_time = time.strftime("%H:%M:%S", time.gmtime(time.time() - time_start)) logger.info('[%s] enrichment phase finished in %s', self.backend_section, spent_time) print("Enrichment for {}: finished after {} hours".format(self.backend_section, spent_time))