def _process_repo_vcs(repo, runtime_storage_inst, record_processor_inst): vcs_inst = vcs.get_vcs(repo, CONF.sources_root) vcs_inst.fetch() gerrit_hostname, _ = rcs.get_socket_tuple_from_uri( repo.get('gerrit_uri', CONF.review_uri) ) for branch in _get_repo_branches(repo): LOG.info('Processing commits in repo: %s, branch: %s', repo['uri'], branch) quoted_uri = six.moves.urllib.parse.quote_plus(repo['uri']) vcs_key = 'vcs:%s:%s' % (quoted_uri, branch) last_id = runtime_storage_inst.get_by_key(vcs_key) commit_iterator = vcs_inst.log(branch, last_id) commit_iterator_review = _param_adder( commit_iterator, 'gerrit_hostname', gerrit_hostname ) commit_iterator_typed = _record_typer(commit_iterator_review, 'commit') processed_commit_iterator = record_processor_inst.process( commit_iterator_typed) runtime_storage_inst.set_records( processed_commit_iterator, _merge_commits) last_id = vcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(vcs_key, last_id)
def process(runtime_storage_inst, default_data, sources_root, force_update): LOG.debug('Process default data') normalizer.normalize_default_data(default_data) dd_changed = _check_default_data_change(runtime_storage_inst, default_data) if 'project_sources' in default_data: if not _retrieve_project_list(default_data): raise Exception('Unable to retrieve project list') _update_default_data(runtime_storage_inst, default_data) if (dd_changed or force_update): LOG.debug('Gather release index for all repos') release_index = {} for repo in utils.load_repos(runtime_storage_inst): vcs_inst = vcs.get_vcs(repo, sources_root) release_index.update(vcs_inst.get_release_index()) record_processor_inst = record_processor.RecordProcessor( runtime_storage_inst) # need to iterate over full view of records and generate valid # users profiles LOG.debug('Iterate all records to create valid users profiles') for record in runtime_storage_inst.get_all_records(): record_processor_inst.update_user(record) # update records according to generated users profiles LOG.debug('Update all records according to users profiles') updated_records = record_processor_inst.update( runtime_storage_inst.get_all_records(), release_index) runtime_storage_inst.set_records(updated_records)
def process(runtime_storage_inst, default_data, sources_root, force_update): LOG.debug('Process default data') normalizer.normalize_default_data(default_data) if (_check_default_data_change(runtime_storage_inst, default_data) or force_update): _update_default_data(runtime_storage_inst, default_data) LOG.debug('Gather release index for all repos') release_index = {} for repo in runtime_storage_inst.get_by_key('repos'): vcs_inst = vcs.get_vcs(repo, sources_root) release_index.update(vcs_inst.get_release_index()) record_processor_inst = record_processor.RecordProcessor( runtime_storage_inst) # need to iterate over full view of records and generate valid # users profiles LOG.debug('Iterate all records to create valid users profiles') for record in runtime_storage_inst.get_all_records(): record_processor_inst.update_user(record) # update records according to generated users profiles LOG.debug('Update all records according to users profiles') updated_records = record_processor_inst.update( runtime_storage_inst.get_all_records(), release_index) runtime_storage_inst.set_records(updated_records) if 'project_sources' in default_data: _retrieve_project_list(runtime_storage_inst, default_data['project_sources'])
def process(runtime_storage_inst, default_data, sources_root, force_update): LOG.debug('Process default data') normalizer.normalize_default_data(default_data) if (_check_default_data_change(runtime_storage_inst, default_data) or force_update): _update_default_data(runtime_storage_inst, default_data) LOG.debug('Gather release index for all repos') release_index = {} for repo in runtime_storage_inst.get_by_key('repos'): vcs_inst = vcs.get_vcs(repo, sources_root) release_index.update(vcs_inst.get_release_index()) record_processor_inst = record_processor.RecordProcessor( runtime_storage_inst) # need to iterate over full view of records and generate valid # users profiles LOG.debug('Iterate all records to create valid users profiles') for record in record_processor_inst.update( runtime_storage_inst.get_all_records(), release_index): pass # update records according to generated users profiles LOG.debug('Update all records according to users profiles') updated_records = record_processor_inst.update( runtime_storage_inst.get_all_records(), release_index) runtime_storage_inst.set_records(updated_records) if 'project_sources' in default_data: _retrieve_project_list(runtime_storage_inst, default_data['project_sources'])
def _process_repo(repo, runtime_storage_inst, record_processor_inst, rcs_inst): uri = repo["uri"] LOG.info("Processing repo uri: %s", uri) LOG.debug("Processing blueprints for repo uri: %s", uri) bp_iterator = lp.log(repo) bp_iterator_typed = _record_typer(bp_iterator, "bp") processed_bp_iterator = record_processor_inst.process(bp_iterator_typed) runtime_storage_inst.set_records(processed_bp_iterator, utils.merge_records) LOG.debug("Processing bugs for repo uri: %s", uri) current_date = utils.date_to_timestamp("now") bug_modified_since = runtime_storage_inst.get_by_key("bug_modified_since-%s" % repo["module"]) bug_iterator = bps.log(repo, bug_modified_since) bug_iterator_typed = _record_typer(bug_iterator, "bug") processed_bug_iterator = record_processor_inst.process(bug_iterator_typed) runtime_storage_inst.set_records(processed_bug_iterator, utils.merge_records) runtime_storage_inst.set_by_key("bug_modified_since-%s" % repo["module"], current_date) vcs_inst = vcs.get_vcs(repo, cfg.CONF.sources_root) vcs_inst.fetch() branches = {repo.get("default_branch", "master")} for release in repo.get("releases"): if "branch" in release: branches.add(release["branch"]) for branch in branches: LOG.debug("Processing commits in repo: %s, branch: %s", uri, branch) vcs_key = "vcs:" + str(parse.quote_plus(uri) + ":" + branch) last_id = runtime_storage_inst.get_by_key(vcs_key) commit_iterator = vcs_inst.log(branch, last_id) commit_iterator_typed = _record_typer(commit_iterator, "commit") processed_commit_iterator = record_processor_inst.process(commit_iterator_typed) runtime_storage_inst.set_records(processed_commit_iterator, _merge_commits) last_id = vcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(vcs_key, last_id) LOG.debug("Processing reviews for repo: %s, branch: %s", uri, branch) rcs_key = "rcs:" + str(parse.quote_plus(uri) + ":" + branch) last_id = runtime_storage_inst.get_by_key(rcs_key) review_iterator = rcs_inst.log(repo, branch, last_id, grab_comments=("ci" in repo)) review_iterator_typed = _record_typer(review_iterator, "review") if "ci" in repo: # add external CI data review_iterator_typed = _process_reviews(review_iterator_typed, repo["ci"], repo["module"], branch) processed_review_iterator = record_processor_inst.process(review_iterator_typed) runtime_storage_inst.set_records(processed_review_iterator, utils.merge_records) last_id = rcs_inst.get_last_id(repo, branch) runtime_storage_inst.set_by_key(rcs_key, last_id)
def _post_process_records(record_processor_inst, repos): LOG.debug('Build release index') release_index = {} for repo in repos: vcs_inst = vcs.get_vcs(repo, cfg.CONF.sources_root) release_index.update(vcs_inst.fetch()) LOG.debug('Post-process all records') record_processor_inst.post_processing(release_index)
def _update_records(runtime_storage_inst, sources_root): LOG.debug("Update existing records") release_index = {} for repo in utils.load_repos(runtime_storage_inst): vcs_inst = vcs.get_vcs(repo, sources_root) release_index.update(vcs_inst.get_release_index()) record_processor_inst = record_processor.RecordProcessor(runtime_storage_inst) record_processor_inst.update(release_index)
def _update_records(runtime_storage_inst, sources_root): LOG.debug('Update existing records') release_index = {} for repo in utils.load_repos(runtime_storage_inst): vcs_inst = vcs.get_vcs(repo, sources_root) release_index.update(vcs_inst.get_release_index()) record_processor_inst = record_processor.RecordProcessor( runtime_storage_inst) record_processor_inst.update(release_index)
def process_repo(repo, runtime_storage_inst, record_processor_inst): uri = repo['uri'] LOG.debug('Processing repo uri %s' % uri) # OpenDaylight is not using launchpad at this time -- ignore # bp_iterator = lp.log(repo) # bp_iterator_typed = _record_typer(bp_iterator, 'bp') # processed_bp_iterator = record_processor_inst.process( # bp_iterator_typed) #runtime_storage_inst.set_records(processed_bp_iterator, # utils.merge_records) vcs_inst = vcs.get_vcs(repo, cfg.CONF.sources_root) vcs_inst.fetch() rcs_inst = rcs.get_rcs(repo, cfg.CONF.review_uri) rcs_inst.setup(key_filename=cfg.CONF.ssh_key_filename, username=cfg.CONF.ssh_username) branches = set(['master']) for release in repo.get('releases'): if 'branch' in release: branches.add(release['branch']) for branch in branches: LOG.debug('Processing repo %s, branch %s', uri, branch) vcs_key = 'vcs:' + str(urllib.quote_plus(uri) + ':' + branch) last_id = runtime_storage_inst.get_by_key(vcs_key) commit_iterator = vcs_inst.log(branch, last_id) commit_iterator_typed = _record_typer(commit_iterator, 'commit') processed_commit_iterator = record_processor_inst.process( commit_iterator_typed) runtime_storage_inst.set_records( processed_commit_iterator, _merge_commits) last_id = vcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(vcs_key, last_id) LOG.debug('Processing reviews for repo %s, branch %s', uri, branch) rcs_key = 'rcs:' + str(urllib.quote_plus(uri) + ':' + branch) last_id = runtime_storage_inst.get_by_key(rcs_key) review_iterator = rcs_inst.log(branch, last_id) review_iterator_typed = _record_typer(review_iterator, 'review') processed_review_iterator = record_processor_inst.process( review_iterator_typed) runtime_storage_inst.set_records(processed_review_iterator, utils.merge_records) last_id = rcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(rcs_key, last_id)
def process_repo(repo, runtime_storage_inst, record_processor_inst): uri = repo['uri'] LOG.debug('Processing repo uri %s' % uri) bp_iterator = lp.log(repo) bp_iterator_typed = _record_typer(bp_iterator, 'bp') processed_bp_iterator = record_processor_inst.process(bp_iterator_typed) runtime_storage_inst.set_records(processed_bp_iterator, utils.merge_records) vcs_inst = vcs.get_vcs(repo, cfg.CONF.sources_root) vcs_inst.fetch() rcs_inst = rcs.get_rcs(repo, cfg.CONF.review_uri) rcs_inst.setup(key_filename=cfg.CONF.ssh_key_filename, username=cfg.CONF.ssh_username) branches = set(['master']) for release in repo.get('releases'): if 'branch' in release: branches.add(release['branch']) for branch in branches: LOG.debug('Processing repo %s, branch %s', uri, branch) vcs_key = 'vcs:' + str(parse.quote_plus(uri) + ':' + branch) last_id = runtime_storage_inst.get_by_key(vcs_key) commit_iterator = vcs_inst.log(branch, last_id) commit_iterator_typed = _record_typer(commit_iterator, 'commit') processed_commit_iterator = record_processor_inst.process( commit_iterator_typed) runtime_storage_inst.set_records(processed_commit_iterator, _merge_commits) last_id = vcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(vcs_key, last_id) LOG.debug('Processing reviews for repo %s, branch %s', uri, branch) rcs_key = 'rcs:' + str(parse.quote_plus(uri) + ':' + branch) last_id = runtime_storage_inst.get_by_key(rcs_key) review_iterator = rcs_inst.log(branch, last_id) review_iterator_typed = _record_typer(review_iterator, 'review') processed_review_iterator = record_processor_inst.process( review_iterator_typed) runtime_storage_inst.set_records(processed_review_iterator, utils.merge_records) last_id = rcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(rcs_key, last_id)
def process_repo(repo, runtime_storage_inst, record_processor_inst): uri = repo['uri'] LOG.debug('Processing repo uri %s' % uri) bp_iterator = lp.log(repo) bp_iterator_typed = _record_typer(bp_iterator, 'bp') processed_bp_iterator = record_processor_inst.process( bp_iterator_typed) runtime_storage_inst.set_records(processed_bp_iterator) vcs_inst = vcs.get_vcs(repo, cfg.CONF.sources_root) vcs_inst.fetch() rcs_inst = rcs.get_rcs(repo, cfg.CONF.review_uri) rcs_inst.setup(key_filename=cfg.CONF.ssh_key_filename, username=cfg.CONF.ssh_username) for branch in repo['branches']: LOG.debug('Processing repo %s, branch %s', uri, branch) vcs_key = 'vcs:' + str(urllib.quote_plus(uri) + ':' + branch) last_id = runtime_storage_inst.get_by_key(vcs_key) commit_iterator = vcs_inst.log(branch, last_id) commit_iterator_typed = _record_typer(commit_iterator, 'commit') processed_commit_iterator = record_processor_inst.process( commit_iterator_typed) runtime_storage_inst.set_records( processed_commit_iterator, _merge_commits) last_id = vcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(vcs_key, last_id) LOG.debug('Processing reviews for repo %s, branch %s', uri, branch) rcs_key = 'rcs:' + str(urllib.quote_plus(uri) + ':' + branch) last_id = runtime_storage_inst.get_by_key(rcs_key) review_iterator = rcs_inst.log(branch, last_id) review_iterator_typed = _record_typer(review_iterator, 'review') processed_review_iterator = record_processor_inst.process( review_iterator_typed) runtime_storage_inst.set_records(processed_review_iterator) last_id = rcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(rcs_key, last_id)
def _process_repo_vcs(repo, runtime_storage_inst, record_processor_inst): vcs_inst = vcs.get_vcs(repo, CONF.sources_root) vcs_inst.fetch() for branch in _get_repo_branches(repo): LOG.info('Processing commits in repo: %s, branch: %s', repo['uri'], branch) quoted_uri = six.moves.urllib.parse.quote_plus(repo['uri']) vcs_key = 'vcs:%s:%s' % (quoted_uri, branch) last_id = runtime_storage_inst.get_by_key(vcs_key) commit_iterator = vcs_inst.log(branch, last_id) commit_iterator_typed = _record_typer(commit_iterator, 'commit') processed_commit_iterator = record_processor_inst.process( commit_iterator_typed) runtime_storage_inst.set_records( processed_commit_iterator, _merge_commits) last_id = vcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(vcs_key, last_id)
def _process_repo_vcs(repo, runtime_storage_inst, record_processor_inst): vcs_inst = vcs.get_vcs(repo, CONF.sources_root) vcs_inst.fetch() for branch in _get_repo_branches(repo): LOG.info('Processing commits in repo: %s, branch: %s', repo['uri'], branch) quoted_uri = six.moves.urllib.parse.quote_plus(repo['uri']) vcs_key = 'vcs:%s:%s' % (quoted_uri, branch) last_id = runtime_storage_inst.get_by_key(vcs_key) commit_iterator = vcs_inst.log(branch, last_id) commit_iterator_typed = _record_typer(commit_iterator, 'commit') processed_commit_iterator = record_processor_inst.process( commit_iterator_typed) runtime_storage_inst.set_records(processed_commit_iterator, _merge_commits) last_id = vcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(vcs_key, last_id)
def process_repo(repo, runtime_storage, record_processor_inst): uri = repo['uri'] LOG.debug('Processing repo uri %s' % uri) vcs_inst = vcs.get_vcs(repo, cfg.CONF.sources_root) vcs_inst.fetch() rcs_inst = rcs.get_rcs(repo, cfg.CONF.review_uri) rcs_inst.setup(key_filename=cfg.CONF.ssh_key_filename, username=cfg.CONF.ssh_username) for branch in repo['branches']: LOG.debug('Processing repo %s, branch %s', uri, branch) vcs_key = 'vcs:' + str(urllib.quote_plus(uri) + ':' + branch) last_id = runtime_storage.get_by_key(vcs_key) commit_iterator = vcs_inst.log(branch, last_id) commit_iterator_typed = _record_typer(commit_iterator, 'commit') processed_commit_iterator = record_processor_inst.process( commit_iterator_typed) runtime_storage.set_records(processed_commit_iterator, _merge_commits) last_id = vcs_inst.get_last_id(branch) runtime_storage.set_by_key(vcs_key, last_id) LOG.debug('Processing reviews for repo %s, branch %s', uri, branch) rcs_key = 'rcs:' + str(urllib.quote_plus(uri) + ':' + branch) last_id = runtime_storage.get_by_key(rcs_key) review_iterator = rcs_inst.log(branch, last_id) review_iterator_typed = _record_typer(review_iterator, 'review') processed_review_iterator = record_processor_inst.process( review_iterator_typed) runtime_storage.set_records(processed_review_iterator) last_id = rcs_inst.get_last_id(branch) runtime_storage.set_by_key(rcs_key, last_id)
def process(runtime_storage_inst, default_data, sources_root): normalizer.normalize_default_data(default_data) if _check_default_data_change(runtime_storage_inst, default_data): _update_default_data(runtime_storage_inst, default_data) release_index = {} for repo in runtime_storage_inst.get_by_key('repos'): vcs_inst = vcs.get_vcs(repo, sources_root) release_index.update(vcs_inst.get_release_index()) record_processor_inst = record_processor.RecordProcessor( runtime_storage_inst) updated_records = record_processor_inst.update( runtime_storage_inst.get_all_records(), release_index) runtime_storage_inst.set_records(updated_records) if 'project_sources' in default_data: _retrieve_project_list(runtime_storage_inst, default_data['project_sources'])
def _process_repo(repo, runtime_storage_inst, record_processor_inst, rcs_inst): uri = repo['uri'] quoted_uri = six.moves.urllib.parse.quote_plus(uri) LOG.info('Processing repo uri: %s', uri) LOG.info('Processing blueprints for repo uri: %s', uri) bp_iterator = lp.log(repo) bp_iterator_typed = _record_typer(bp_iterator, 'bp') processed_bp_iterator = record_processor_inst.process(bp_iterator_typed) runtime_storage_inst.set_records(processed_bp_iterator, utils.merge_records) LOG.info('Processing bugs for repo uri: %s', uri) current_date = utils.date_to_timestamp('now') bug_modified_since = runtime_storage_inst.get_by_key( 'bug_modified_since-%s' % repo['module']) bug_iterator = bps.log(repo, bug_modified_since) bug_iterator_typed = _record_typer(bug_iterator, 'bug') processed_bug_iterator = record_processor_inst.process(bug_iterator_typed) runtime_storage_inst.set_records(processed_bug_iterator, utils.merge_records) runtime_storage_inst.set_by_key('bug_modified_since-%s' % repo['module'], current_date) vcs_inst = vcs.get_vcs(repo, cfg.CONF.sources_root) vcs_inst.fetch() branches = {repo.get('default_branch', 'master')} for release in repo.get('releases'): if 'branch' in release: branches.add(release['branch']) for branch in branches: LOG.info('Processing commits in repo: %s, branch: %s', uri, branch) vcs_key = 'vcs:%s:%s' % (quoted_uri, branch) last_id = runtime_storage_inst.get_by_key(vcs_key) commit_iterator = vcs_inst.log(branch, last_id) commit_iterator_typed = _record_typer(commit_iterator, 'commit') processed_commit_iterator = record_processor_inst.process( commit_iterator_typed) runtime_storage_inst.set_records(processed_commit_iterator, _merge_commits) last_id = vcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(vcs_key, last_id) LOG.info('Processing reviews for repo: %s, branch: %s', uri, branch) rcs_key = 'rcs:%s:%s' % (quoted_uri, branch) last_retrieval_time = runtime_storage_inst.get_by_key(rcs_key) current_retrieval_time = int(time.time()) review_iterator = itertools.chain( rcs_inst.log(repo, branch, last_retrieval_time, status='open'), rcs_inst.log(repo, branch, last_retrieval_time, status='merged'), rcs_inst.log(repo, branch, last_retrieval_time, status='abandoned', grab_comments=True), ) review_iterator_typed = _record_typer(review_iterator, 'review') processed_review_iterator = record_processor_inst.process( review_iterator_typed) runtime_storage_inst.set_records(processed_review_iterator, utils.merge_records) runtime_storage_inst.set_by_key(rcs_key, current_retrieval_time) if 'drivers' in repo: LOG.info('Processing CI votes for repo: %s, branch: %s', uri, branch) rcs_key = 'ci:%s:%s' % (quoted_uri, branch) last_retrieval_time = runtime_storage_inst.get_by_key(rcs_key) current_retrieval_time = int(time.time()) review_iterator = rcs_inst.log(repo, branch, last_retrieval_time, status='merged', grab_comments=True) review_iterator = driverlog.log(review_iterator, repo['drivers']) review_iterator_typed = _record_typer(review_iterator, 'ci') processed_review_iterator = record_processor_inst.process( review_iterator_typed) runtime_storage_inst.set_records(processed_review_iterator, utils.merge_records) runtime_storage_inst.set_by_key(rcs_key, current_retrieval_time)
def _process_repo(repo, runtime_storage_inst, record_processor_inst, bug_modified_since): uri = repo['uri'] LOG.info('Processing repo uri: %s', uri) LOG.debug('Processing blueprints for repo uri: %s', uri) bp_iterator = lp.log(repo) bp_iterator_typed = _record_typer(bp_iterator, 'bp') processed_bp_iterator = record_processor_inst.process( bp_iterator_typed) runtime_storage_inst.set_records(processed_bp_iterator, utils.merge_records) LOG.debug('Processing bugs for repo uri: %s', uri) bug_iterator = bps.log(repo, bug_modified_since) bug_iterator_typed = _record_typer(bug_iterator, 'bug') processed_bug_iterator = record_processor_inst.process( bug_iterator_typed) runtime_storage_inst.set_records(processed_bug_iterator, utils.merge_records) vcs_inst = vcs.get_vcs(repo, cfg.CONF.sources_root) vcs_inst.fetch() rcs_inst = rcs.get_rcs(repo, cfg.CONF.review_uri) rcs_inst.setup(key_filename=cfg.CONF.ssh_key_filename, username=cfg.CONF.ssh_username) branches = set(['master']) for release in repo.get('releases'): if 'branch' in release: branches.add(release['branch']) for branch in branches: LOG.debug('Processing commits in repo: %s, branch: %s', uri, branch) vcs_key = 'vcs:' + str(parse.quote_plus(uri) + ':' + branch) last_id = runtime_storage_inst.get_by_key(vcs_key) commit_iterator = vcs_inst.log(branch, last_id) commit_iterator_typed = _record_typer(commit_iterator, 'commit') processed_commit_iterator = record_processor_inst.process( commit_iterator_typed) runtime_storage_inst.set_records( processed_commit_iterator, _merge_commits) last_id = vcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(vcs_key, last_id) LOG.debug('Processing reviews for repo: %s, branch: %s', uri, branch) rcs_key = 'rcs:' + str(parse.quote_plus(uri) + ':' + branch) last_id = runtime_storage_inst.get_by_key(rcs_key) review_iterator = rcs_inst.log(branch, last_id, grab_comments=('ci' in repo)) review_iterator_typed = _record_typer(review_iterator, 'review') if 'ci' in repo: # add external CI data review_iterator_typed = _process_reviews( review_iterator_typed, repo['ci'], repo['module'], branch) processed_review_iterator = record_processor_inst.process( review_iterator_typed) runtime_storage_inst.set_records(processed_review_iterator, utils.merge_records) last_id = rcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(rcs_key, last_id)
def _process_repo(repo, runtime_storage_inst, record_processor_inst, bug_modified_since): uri = repo['uri'] LOG.info('Processing repo uri: %s', uri) LOG.debug('Processing blueprints for repo uri: %s', uri) bp_iterator = lp.log(repo) bp_iterator_typed = _record_typer(bp_iterator, 'bp') processed_bp_iterator = record_processor_inst.process(bp_iterator_typed) runtime_storage_inst.set_records(processed_bp_iterator, utils.merge_records) LOG.debug('Processing bugs for repo uri: %s', uri) bug_iterator = bps.log(repo, bug_modified_since) bug_iterator_typed = _record_typer(bug_iterator, 'bug') processed_bug_iterator = record_processor_inst.process(bug_iterator_typed) runtime_storage_inst.set_records(processed_bug_iterator, utils.merge_records) vcs_inst = vcs.get_vcs(repo, cfg.CONF.sources_root) vcs_inst.fetch() rcs_inst = rcs.get_rcs(repo, cfg.CONF.review_uri) rcs_inst.setup(key_filename=cfg.CONF.ssh_key_filename, username=cfg.CONF.ssh_username) branches = set(['master']) for release in repo.get('releases'): if 'branch' in release: branches.add(release['branch']) for branch in branches: LOG.debug('Processing commits in repo: %s, branch: %s', uri, branch) vcs_key = 'vcs:' + str(parse.quote_plus(uri) + ':' + branch) last_id = runtime_storage_inst.get_by_key(vcs_key) commit_iterator = vcs_inst.log(branch, last_id) commit_iterator_typed = _record_typer(commit_iterator, 'commit') processed_commit_iterator = record_processor_inst.process( commit_iterator_typed) runtime_storage_inst.set_records(processed_commit_iterator, _merge_commits) last_id = vcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(vcs_key, last_id) LOG.debug('Processing reviews for repo: %s, branch: %s', uri, branch) rcs_key = 'rcs:' + str(parse.quote_plus(uri) + ':' + branch) last_id = runtime_storage_inst.get_by_key(rcs_key) review_iterator = rcs_inst.log(branch, last_id, grab_comments=('ci' in repo)) review_iterator_typed = _record_typer(review_iterator, 'review') if 'ci' in repo: # add external CI data review_iterator_typed = _process_reviews(review_iterator_typed, repo['ci'], repo['module'], branch) processed_review_iterator = record_processor_inst.process( review_iterator_typed) runtime_storage_inst.set_records(processed_review_iterator, utils.merge_records) last_id = rcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(rcs_key, last_id)
def _process_repo(repo, runtime_storage_inst, record_processor_inst, rcs_inst): uri = repo['uri'] quoted_uri = six.moves.urllib.parse.quote_plus(uri) LOG.info('Processing repo uri: %s', uri) LOG.info('Processing blueprints for repo uri: %s', uri) bp_iterator = lp.log(repo) bp_iterator_typed = _record_typer(bp_iterator, 'bp') processed_bp_iterator = record_processor_inst.process( bp_iterator_typed) runtime_storage_inst.set_records(processed_bp_iterator, utils.merge_records) LOG.info('Processing bugs for repo uri: %s', uri) current_date = utils.date_to_timestamp('now') bug_modified_since = runtime_storage_inst.get_by_key( 'bug_modified_since-%s' % repo['module']) bug_iterator = bps.log(repo, bug_modified_since) bug_iterator_typed = _record_typer(bug_iterator, 'bug') processed_bug_iterator = record_processor_inst.process( bug_iterator_typed) runtime_storage_inst.set_records(processed_bug_iterator, utils.merge_records) runtime_storage_inst.set_by_key( 'bug_modified_since-%s' % repo['module'], current_date) vcs_inst = vcs.get_vcs(repo, cfg.CONF.sources_root) vcs_inst.fetch() branches = {repo.get('default_branch', 'master')} for release in repo.get('releases'): if 'branch' in release: branches.add(release['branch']) for branch in branches: LOG.info('Processing commits in repo: %s, branch: %s', uri, branch) vcs_key = 'vcs:%s:%s' % (quoted_uri, branch) last_id = runtime_storage_inst.get_by_key(vcs_key) commit_iterator = vcs_inst.log(branch, last_id) commit_iterator_typed = _record_typer(commit_iterator, 'commit') processed_commit_iterator = record_processor_inst.process( commit_iterator_typed) runtime_storage_inst.set_records( processed_commit_iterator, _merge_commits) last_id = vcs_inst.get_last_id(branch) runtime_storage_inst.set_by_key(vcs_key, last_id) if 'has_gerrit' not in repo: continue # do not poll reviews for those that do not have them LOG.info('Processing reviews for repo: %s, branch: %s', uri, branch) rcs_key = 'rcs:%s:%s' % (quoted_uri, branch) last_retrieval_time = runtime_storage_inst.get_by_key(rcs_key) current_retrieval_time = int(time.time()) review_iterator = itertools.chain( rcs_inst.log(repo, branch, last_retrieval_time, status='open'), rcs_inst.log(repo, branch, last_retrieval_time, status='merged'), rcs_inst.log(repo, branch, last_retrieval_time, status='abandoned', grab_comments=True), ) review_iterator_typed = _record_typer(review_iterator, 'review') processed_review_iterator = record_processor_inst.process( review_iterator_typed) runtime_storage_inst.set_records(processed_review_iterator, utils.merge_records) runtime_storage_inst.set_by_key(rcs_key, current_retrieval_time) if 'drivers' in repo: LOG.info('Processing CI votes for repo: %s, branch: %s', uri, branch) rcs_key = 'ci:%s:%s' % (quoted_uri, branch) last_retrieval_time = runtime_storage_inst.get_by_key(rcs_key) current_retrieval_time = int(time.time()) review_iterator = rcs_inst.log(repo, branch, last_retrieval_time, status='merged', grab_comments=True) review_iterator = driverlog.log(review_iterator, repo['drivers']) review_iterator_typed = _record_typer(review_iterator, 'ci') processed_review_iterator = record_processor_inst.process( review_iterator_typed) runtime_storage_inst.set_records(processed_review_iterator, utils.merge_records) runtime_storage_inst.set_by_key(rcs_key, current_retrieval_time)