def __init__(self, project_name, project_info, local_dir, fetch=True): self.project_name = project_name self.recombinations = dict() self.commits = dict() log.info('Current project:\n' + pprint.pformat(project_info)) self.original_project = project_info['original'] self.replica_project = project_info['replica'] self.deploy_name = project_info['deploy-name'] self.rev_deps = None if 'rev-deps' in project_info: self.rev_deps = project_info['rev-deps'] self.test_types = [] if "tests" in project_info['replica'] and project_info["replica"]["tests"] is not None: self.test_types = project_info["replica"]["tests"] self.replication_strategy = project_info['replication-strategy'] self.test_minimum_score = 0 self.patches_branch_suffix = "-patches" self.target_branch_suffix = "-tag" # Set up local repo self.underlayer = Underlayer(project_name, local_dir) # Set up remotes self.underlayer.set_replica(self.replica_project['location'], self.replica_project['name'], fetch=fetch) self.underlayer.set_original(self.original_project['type'], self.original_project['location'], self.original_project['name'], fetch=fetch) if "mirror" in project_info['replica']: self.underlayer.set_replica_mirror(project_info['replica']['mirror'], self.replica_project['name'],fetch=fetch) # Set up branches hypermap # get branches from original # self.original_branches = self.underlayer.list_branches('original') self.original_branches = project_info['original']['watch-branches'] self.backports_startref = dict() for original_branch in self.original_branches: if 'backports-start' in self.original_project: self.backports_startref[original_branch] = self.original_project['backports-start'][original_branch] # apply mapping to find target branch try: replica_branch = project_info['replica']['branch-mappings'][original_branch] except KeyError: replica_branch = original_branch target_branch = '%s%s' % (replica_branch, self.target_branch_suffix) patches_branch = '%s%s' % (replica_branch, self.patches_branch_suffix) self.underlayer.set_branch_maps(original_branch, replica_branch, target_branch, patches_branch) self.recombinations[replica_branch] = None self.commits[replica_branch] = {} self.ref_locks = dict() if 'ref-locks' in self.replica_project: for branch in self.replica_project['ref-locks']: # no advancement will be performed past this revision on this branch self.ref_locks[branch] = self.replica_project['ref-locks'][branch]
def __init__(self, project_name, project_info, local_dir, fetch=True): self.project_name = project_name self.recombinations = dict() self.commits = dict() log.info('Current project:\n' + pprint.pformat(project_info)) self.original_project = project_info['original'] self.replica_project = project_info['replica'] self.deploy_name = project_info['deploy-name'] self.rev_deps = None if 'rev-deps' in project_info: self.rev_deps = project_info['rev-deps'] self.test_types = [] if "tests" in project_info['replica'] and project_info["replica"][ "tests"] is not None: self.test_types = project_info["replica"]["tests"] self.replication_strategy = project_info['replication-strategy'] self.test_minimum_score = 0 self.patches_branch_suffix = "-patches" self.target_branch_suffix = "-tag" # Set up local repo self.underlayer = Underlayer(project_name, local_dir) # Set up remotes self.underlayer.set_replica(self.replica_project['location'], self.replica_project['name'], fetch=fetch) self.underlayer.set_original(self.original_project['type'], self.original_project['location'], self.original_project['name'], fetch=fetch) if "mirror" in project_info['replica']: self.underlayer.set_replica_mirror( project_info['replica']['mirror'], self.replica_project['name'], fetch=fetch) # Set up branches hypermap # get branches from original # self.original_branches = self.underlayer.list_branches('original') self.original_branches = project_info['original']['watch-branches'] self.backports_startref = dict() for original_branch in self.original_branches: if 'backports-start' in self.original_project: self.backports_startref[ original_branch] = self.original_project[ 'backports-start'][original_branch] # apply mapping to find target branch try: replica_branch = project_info['replica']['branch-mappings'][ original_branch] except KeyError: replica_branch = original_branch target_branch = '%s%s' % (replica_branch, self.target_branch_suffix) patches_branch = '%s%s' % (replica_branch, self.patches_branch_suffix) self.underlayer.set_branch_maps(original_branch, replica_branch, target_branch, patches_branch) self.recombinations[replica_branch] = None self.commits[replica_branch] = {} self.ref_locks = dict() if 'ref-locks' in self.replica_project: for branch in self.replica_project['ref-locks']: # no advancement will be performed past this revision on this branch self.ref_locks[branch] = self.replica_project['ref-locks'][ branch]
class Project(object): status_impact = {"MERGED": 2, "APPROVED": 1, "PRESENT": 1, "MISSING": 0} def __init__(self, project_name, project_info, local_dir, fetch=True): self.project_name = project_name self.recombinations = dict() self.commits = dict() log.info('Current project:\n' + pprint.pformat(project_info)) self.original_project = project_info['original'] self.replica_project = project_info['replica'] self.deploy_name = project_info['deploy-name'] self.rev_deps = None if 'rev-deps' in project_info: self.rev_deps = project_info['rev-deps'] self.test_types = [] if "tests" in project_info['replica'] and project_info["replica"][ "tests"] is not None: self.test_types = project_info["replica"]["tests"] self.replication_strategy = project_info['replication-strategy'] self.test_minimum_score = 0 self.patches_branch_suffix = "-patches" self.target_branch_suffix = "-tag" # Set up local repo self.underlayer = Underlayer(project_name, local_dir) # Set up remotes self.underlayer.set_replica(self.replica_project['location'], self.replica_project['name'], fetch=fetch) self.underlayer.set_original(self.original_project['type'], self.original_project['location'], self.original_project['name'], fetch=fetch) if "mirror" in project_info['replica']: self.underlayer.set_replica_mirror( project_info['replica']['mirror'], self.replica_project['name'], fetch=fetch) # Set up branches hypermap # get branches from original # self.original_branches = self.underlayer.list_branches('original') self.original_branches = project_info['original']['watch-branches'] self.backports_startref = dict() for original_branch in self.original_branches: if 'backports-start' in self.original_project: self.backports_startref[ original_branch] = self.original_project[ 'backports-start'][original_branch] # apply mapping to find target branch try: replica_branch = project_info['replica']['branch-mappings'][ original_branch] except KeyError: replica_branch = original_branch target_branch = '%s%s' % (replica_branch, self.target_branch_suffix) patches_branch = '%s%s' % (replica_branch, self.patches_branch_suffix) self.underlayer.set_branch_maps(original_branch, replica_branch, target_branch, patches_branch) self.recombinations[replica_branch] = None self.commits[replica_branch] = {} self.ref_locks = dict() if 'ref-locks' in self.replica_project: for branch in self.replica_project['ref-locks']: # no advancement will be performed past this revision on this branch self.ref_locks[branch] = self.replica_project['ref-locks'][ branch] def get_slices(self, recombinations): slices = { "MERGED": [], "APPROVED": [], "PRESENT": [], "MISSING": [], } previous_status = None previous_impact = None current_slice = {} # Slice if recombinations: recomb_list = list(recombinations) else: recomb_list = [] for index, recomb_id in enumerate(recomb_list): replica_change = recombinations[recomb_id] # creates slices to apply to change list # every status may have multiple slices, but this situation is tolerated # only between PRESENT' and 'APPROVED' statuses # any other complicated mix is a violation of upstream order status = replica_change.status impact = self.status_impact[status] previous_change_id = None # Handle current status slice, archive previous slice try: segment = current_slice[status] except KeyError: if previous_status: slices[previous_status].append( copy.deepcopy(current_slice[previous_status])) del (current_slice[previous_status]) current_slice[status] = {} segment = current_slice[status] segment['start'] = index segment['end'] = index + 1 # init/extend current slice if previous_status: if impact > previous_impact: log.critical( "Constraint violation error: status %s at index %d (change:%s) of changes list in interval is more advanced than previous status %s at index %d (change: %s)" % (status, index, recomb_id, previous_status, index - 1, previous_change_id)) log.critical("This means that midstream is broken") raise ConstrainViolationError if status == previous_status: segment['end'] = segment['end'] + 1 # end of status list if index == len(list(recombinations)) - 1: slices[status].append(copy.deepcopy(current_slice[status])) del (current_slice[status]) previous_status = status previous_impact = impact previous_change_id = recomb_id return slices def scan_original_distance(self, original_branch): replica_branch = self.underlayer.branch_maps['original->replica'][ original_branch] target_branch = self.underlayer.branch_maps['original->target'][ original_branch] log.debug("Scanning distance from original branch %s" % original_branch) # if self.replication_strategy == "change-by-change" and revision_exists(self.ref_locks[replica_branch], replica_branch): # log.info("Cannot replicate branch past the specified lock") self.recombinations[ original_branch] = self.get_recombinations_by_interval( original_branch) slices = self.get_slices(self.recombinations[original_branch]) recombinations = self.recombinations[original_branch] log.debugvar('slices') # Master sync on merged changes # we really need only the last commit in the slice # we advance the master to that, and all the others will be merged too if slices['MERGED']: # one or more changes are merged in midstream, but missing in master # master is out of sync, changes need to be pushed # but check first if the change was changed with a merge commit # if yes, push THAT to master, if not, it's just a fast forward segment = slices['MERGED'][0] recomb_id = list(recombinations)[segment['end'] - 1] recombination = recombinations[recomb_id] recombination.handle_status() # Gerrit operations from approved changes # NOthing 'approved' can be merged if it has some "present" before in the history skip_list = set() for index, approved_segment in enumerate(slices['APPROVED']): for present_segment in slices['PRESENT']: if present_segment['start'] < approved_segment['start']: skip_list.add(index) for index in list(skip_list)[::-1]: segment = slices['APPROVED'].pop(index) for recomb_id in list( recombinations)[segment['start']:segment['end']]: log.warning( "Recombination %s is approved but waiting for previous unapproved changes, skipping" % recomb_id) # Merge what remains for segment in slices['APPROVED']: for recomb_id in list( recombinations)[segment['start']:segment['end']]: recombination = recombinations[recomb_id] recombination.handle_status() # Notify of presence for segment in slices['PRESENT']: for recomb_id in list( recombinations)[segment['start']:segment['end']]: recombination = recombinations[recomb_id] log.warning( "Recombination %s already present in replica gerrit as change %s and waiting for approval" % (recomb_id, recombination.number)) recombination.handle_status() # Gerrit operations for missing changes for segment in slices['MISSING']: for recomb_id in list( recombinations)[segment['start']:segment['end']]: log.warning("Recombination %s is missing from replica gerrit" % recomb_id) recombination = recombinations[recomb_id] recombination.handle_status() return True def poll_original_branches(self): for branch in self.original_branches: self.scan_original_distance(branch) def get_recombinations_by_interval(self, original_branch): ref_end = 'original/%s' % (original_branch) replica_branch = self.underlayer.branch_maps['original->replica'][ original_branch] patches_branch = self.underlayer.branch_maps['original->patches'][ original_branch] if self.replication_strategy == "change-by-change": ref_start = 'replica/%s' % (replica_branch) self.commits[replica_branch] = self.underlayer.get_commits( ref_start, ref_end) elif self.replication_strategy == "lock-and-backports": if original_branch in self.backports_startref: ref_start = self.backports_startref[original_branch] else: ref_start = self.ref_locks[replica_branch] self.commits[replica_branch] = self.underlayer.get_commits( ref_start, ref_end, first_parent=False, no_merges=True) commits = self.commits[replica_branch] diversity_refname = "replica/%s" % (patches_branch) original_ids = self.underlayer.get_original_ids(commits) replica_lock = None if replica_branch in self.ref_locks: replica_lock = self.ref_locks[replica_branch] if original_ids: recombinations = self.underlayer.get_recombinations_from_original( original_branch, original_ids, diversity_refname, self.replication_strategy, replica_lock) return recombinations return None def scan_replica_patches(self, patches_branch=None): # Mutations are only handled one at a time per branch if patches_branch: patches_branches = [patches_branch] else: patches_branches = list() for original_branch in self.original_branches: patches_branches.append( self.underlayer.branch_maps['original->patches'] [original_branch]) for patches_branch in patches_branches: recombination, remaining_changes = self.underlayer.get_recombination_from_patches( patches_branch) # TODO: handle new patchset on same branch-patches review. if recombination: recomb = recombination.__dict__ log.debugvar('recomb') recombination.handle_status() if remaining_changes: log.warning( "Remaining mutation changes %s will be handled in order one at a time after recombination %s is completed " % (' '.join(remaining_changes), recombination.uuid)) else: logsummary.info( "Project %s no new patches in patches branch %s" % (self.project_name, patches_branch)) def check_approved_recombinations(self, recomb_id=None): if recomb_id: recomb_type, branch = self.underlayer.get_scaninfo_by_recomb_id( recomb_id) if recomb_type == 'replica-mutation': patches_branch = self.underlayer.branch_maps[ 'replica->patches'][branch] self.scan_replica_patches(patches_branch=patches_branch) elif recomb_type == 'original-diversity' or recomb_type == "evolution-diversity": return self.scan_original_distance(branch) else: for branch in self.original_branches: patches_branch = self.underlayer.branch_maps[ 'original->patches'][branch] self.scan_replica_patches(patches_branch=patches_branch) self.scan_original_distance(branch) def get_reverse_dependencies(self, tags=[]): rev_deps = dict() for project in self.rev_deps: for tag in tags: if tag in self.rev_deps[project]["tags"]: rev_deps[project] = self.rev_deps[project]["tests"] break return rev_deps def fetch_untested_recombinations(self, test_basedir, recomb_id=None): changes_infos = dict() dirlist = self.underlayer.fetch_recombinations(test_basedir, "untested", recomb_id=recomb_id) if not dirlist: logsummary.info("Project '%s': no untested recombinations" % self.project_name) if not self.test_types: logsummary.info("Project '%s': no tests specified" % self.project_name) else: for change_number in dirlist: tests = dict() projects = self.get_reverse_dependencies(tags=[ 'included', 'contained', 'required', 'classes', 'functions' ]) projects[self.project_name] = self.test_types log.debugvar('projects') for name in projects: tests[name] = dict() tests[name]["types"] = dict() for test_type in projects[name]: result_file = "%s/%s/results/%s/%s_results.xml" % ( self.project_name, change_number, test_type, name) tests[name]["types"][test_type] = result_file changes_infos[change_number] = { "target_project": self.project_name, 'recombination_dir': dirlist[change_number], "recombination_id": change_number, "tests": tests, } logsummary.info("Fetched recombination %s on dir %s" % (change_number, dirlist[change_number])) return changes_infos def get_test_score(self, test_results): for project_name in test_results: for test_type in test_results[project_name]: test_output = test_results[project_name][test_type] if test_output is None: # We are accepting everything for now return (100, None) #return (0, "missing test results") return (100, None) def vote_recombinations(self, test_results, recomb_id=None): if recomb_id: recombs = [recomb_id] else: recombs = [recomb for recomb in test_results] for recomb_id in recombs: recombination = self.underlayer.get_recombination(recomb_id) test_score, test_analysis = self.get_test_score( test_results[recomb_id]) if test_score > self.test_minimum_score: if self.replication_strategy == "lock-and-backports": comment_data = dict() comment_data['backport-test-results'] = dict() build_url = os.environ.get('BUILD_URL') if build_url: comment_data['backport-test-results'][ 'message'] = "test-link: %s" % build_url else: comment_data['backport-test-results']['message'] = "" comment_data['backport-test-results']['Code-Review'] = 0 comment_data['backport-test-results']['Verified'] = "1" comment_data['backport-test-results'][ 'reviewers'] = self.replica_project[ 'success_reviewers_list'] comment = yaml.dump(comment_data) recombination.comment(comment) recombination.approve() logsummary.info("Recombination %s Approved" % recomb_id) else: recombination.reject() logsummary.info("Recombination %s Rejected: %s" % (recomb_id, test_analysis)) def delete_service_branches(self): # cleanup github repos from recomb branches WIP self.underlayer.delete_service_branches() def delete_stale_branches(self): recomb_active_branches = list() target_stale_branches = list() recomb_all_branches = self.underlayer.list_branches('replica', pattern='recomb*') infos = self.underlayer.replica_remote.query_changes_json( '"status:open AND project:%s"' % self.replica_project['name']) for info in infos: recomb_active_branches.append(info['branch']) log.debugvar('recomb_active_branches') recomb_stale_branches = list( set(recomb_all_branches) - set(recomb_active_branches)) log.debugvar('recomb_stale_branches') self.underlayer.delete_remote_branches('replica', recomb_stale_branches) for recomb_branch in recomb_stale_branches: target_stale_branches.append( re.sub('recomb-', 'target-', recomb_branch)) self.underlayer.delete_remote_branches('replica', target_stale_branches)
class Project(object): status_impact = { "MERGED": 2, "APPROVED": 1, "PRESENT": 1, "MISSING": 0 } def __init__(self, project_name, project_info, local_dir, fetch=True): self.project_name = project_name self.recombinations = dict() self.commits = dict() log.info('Current project:\n' + pprint.pformat(project_info)) self.original_project = project_info['original'] self.replica_project = project_info['replica'] self.deploy_name = project_info['deploy-name'] self.rev_deps = None if 'rev-deps' in project_info: self.rev_deps = project_info['rev-deps'] self.test_types = [] if "tests" in project_info['replica'] and project_info["replica"]["tests"] is not None: self.test_types = project_info["replica"]["tests"] self.replication_strategy = project_info['replication-strategy'] self.test_minimum_score = 0 self.patches_branch_suffix = "-patches" self.target_branch_suffix = "-tag" # Set up local repo self.underlayer = Underlayer(project_name, local_dir) # Set up remotes self.underlayer.set_replica(self.replica_project['location'], self.replica_project['name'], fetch=fetch) self.underlayer.set_original(self.original_project['type'], self.original_project['location'], self.original_project['name'], fetch=fetch) if "mirror" in project_info['replica']: self.underlayer.set_replica_mirror(project_info['replica']['mirror'], self.replica_project['name'],fetch=fetch) # Set up branches hypermap # get branches from original # self.original_branches = self.underlayer.list_branches('original') self.original_branches = project_info['original']['watch-branches'] self.backports_startref = dict() for original_branch in self.original_branches: if 'backports-start' in self.original_project: self.backports_startref[original_branch] = self.original_project['backports-start'][original_branch] # apply mapping to find target branch try: replica_branch = project_info['replica']['branch-mappings'][original_branch] except KeyError: replica_branch = original_branch target_branch = '%s%s' % (replica_branch, self.target_branch_suffix) patches_branch = '%s%s' % (replica_branch, self.patches_branch_suffix) self.underlayer.set_branch_maps(original_branch, replica_branch, target_branch, patches_branch) self.recombinations[replica_branch] = None self.commits[replica_branch] = {} self.ref_locks = dict() if 'ref-locks' in self.replica_project: for branch in self.replica_project['ref-locks']: # no advancement will be performed past this revision on this branch self.ref_locks[branch] = self.replica_project['ref-locks'][branch] def get_slices(self, recombinations): slices = { "MERGED": [], "APPROVED": [], "PRESENT": [], "MISSING": [], } previous_status = None previous_impact = None current_slice = {} # Slice if recombinations: recomb_list = list(recombinations) else: recomb_list = [] for index, recomb_id in enumerate(recomb_list): replica_change = recombinations[recomb_id] # creates slices to apply to change list # every status may have multiple slices, but this situation is tolerated # only between PRESENT' and 'APPROVED' statuses # any other complicated mix is a violation of upstream order status = replica_change.status impact = self.status_impact[status] previous_change_id = None # Handle current status slice, archive previous slice try: segment = current_slice[status] except KeyError: if previous_status: slices[previous_status].append(copy.deepcopy(current_slice[previous_status])) del(current_slice[previous_status]) current_slice[status] = {} segment = current_slice[status] segment['start'] = index segment['end'] = index + 1 # init/extend current slice if previous_status: if impact > previous_impact: log.critical("Constraint violation error: status %s at index %d (change:%s) of changes list in interval is more advanced than previous status %s at index %d (change: %s)" % (status, index, recomb_id, previous_status, index-1, previous_change_id)) log.critical("This means that midstream is broken") raise ConstrainViolationError if status == previous_status: segment['end'] = segment['end'] + 1 # end of status list if index == len(list(recombinations)) - 1: slices[status].append(copy.deepcopy(current_slice[status])) del(current_slice[status]) previous_status = status previous_impact = impact previous_change_id = recomb_id return slices def scan_original_distance(self, original_branch): replica_branch = self.underlayer.branch_maps['original->replica'][original_branch] target_branch = self.underlayer.branch_maps['original->target'][original_branch] log.debug("Scanning distance from original branch %s" % original_branch) # if self.replication_strategy == "change-by-change" and revision_exists(self.ref_locks[replica_branch], replica_branch): # log.info("Cannot replicate branch past the specified lock") self.recombinations[original_branch] = self.get_recombinations_by_interval(original_branch) slices = self.get_slices(self.recombinations[original_branch]) recombinations = self.recombinations[original_branch] log.debugvar('slices') # Master sync on merged changes # we really need only the last commit in the slice # we advance the master to that, and all the others will be merged too if slices['MERGED']: # one or more changes are merged in midstream, but missing in master # master is out of sync, changes need to be pushed # but check first if the change was changed with a merge commit # if yes, push THAT to master, if not, it's just a fast forward segment = slices['MERGED'][0] recomb_id = list(recombinations)[segment['end'] - 1] recombination = recombinations[recomb_id] recombination.handle_status() # Gerrit operations from approved changes # NOthing 'approved' can be merged if it has some "present" before in the history skip_list = set() for index, approved_segment in enumerate(slices['APPROVED']): for present_segment in slices['PRESENT']: if present_segment['start'] < approved_segment['start']: skip_list.add(index) for index in list(skip_list)[::-1]: segment = slices['APPROVED'].pop(index) for recomb_id in list(recombinations)[segment['start']:segment['end']]: log.warning("Recombination %s is approved but waiting for previous unapproved changes, skipping" % recomb_id) # Merge what remains for segment in slices['APPROVED']: for recomb_id in list(recombinations)[segment['start']:segment['end']]: recombination = recombinations[recomb_id] recombination.handle_status() # Notify of presence for segment in slices['PRESENT']: for recomb_id in list(recombinations)[segment['start']:segment['end']]: recombination = recombinations[recomb_id] log.warning("Recombination %s already present in replica gerrit as change %s and waiting for approval" % (recomb_id, recombination.number)) recombination.handle_status() # Gerrit operations for missing changes for segment in slices['MISSING']: for recomb_id in list(recombinations)[segment['start']:segment['end']]: log.warning("Recombination %s is missing from replica gerrit" % recomb_id) recombination = recombinations[recomb_id] recombination.handle_status() return True def poll_original_branches(self): for branch in self.original_branches: self.scan_original_distance(branch) def get_recombinations_by_interval(self, original_branch): ref_end = 'original/%s' % (original_branch) replica_branch = self.underlayer.branch_maps['original->replica'][original_branch] patches_branch = self.underlayer.branch_maps['original->patches'][original_branch] if self.replication_strategy == "change-by-change": ref_start = 'replica/%s' % (replica_branch) self.commits[replica_branch] = self.underlayer.get_commits(ref_start, ref_end) elif self.replication_strategy == "lock-and-backports": if original_branch in self.backports_startref: ref_start = self.backports_startref[original_branch] else: ref_start = self.ref_locks[replica_branch] self.commits[replica_branch] = self.underlayer.get_commits(ref_start, ref_end, first_parent=False, no_merges=True) commits = self.commits[replica_branch] diversity_refname = "replica/%s" % (patches_branch) original_ids = self.underlayer.get_original_ids(commits) replica_lock = None if replica_branch in self.ref_locks: replica_lock = self.ref_locks[replica_branch] if original_ids: recombinations = self.underlayer.get_recombinations_from_original(original_branch, original_ids, diversity_refname, self.replication_strategy, replica_lock) return recombinations return None def scan_replica_patches(self, patches_branch=None): # Mutations are only handled one at a time per branch if patches_branch: patches_branches = [patches_branch] else: patches_branches = list() for original_branch in self.original_branches: patches_branches.append(self.underlayer.branch_maps['original->patches'][original_branch]) for patches_branch in patches_branches: recombination, remaining_changes = self.underlayer.get_recombination_from_patches(patches_branch) # TODO: handle new patchset on same branch-patches review. if recombination: recomb = recombination.__dict__ log.debugvar('recomb') recombination.handle_status() if remaining_changes: log.warning("Remaining mutation changes %s will be handled in order one at a time after recombination %s is completed " % (' '.join(remaining_changes), recombination.uuid)) else: logsummary.info("Project %s no new patches in patches branch %s" % (self.project_name, patches_branch)) def check_approved_recombinations(self, recomb_id=None): if recomb_id: recomb_type, branch = self.underlayer.get_scaninfo_by_recomb_id(recomb_id) if recomb_type == 'replica-mutation': patches_branch = self.underlayer.branch_maps['replica->patches'][branch] self.scan_replica_patches(patches_branch=patches_branch) elif recomb_type == 'original-diversity' or recomb_type == "evolution-diversity": return self.scan_original_distance(branch) else: for branch in self.original_branches: patches_branch = self.underlayer.branch_maps['original->patches'][branch] self.scan_replica_patches(patches_branch=patches_branch) self.scan_original_distance(branch) def get_reverse_dependencies(self, tags=[]): rev_deps = dict() for project in self.rev_deps: for tag in tags: if tag in self.rev_deps[project]["tags"]: rev_deps[project] = self.rev_deps[project]["tests"] break return rev_deps def fetch_untested_recombinations(self, test_basedir, recomb_id=None): changes_infos = dict() dirlist = self.underlayer.fetch_recombinations(test_basedir, "untested", recomb_id=recomb_id) if not dirlist: logsummary.info("Project '%s': no untested recombinations" % self.project_name) if not self.test_types: logsummary.info("Project '%s': no tests specified" % self.project_name) else: for change_number in dirlist: tests = dict() projects = self.get_reverse_dependencies(tags=['included','contained','required','classes', 'functions']) projects[self.project_name] = self.test_types log.debugvar('projects') for name in projects: tests[name] = dict() tests[name]["types"] = dict() for test_type in projects[name]: result_file = "%s/%s/results/%s/%s_results.xml" % (self.project_name, change_number, test_type, name) tests[name]["types"][test_type] = result_file changes_infos[change_number] = { "target_project" : self.project_name, 'recombination_dir': dirlist[change_number], "recombination_id" : change_number, "tests": tests , } logsummary.info("Fetched recombination %s on dir %s" % (change_number, dirlist[change_number])) return changes_infos def get_test_score(self, test_results): for project_name in test_results: for test_type in test_results[project_name]: test_output = test_results[project_name][test_type] if test_output is None: # We are accepting everything for now return (100, None) #return (0, "missing test results") return (100, None) def vote_recombinations(self, test_results, recomb_id=None): if recomb_id: recombs = [recomb_id] else: recombs = [recomb for recomb in test_results] for recomb_id in recombs: recombination = self.underlayer.get_recombination(recomb_id) test_score, test_analysis = self.get_test_score(test_results[recomb_id]) if test_score > self.test_minimum_score: if self.replication_strategy == "lock-and-backports": comment_data = dict() comment_data['backport-test-results'] = dict() build_url = os.environ.get('BUILD_URL') if build_url: comment_data['backport-test-results']['message'] = "test-link: %s" % build_url else: comment_data['backport-test-results']['message'] = "" comment_data['backport-test-results']['Code-Review'] = 0 comment_data['backport-test-results']['Verified'] = "1" comment_data['backport-test-results']['reviewers'] = self.replica_project['success_reviewers_list'] comment = yaml.dump(comment_data) recombination.comment(comment) recombination.approve() logsummary.info("Recombination %s Approved" % recomb_id) else: recombination.reject() logsummary.info("Recombination %s Rejected: %s" % (recomb_id, test_analysis)) def delete_service_branches(self): # cleanup github repos from recomb branches WIP self.underlayer.delete_service_branches() def delete_stale_branches(self): recomb_active_branches = list() target_stale_branches = list() recomb_all_branches = self.underlayer.list_branches('replica', pattern='recomb*') infos = self.underlayer.replica_remote.query_changes_json('"status:open AND project:%s"' % self.replica_project['name']) for info in infos: recomb_active_branches.append(info['branch']) log.debugvar('recomb_active_branches') recomb_stale_branches = list(set(recomb_all_branches) - set(recomb_active_branches)) log.debugvar('recomb_stale_branches') self.underlayer.delete_remote_branches('replica', recomb_stale_branches) for recomb_branch in recomb_stale_branches: target_stale_branches.append(re.sub('recomb-','target-',recomb_branch)) self.underlayer.delete_remote_branches('replica', target_stale_branches)