def processCommits(self, commits, commit_started_callback, patch_callback, commit_finished_callback): if commits is None: return None # Process oldest first commits = commits[::-1] logger.debug("processing %d commits", len(commits)) for github_commit in commits: logger.debug("sha: %s", github_commit.sha) if github_commit.sha: github_commit = self._github_repo.commit(github_commit.sha) repo_commit = RepoCommit() repo_commit.url = github_commit.html_url repo_commit.repo_source = self if github_commit.date is not None: self._last_date = dateutil.parser.parse(github_commit.date) repo_commit.date = self._last_date self._last_date += datetime.timedelta(seconds=1) repo_commit.identifier = self._last_date.isoformat() repo_commit.committer_email = github_commit.committer_email repo_commit.committer_name = github_commit.committer_name repo_commit.username = github_commit.committer_login repo_commit.message = github_commit.message repo_commit.sha = github_commit.sha commit_started_callback(repo_commit) if github_commit.files: for file_info in github_commit.files: if file_info.get('patch'): filename = committer_username = None diff = DiffParser(file_info['patch']) repo_patch = RepoPatch(repo_commit=repo_commit) repo_patch.diff = diff repo_patch.filename = file_info.get("filename") patch_callback(repo_patch) commit_finished_callback(repo_commit) logger.debug("commit sha: %s processing complete", github_commit.sha)
def processCommits(self, commits, commit_started_callback, patch_callback, commit_finished_callback): if commits is None: return None # Process oldest first commits = commits[::-1] logger.debug("process %d commits", len(commits)) for github_commit in commits: logger.debug("sha: %s", github_commit.sha) if github_commit.sha: github_commit = self._github_repo.commit(github_commit.sha) repo_commit = RepoCommit(); repo_commit.url = github_commit.html_url repo_commit.repo_source = self if github_commit.date is not None: self._last_date = dateutil.parser.parse(github_commit.date) repo_commit.date = self._last_date self._last_date += datetime.timedelta(seconds=1) repo_commit.identifier = self._last_date.isoformat() repo_commit.committer_email = github_commit.committer_email repo_commit.committer_name = github_commit.committer_name repo_commit.username = github_commit.committer_login repo_commit.message = github_commit.message repo_commit.sha = github_commit.sha commit_started_callback(repo_commit) if github_commit.files: for file_info in github_commit.files: if file_info.get('patch'): filename = committer_username = None diff = DiffParser(file_info['patch']) repo_patch = RepoPatch(repo_commit=repo_commit) repo_patch.diff = diff repo_patch.filename = file_info.get("filename") patch_callback(repo_patch) commit_finished_callback(repo_commit) logger.debug("batch fof files processed") logger.debug("done")
def test_config_one(self): testDiff = '''@@ -25,11 +25,15 @@ ################################################################################ +Test exec abc -Removed This Line ''' diff = DiffParser(testDiff) test_commit = RepoCommit(url="http://www.example.com/commit/1") test_patch = RepoPatch(repo_commit=test_commit, diff=diff) filename = os.path.join(os.path.dirname(__file__), "config_one.json") rule_engine = RepoPatchRegexRuleEngine(json.load(open(filename)), test_mode=True) rule_engine.process(test_patch)
def process_patches(process_from_change_id, process_to_change_id=-1, patches_processed=0): if process_to_change_id == -1: process_to = "#head" else: process_to = "@" + str(process_to_change_id) changes = p4.run( 'changes', '-m', '%s' % (max_patches), '%s@%s,%s' % (directory, process_from_change_id, process_to)) changes = changes[::-1] if len(changes) == 0: return patches_processed earliest_change_id = changes[0].get("change") try: if earliest_change_id is None or long( earliest_change_id) < long(process_from_change_id): return patches_processed if changes[0]['change'] != process_from_change_id: patches_processed = process_patches( process_from_change_id, str(long(earliest_change_id) - 1), patches_processed) except: pass logger.info("processing " + str(len(changes)) + " commit from " + process_from_change_id + " to " + process_to) for change in changes: try: if patches_processed >= max_patches: return patches_processed patches_processed += 1 change_id = change.get("change") if change_id is None: logger.log("Change id was none: " + directory + search_for) continue change_time = change.get('time') change_datetime = datetime.datetime.utcfromtimestamp( long(change_time)) # option to debug a single commit debug_change_id = settings.get_change_id() if not settings.in_production() and debug_change_id: change_id = debug_change_id describes = p4.run('describe', change_id) if describes is None or len(describes) < 1: continue describe = describes[0] if isinstance(describe, basestring): continue user = describe.get("user") description = describe.get("desc") status = describe.get("status") old_change_id = describe.get("oldChange") files = describe.get("depotFile") types = describe.get("type") actions = describe.get("action") digests = describe.get("digest") revs = describe.get("rev") perforce_commit = RepoCommit() perforce_commit.sha = None perforce_commit.identifier = change_id perforce_commit.repo_source = self perforce_commit.url = "http://www.example.com/?change=" + change_id # get the real email of the user email_address = p4.run('users', user) if len(email_address) >= 1: perforce_commit.committer_email = email_address[ 0].get('Email') perforce_commit.date = change_datetime perforce_commit.committer_login = user perforce_commit.committer_name = user perforce_commit.committer_type = 'user' perforce_commit.files = files perforce_commit.message = description perforce_commit.username = user commit_started_callback(perforce_commit) if files is None: #Todo: we should still record something here, behavioral continue logger.info("processing commit " + change_id + " in repo " + self.directory) for i in range(len(files)): filename = files[i] # only check added and edited files if "edit" not in actions[ i] and "add" not in actions[i]: continue if old_change_id == None: # fabricate a changelist if a new file is added. (because there is no previous file version to diff against) file_string = self.retrieve_file(filename) diff2 = file_string.split('\n') diff2.insert(0, '') for i in range(len(diff2)): diff2[i] = '+ ' + diff2[i] else: diff2 = p4.run('diff2', '-u', "%s@%s" % (filename, old_change_id), "%s@%s" % (filename, change_id), tagged=False) # diff2 is an array now and well-formatted # change diff2 to a string before passing to diffparser if diff2 is not None and len(diff2) > 1: diffstring = '\n'.join(diff2[1:]) diff = DiffParser(diffstring) repo_patch = RepoPatch( repo_commit=perforce_commit) repo_patch.diff = diff repo_patch.filename = filename patch_callback(repo_patch) self._last_identifier = change_id commit_finished_callback(perforce_commit) except: print "Unexpected error:", sys.exc_info()[0] return patches_processed
def process_patches(process_from_change_id, process_to_change_id=-1, patches_processed=0): if process_to_change_id == -1: process_to = "#head" else: process_to = "@" + str(process_to_change_id) changes = p4.run( 'changes', '-m','%s' %(max_patches),'%s@%s,%s' % (directory, process_from_change_id, process_to)) changes = changes[::-1] if len(changes) == 0: return patches_processed; earliest_change_id = changes[0].get("change") try: if earliest_change_id is None or long(earliest_change_id) < long(process_from_change_id): return patches_processed; if changes[0]['change'] != process_from_change_id: patches_processed = process_patches(process_from_change_id, str(long(earliest_change_id)-1), patches_processed) except: pass logger.info("processing " + str(len(changes)) + " commit from " + process_from_change_id + " to " + process_to) for change in changes: try: if patches_processed >= max_patches: return patches_processed patches_processed += 1 change_id = change.get("change") if change_id is None: logger.log("Change id was none: " + directory + search_for) continue change_time = change.get('time') change_datetime = datetime.datetime.utcfromtimestamp(long(change_time)) # option to debug a single commit debug_change_id = settings.get_change_id() if not settings.in_production() and debug_change_id: change_id = debug_change_id describes = p4.run( 'describe', change_id) if describes is None or len(describes) < 1: continue describe = describes[0] if isinstance(describe, basestring): continue user = describe.get("user") description = describe.get("desc") status = describe.get("status") old_change_id = describe.get("oldChange") files = describe.get("depotFile") types = describe.get("type") actions = describe.get("action") digests = describe.get("digest") revs = describe.get("rev") perforce_commit = RepoCommit() perforce_commit.sha = None perforce_commit.identifier = change_id perforce_commit.repo_source = self perforce_commit.url = "http://www.example.com/?change=" + change_id # get the real email of the user email_address = p4.run('users', user) if len(email_address) >= 1: perforce_commit.committer_email = email_address[0].get('Email') perforce_commit.date = change_datetime perforce_commit.committer_login = user perforce_commit.committer_name = user perforce_commit.committer_type = 'user' perforce_commit.files = files perforce_commit.message = description perforce_commit.username = user commit_started_callback(perforce_commit) if files is None: #Todo: we should still record something here, behavioral continue logger.info("processing commit " + change_id + " in repo " + self.directory) for i in range(len(files)): filename = files[i] # only check added and edited files if "edit" not in actions[i] and "add" not in actions[i]: continue if old_change_id == None: # fabricate a changelist if a new file is added. (because there is no previous file version to diff against) file_string = self.retrieve_file(filename) diff2 = file_string.split('\n') diff2.insert(0, '') for i in range(len(diff2)): diff2[i] = '+ ' + diff2[i] else: diff2 = p4.run('diff2','-u', "%s@%s" % (filename, old_change_id), "%s@%s" % (filename, change_id), tagged=False); # diff2 is an array now and well-formatted # change diff2 to a string before passing to diffparser if diff2 is not None and len(diff2) > 1: diffstring = '\n'.join(diff2[1:]) diff = DiffParser(diffstring) repo_patch = RepoPatch(repo_commit=perforce_commit) repo_patch.diff = diff repo_patch.filename = filename patch_callback(repo_patch) self._last_identifier = change_id commit_finished_callback(perforce_commit) except: print "Unexpected error:", sys.exc_info()[0] return patches_processed