def processCommits(self, commits, commit_started_callback, patch_callback,
commit_finished_callback):
if commits is None:
return None
# Process oldest first
commits = commits[::-1]
logger.debug("processing %d commits", len(commits))
for github_commit in commits:
logger.debug("sha: %s", github_commit.sha)
if github_commit.sha:
github_commit = self._github_repo.commit(github_commit.sha)
repo_commit = RepoCommit()
repo_commit.url = github_commit.html_url
repo_commit.repo_source = self
if github_commit.date is not None:
self._last_date = dateutil.parser.parse(github_commit.date)
repo_commit.date = self._last_date
self._last_date += datetime.timedelta(seconds=1)
repo_commit.identifier = self._last_date.isoformat()
repo_commit.committer_email = github_commit.committer_email
repo_commit.committer_name = github_commit.committer_name
repo_commit.username = github_commit.committer_login
repo_commit.message = github_commit.message
repo_commit.sha = github_commit.sha
commit_started_callback(repo_commit)
if github_commit.files:
for file_info in github_commit.files:
if file_info.get('patch'):
filename = committer_username = None
diff = DiffParser(file_info['patch'])
repo_patch = RepoPatch(repo_commit=repo_commit)
repo_patch.diff = diff
repo_patch.filename = file_info.get("filename")
patch_callback(repo_patch)
commit_finished_callback(repo_commit)
logger.debug("commit sha: %s processing complete",
github_commit.sha)
def processCommits(self, commits, commit_started_callback, patch_callback, commit_finished_callback):
if commits is None:
return None
# Process oldest first
commits = commits[::-1]
logger.debug("process %d commits", len(commits))
for github_commit in commits:
logger.debug("sha: %s", github_commit.sha)
if github_commit.sha:
github_commit = self._github_repo.commit(github_commit.sha)
repo_commit = RepoCommit();
repo_commit.url = github_commit.html_url
repo_commit.repo_source = self
if github_commit.date is not None:
self._last_date = dateutil.parser.parse(github_commit.date)
repo_commit.date = self._last_date
self._last_date += datetime.timedelta(seconds=1)
repo_commit.identifier = self._last_date.isoformat()
repo_commit.committer_email = github_commit.committer_email
repo_commit.committer_name = github_commit.committer_name
repo_commit.username = github_commit.committer_login
repo_commit.message = github_commit.message
repo_commit.sha = github_commit.sha
commit_started_callback(repo_commit)
if github_commit.files:
for file_info in github_commit.files:
if file_info.get('patch'):
filename = committer_username = None
diff = DiffParser(file_info['patch'])
repo_patch = RepoPatch(repo_commit=repo_commit)
repo_patch.diff = diff
repo_patch.filename = file_info.get("filename")
patch_callback(repo_patch)
commit_finished_callback(repo_commit)
logger.debug("batch fof files processed")
logger.debug("done")
def test_config_one(self):
testDiff = '''@@ -25,11 +25,15 @@
################################################################################
+Test exec abc
-Removed This Line
'''
diff = DiffParser(testDiff)
test_commit = RepoCommit(url="http://www.example.com/commit/1")
test_patch = RepoPatch(repo_commit=test_commit, diff=diff)
filename = os.path.join(os.path.dirname(__file__), "config_one.json")
rule_engine = RepoPatchRegexRuleEngine(json.load(open(filename)),
test_mode=True)
rule_engine.process(test_patch)
def process_patches(process_from_change_id,
process_to_change_id=-1,
patches_processed=0):
if process_to_change_id == -1:
process_to = "#head"
else:
process_to = "@" + str(process_to_change_id)
changes = p4.run(
'changes', '-m', '%s' % (max_patches), '%s@%s,%s' %
(directory, process_from_change_id, process_to))
changes = changes[::-1]
if len(changes) == 0:
return patches_processed
earliest_change_id = changes[0].get("change")
try:
if earliest_change_id is None or long(
earliest_change_id) < long(process_from_change_id):
return patches_processed
if changes[0]['change'] != process_from_change_id:
patches_processed = process_patches(
process_from_change_id,
str(long(earliest_change_id) - 1),
patches_processed)
except:
pass
logger.info("processing " + str(len(changes)) +
" commit from " + process_from_change_id + " to " +
process_to)
for change in changes:
try:
if patches_processed >= max_patches:
return patches_processed
patches_processed += 1
change_id = change.get("change")
if change_id is None:
logger.log("Change id was none: " + directory +
search_for)
continue
change_time = change.get('time')
change_datetime = datetime.datetime.utcfromtimestamp(
long(change_time))
# option to debug a single commit
debug_change_id = settings.get_change_id()
if not settings.in_production() and debug_change_id:
change_id = debug_change_id
describes = p4.run('describe', change_id)
if describes is None or len(describes) < 1:
continue
describe = describes[0]
if isinstance(describe, basestring):
continue
user = describe.get("user")
description = describe.get("desc")
status = describe.get("status")
old_change_id = describe.get("oldChange")
files = describe.get("depotFile")
types = describe.get("type")
actions = describe.get("action")
digests = describe.get("digest")
revs = describe.get("rev")
perforce_commit = RepoCommit()
perforce_commit.sha = None
perforce_commit.identifier = change_id
perforce_commit.repo_source = self
perforce_commit.url = "http://www.example.com/?change=" + change_id
# get the real email of the user
email_address = p4.run('users', user)
if len(email_address) >= 1:
perforce_commit.committer_email = email_address[
0].get('Email')
perforce_commit.date = change_datetime
perforce_commit.committer_login = user
perforce_commit.committer_name = user
perforce_commit.committer_type = 'user'
perforce_commit.files = files
perforce_commit.message = description
perforce_commit.username = user
commit_started_callback(perforce_commit)
if files is None:
#Todo: we should still record something here, behavioral
continue
logger.info("processing commit " + change_id +
" in repo " + self.directory)
for i in range(len(files)):
filename = files[i]
# only check added and edited files
if "edit" not in actions[
i] and "add" not in actions[i]:
continue
if old_change_id == None:
# fabricate a changelist if a new file is added. (because there is no previous file version to diff against)
file_string = self.retrieve_file(filename)
diff2 = file_string.split('\n')
diff2.insert(0, '')
for i in range(len(diff2)):
diff2[i] = '+ ' + diff2[i]
else:
diff2 = p4.run('diff2',
'-u',
"%s@%s" %
(filename, old_change_id),
"%s@%s" % (filename, change_id),
tagged=False)
# diff2 is an array now and well-formatted
# change diff2 to a string before passing to diffparser
if diff2 is not None and len(diff2) > 1:
diffstring = '\n'.join(diff2[1:])
diff = DiffParser(diffstring)
repo_patch = RepoPatch(
repo_commit=perforce_commit)
repo_patch.diff = diff
repo_patch.filename = filename
patch_callback(repo_patch)
self._last_identifier = change_id
commit_finished_callback(perforce_commit)
except:
print "Unexpected error:", sys.exc_info()[0]
return patches_processed
def process_patches(process_from_change_id, process_to_change_id=-1, patches_processed=0):
if process_to_change_id == -1:
process_to = "#head"
else:
process_to = "@" + str(process_to_change_id)
changes = p4.run( 'changes', '-m','%s' %(max_patches),'%s@%s,%s' % (directory, process_from_change_id, process_to))
changes = changes[::-1]
if len(changes) == 0:
return patches_processed;
earliest_change_id = changes[0].get("change")
try:
if earliest_change_id is None or long(earliest_change_id) < long(process_from_change_id):
return patches_processed;
if changes[0]['change'] != process_from_change_id:
patches_processed = process_patches(process_from_change_id, str(long(earliest_change_id)-1), patches_processed)
except:
pass
logger.info("processing " + str(len(changes)) + " commit from " + process_from_change_id + " to " + process_to)
for change in changes:
try:
if patches_processed >= max_patches:
return patches_processed
patches_processed += 1
change_id = change.get("change")
if change_id is None:
logger.log("Change id was none: " + directory + search_for)
continue
change_time = change.get('time')
change_datetime = datetime.datetime.utcfromtimestamp(long(change_time))
# option to debug a single commit
debug_change_id = settings.get_change_id()
if not settings.in_production() and debug_change_id:
change_id = debug_change_id
describes = p4.run( 'describe', change_id)
if describes is None or len(describes) < 1:
continue
describe = describes[0]
if isinstance(describe, basestring):
continue
user = describe.get("user")
description = describe.get("desc")
status = describe.get("status")
old_change_id = describe.get("oldChange")
files = describe.get("depotFile")
types = describe.get("type")
actions = describe.get("action")
digests = describe.get("digest")
revs = describe.get("rev")
perforce_commit = RepoCommit()
perforce_commit.sha = None
perforce_commit.identifier = change_id
perforce_commit.repo_source = self
perforce_commit.url = "http://www.example.com/?change=" + change_id
# get the real email of the user
email_address = p4.run('users', user)
if len(email_address) >= 1:
perforce_commit.committer_email = email_address[0].get('Email')
perforce_commit.date = change_datetime
perforce_commit.committer_login = user
perforce_commit.committer_name = user
perforce_commit.committer_type = 'user'
perforce_commit.files = files
perforce_commit.message = description
perforce_commit.username = user
commit_started_callback(perforce_commit)
if files is None:
#Todo: we should still record something here, behavioral
continue
logger.info("processing commit " + change_id + " in repo " + self.directory)
for i in range(len(files)):
filename = files[i]
# only check added and edited files
if "edit" not in actions[i] and "add" not in actions[i]:
continue
if old_change_id == None:
# fabricate a changelist if a new file is added. (because there is no previous file version to diff against)
file_string = self.retrieve_file(filename)
diff2 = file_string.split('\n')
diff2.insert(0, '')
for i in range(len(diff2)):
diff2[i] = '+ ' + diff2[i]
else:
diff2 = p4.run('diff2','-u',
"%s@%s" % (filename, old_change_id),
"%s@%s" % (filename, change_id),
tagged=False);
# diff2 is an array now and well-formatted
# change diff2 to a string before passing to diffparser
if diff2 is not None and len(diff2) > 1:
diffstring = '\n'.join(diff2[1:])
diff = DiffParser(diffstring)
repo_patch = RepoPatch(repo_commit=perforce_commit)
repo_patch.diff = diff
repo_patch.filename = filename
patch_callback(repo_patch)
self._last_identifier = change_id
commit_finished_callback(perforce_commit)
except:
print "Unexpected error:", sys.exc_info()[0]
return patches_processed