self.cursor.execute("SELECT id FROM branches")
branches = [row[0] for row in self.cursor.fetchall()]
# Fill in months, with months considered
theTableMonths = TableMonths(db, cnn, repo_id)
for period in range(0, lastMonth):
month = (minDate.month + period) % 12 + 1
year = minDate.year + (period + minDate.month) // 12
date = str(year) + "-" + str(month) + "-01"
theTableMonths.add_pending_row((None, date))
theTableMonths.insert_rows(write_cursor)
# Fill in metrics_evo, with the metrics for monthly snapshots
theTableMetricsEvo = TableMetricsEvo(db, cnn, repo_id)
for branch in branches:
for period in range(0, lastMonth):
month = (minDate.month + period) % 12 + 1
year = minDate.year + (period + minDate.month) // 12
date = str(year) + "-" + str(month) + "-01"
(loc, sloc, files) = self._metrics_period(branch, date)
theTableMetricsEvo.add_pending_row(
(None, branch, date, loc, sloc, files))
theTableMetricsEvo.insert_rows(write_cursor)
cnn.commit()
write_cursor.close()
self.cursor.close()
cnn.close()
register_extension("MetricsEvo", MetricsEvo)
cursor.execute("SELECT MIN(date) FROM words_freq")
minDate = cursor.fetchone()[0]
cursor.execute("SELECT MAX(date) FROM words_freq")
maxDate = cursor.fetchone()[0]
lastMonth = (maxDate.year - minDate.year) * 12 + \
maxDate.month - minDate.month
for period in range(0, lastMonth):
wordsFreq = {}
month = (minDate.month + period) % 12 + 1
year = minDate.year + (period + minDate.month) // 12
date = str(year) + "-" + str(month) + "-01"
query = """SELECT word, times
FROM words_freq
WHERE date = '%s' ORDER BY times DESC"""
cursor.execute(query % date)
rows = cursor.fetchall()
print '*** ' + date + ":",
count = 0
for (text, times) in rows:
if not self._wordToExclude(text):
count += 1
print text + " (" + str(times) + ")",
if count > 10:
break
print
register_extension("MessageWordsPrint", MessageWordsPrint)
for file_id, file_name in rs:
if file_id in files:
continue
type = guess_file_type(file_name)
types.append(DBFileType(None, type, file_id))
if types:
file_types = [(type.id, type.file_id, type.type) \
for type in types]
write_cursor.executemany(
statement(DBFileType.__insert__, self.db.place_holder),
file_types)
rs = cursor.fetchmany()
cnn.commit()
write_cursor.close()
cursor.close()
cnn.close()
def backout(self, repo, uri, db):
update_statement = """delete from file_types where
file_id in (select id from files f
where f.repository_id = ?)"""
self._do_backout(repo, uri, db, update_statement)
register_extension("FileTypes", FileTypes)
where id = ?"""
if self.fixes_bug(commit_message):
is_bug_fix = 1
else:
is_bug_fix = 0
execute_statement(statement(update, db.place_holder),
(is_bug_fix, row_id),
write_cursor,
db,
"Couldn't update scmlog",
exception=ExtensionRunError)
read_cursor.close()
connection.commit()
connection.close()
# This turns off the profiler and deletes its timings
profiler_stop("Running BugFixMessage extension", delete=True)
def backout(self, repo, uri, db):
backout_statement = """update scmlog
set is_bug_fix = NULL
where repository_id = ?"""
self._do_backout(repo, uri, db, backout_statement)
register_extension("BugFixMessage", BugFixMessage)
year = minDate.year + (period + minDate.month) // 12
date = str(year) + "-" + str(month) + "-01"
query = "SELECT log.message " + \
"FROM scmlog log " + \
"WHERE year(log.date) = %s " + \
"AND month(log.date) = %s "
cursor.execute(query % (year, month))
rows = cursor.fetchall()
for message in rows:
words = [
word.strip(":()[],.#<>*'`~")
for word in message[0].lower().split()
]
for word in words:
if len(word) > 2:
if word in wordsFreq:
wordsFreq[word] += 1
else:
wordsFreq[word] = 1
for word in wordsFreq:
theTableWords.add_pending_row(
(None, date, word, wordsFreq[word]))
theTableWords.insert_rows(write_cursor)
cnn.commit()
write_cursor.close()
cursor.close()
cnn.close()
register_extension("MessageWords", MessageWords)
job_pool, write_cursor, db)
connection.commit()
i = i - processed_jobs
if processed_jobs < (queuesize / 5):
job_pool.join()
job_pool.join()
self.__process_finished_jobs(job_pool, write_cursor, db)
profiler_start("Inserting results in db")
#self.__insert_many(write_cursor)
connection.commit()
profiler_stop("Inserting results in db")
read_cursor.close()
write_cursor.close()
connection.close()
# This turns off the profiler and deletes it's timings
profiler_stop("Running content extension", delete=True)
def backout(self, repo, uri, db):
update_statement = """delete from content where
commit_id in (select id from scmlog s
where s.repository_id = ?)"""
self._do_backout(repo, uri, db, update_statement)
register_extension("Content", Content)
execute_statement(statement(insert, db.place_holder),
(file_id, commit_id, hunk.old_start_line,
hunk.old_end_line, hunk.new_start_line,
hunk.new_end_line),
write_cursor,
db,
"Couldn't insert hunk, dup record?",
exception=ExtensionRunError)
connection.commit()
progress.finished_one()
read_cursor.close()
connection.commit()
connection.close()
progress.done()
# This turns off the profiler and deletes its timings
profiler_stop("Running hunks extension", delete=True)
def backout(self, repo, uri, db):
update_statement = """delete from hunks
where commit_id in (select s.id from scmlog s
where s.repository_id = ?)"""
self._do_backout(repo, uri, db, update_statement)
register_extension("Hunks", Hunks)
if composed:
rev = revision.split ("|")[0]
else:
rev = revision
relative_path = fr.get_path ()
printdbg ("Path for %d at %s -> %s", (file_id, rev, relative_path))
if repo.get_type () == 'svn' and relative_path == 'tags':
printdbg ("Skipping file %s", (relative_path,))
continue
job = BlameJob (file_id, commit_id, relative_path, rev)
job_pool.push (job)
n_blames += 1
if n_blames >= self.MAX_BLAMES:
job_pool.join()
self.process_finished_jobs (job_pool, write_cursor)
n_blames = 0
job_pool.join ()
self.process_finished_jobs (job_pool, write_cursor, True)
read_cursor.close ()
write_cursor.close ()
cnn.close()
profiler_stop ("Running Blame extension", delete = True)
register_extension ("Blame", Blame)
set is_bug_fix = ?
where id = ?"""
if self.fixes_bug(commit_message):
is_bug_fix = 1
else:
is_bug_fix = 0
execute_statement(statement(update, db.place_holder),
(is_bug_fix, row_id),
write_cursor,
db,
"Couldn't update scmlog",
exception=ExtensionRunError)
read_cursor.close()
connection.commit()
connection.close()
# This turns off the profiler and deletes its timings
profiler_stop("Running BugFixMessage extension", delete=True)
def backout(self, repo, uri, db):
backout_statement = """update scmlog
set is_bug_fix = NULL
where repository_id = ?"""
self._do_backout(repo, uri, db, backout_statement)
register_extension("BugFixMessage", BugFixMessage)
execute_statement(statement(insert, db.place_holder),
(file_id, commit_id,
hunk.old_start_line,
hunk.old_end_line,
hunk.new_start_line,
hunk.new_end_line),
write_cursor,
db,
"Couldn't insert hunk, dup record?",
exception=ExtensionRunError)
connection.commit()
progress.finished_one()
read_cursor.close()
connection.commit()
connection.close()
progress.done()
# This turns off the profiler and deletes its timings
profiler_stop("Running hunks extension", delete=True)
def backout(self, repo, uri, db):
update_statement = """delete from hunks
where commit_id in (select s.id from scmlog s
where s.repository_id = ?)"""
self._do_backout(repo, uri, db, update_statement)
register_extension("Hunks", Hunks)
icursor = ICursor(cursor, self.INTERVAL_SIZE)
icursor.execute(statement("SELECT id, rev, composed_rev " + \
"from scmlog where repository_id = ?",
db.place_holder), (repo_id,))
rs = icursor.fetchmany()
while rs:
for commit_id, revision, composed_rev in rs:
# Get the patch
pj = PatchJob(revision, commit_id)
path = uri_to_filename(repo_uri)
pj.run(repo, path or repo.get_uri())
p = DBPatch(db, commit_id, pj.data)
# Yield the patch to hunks
for file_id, patch in p.file_patches():
yield (pj.commit_id, file_id, patch, pj.rev)
rs = icursor.fetchmany()
profiler_start("Running PatchesAndHunks extension")
hunks = Hunks()
hunks.get_patches = patch_generator
hunks.run(repo, uri, db)
register_extension("PatchesAndHunks", PatchesAndHunks)
cadded = cremoved = 0
if not theTableComLines.in_table(id):
(cadded, cremoved) = counter.get_lines_for_commit(commit)
theTableComLines.add_pending_row(
(None, id, cadded, cremoved))
tadded = tremoved = 0
for path in counter.get_paths_for_commit(commit):
if not theTableComFilLines.in_table(str(id) + ',' + path):
(added, removed) = \
counter.get_lines_for_commit_file(commit, path)
theTableComFilLines.add_pending_row(
(None, id, path, added, removed))
tadded += int(added)
tremoved += int(removed)
# Sanity check
if (cadded != tadded) or (cremoved != tremoved):
printerr("Sanity check failed: %d, %s, %d, %d, %d, %d" %
(id, commit, cadded, tadded, cremoved, tremoved))
printerr(counter.get_paths_for_commit(commit))
theTableComLines.insert_rows(write_cursor)
theTableComFilLines.insert_rows(write_cursor)
if not rows:
rows_left = False
cnn.commit()
write_cursor.close()
cursor.close()
cnn.close()
register_extension("CommitsLOCDet", CommitsLOCDet)
# The ISO year consists of 52 or 53 full weeks
weeks_year_real = 52.1775
weeks_year = int(weeks_year_real) + 1
minDateWeek = minDate.date().isocalendar()[1]
maxDateWeek = maxDate.date().isocalendar()[1]
firstWeek = minDate.year * int(weeks_year) + minDateWeek
lastWeek = maxDate.year * int(weeks_year) + maxDateWeek
for period in range(firstWeek, lastWeek + 1):
week = (period - 1) % weeks_year + 1
year = (period - 1) // weeks_year
# When used with the strptime() method, %U and %W are only used in
# calculations when the day of the week and the year are specified.
date_time = datetime.strptime(
str(year) + " " + str(week) + " 0", "%Y %U %w")
if (date_time.year > year): continue
date = date_time.strftime("%y-%m-%d")
theWeeksTable.add_pending_row((period, year, week, date))
theWeeksTable.insert_rows(write_cursor)
cnn.commit()
write_cursor.close()
cursor.close()
cnn.close()
# Register in the CVSAnalY extension system
register_extension("Weeks", Weeks)
wordsFreq = {}
month = (minDate.month + period) % 12 + 1
year = minDate.year + (period + minDate.month) // 12
date = str(year) + "-" + str(month) + "-01"
query = "SELECT log.message " + \
"FROM scmlog log " + \
"WHERE year(log.date) = %s " + \
"AND month(log.date) = %s "
cursor.execute (query % (year, month))
rows = cursor.fetchall()
for message in rows:
words = [word.strip(":()[],.#<>*'`~")
for word in message[0].lower().split ()]
for word in words:
if len(word) > 2:
if word in wordsFreq:
wordsFreq[word] += 1
else:
wordsFreq[word] = 1
for word in wordsFreq:
theTableWords.add_pending_row ((None, date,
word, wordsFreq[word]))
theTableWords.insert_rows (write_cursor)
cnn.commit ()
write_cursor.close ()
cursor.close ()
cnn.close ()
register_extension ("MessageWords", MessageWords)
cursor.execute(statement(query, self.db.place_holder), (h.rev, ))
fetched_row = cursor.fetchone()
if fetched_row is not None:
args.append((job.file_id, job.commit_id, h.start, h.end,
fetched_row[0]))
cursor.close()
cnn.close()
return args
def get_max_id(self, db):
return None
def get_blames(self, cursor, repoid):
query = "select distinct b.file_id, b.commit_id from line_blames b, files f " + \
"where b.file_id = f.id and repository_id = ?"
cursor.execute(statement(query, self.db.place_holder), (repoid, ))
return [(res[0], res[1]) for res in cursor.fetchall()]
def backout(self, repo, uri, db):
update_statement = """delete from lineblames where
commit_id in (select s.id from scmlog s
where s.repository_id = ?)"""
self._do_backout(repo, uri, db, update_statement)
register_extension("LineBlame", LineBlame)
theWeeksTable = WeeksTable(db, cnn, repo)
# The ISO year consists of 52 or 53 full weeks
weeks_year_real = 52.1775
weeks_year = int(weeks_year_real)+1
minDateWeek = minDate.date().isocalendar()[1]
maxDateWeek = maxDate.date().isocalendar()[1]
firstWeek = minDate.year * int(weeks_year) + minDateWeek
lastWeek = maxDate.year * int(weeks_year) + maxDateWeek
for period in range (firstWeek, lastWeek+1):
week = (period -1 ) % weeks_year + 1
year = (period - 1)// weeks_year
# When used with the strptime() method, %U and %W are only used in
# calculations when the day of the week and the year are specified.
date_time = datetime.strptime(str(year) + " " + str(week)+" 0", "%Y %U %w")
if (date_time.year > year): continue
date = date_time.strftime("%y-%m-%d")
theWeeksTable.add_pending_row ((period, year, week, date))
theWeeksTable.insert_rows (write_cursor)
cnn.commit ()
write_cursor.close ()
cursor.close ()
cnn.close ()
# Register in the CVSAnalY extension system
register_extension ("Weeks", Weeks)
execute_statement(statement(__insert__,
self.db.place_holder),
(patch_id, commit_id_new, file_id, old_class, new_class, old_function, new_function, 0),
write_cursor,
db,
"\nCouldn't insert, duplicate patch?",
exception=ExtensionRunError)
#clear
old_cla.clear()
new_cla.clear()
old_func.clear()
new_func.clear()
cnn.commit()
write_cursor.close()
cursor.close()
cnn.close()
profiler_stop("Running Patches extension", delete=True)
end = time.time()
print function_name_change_count, 'file change name!'
print 'num of source file:', num_of_source
print 'num of exception:', num_of_exception
print 'num of non_source_file:', non_source_file
print 'num of files can not be recovered:', num_of_unrecovered
print 'num_of_id1:', num_of_id1
print 'consuming time: %ss' % str(end - start)
register_extension("Analyse_patch", Analyse_patch)
cursor.execute(statement(query, self.db.place_holder),
(h.rev,))
fetched_row = cursor.fetchone()
if fetched_row is not None:
args.append((job.file_id, job.commit_id, h.start, h.end, fetched_row[0]))
cursor.close()
cnn.close()
return args
def get_max_id(self, db):
return None
def get_blames(self, cursor, repoid):
query = "select distinct b.file_id, b.commit_id from line_blames b, files f " + \
"where b.file_id = f.id and repository_id = ?"
cursor.execute(statement(query, self.db.place_holder), (repoid,))
return [(res[0], res[1]) for res in cursor.fetchall()]
def backout(self, repo, uri, db):
update_statement = """delete from lineblames where
commit_id in (select s.id from scmlog s
where s.repository_id = ?)"""
self._do_backout(repo, uri, db, update_statement)
register_extension("LineBlame", LineBlame)
cursor.execute("SELECT MIN(date) FROM words_freq")
minDate = cursor.fetchone()[0]
cursor.execute("SELECT MAX(date) FROM words_freq")
maxDate = cursor.fetchone()[0]
lastMonth = (maxDate.year - minDate.year) * 12 + maxDate.month - minDate.month
for period in range(0, lastMonth):
wordsFreq = {}
month = (minDate.month + period) % 12 + 1
year = minDate.year + (period + minDate.month) // 12
date = str(year) + "-" + str(month) + "-01"
query = """SELECT word, times
FROM words_freq
WHERE date = '%s' ORDER BY times DESC"""
cursor.execute(query % date)
rows = cursor.fetchall()
print '*** ' + date + ":",
count = 0
for (text, times) in rows:
if not self._wordToExclude(text):
count += 1
print text + " (" + str(times) + ")",
if count > 10:
break
print
register_extension("MessageWordsPrint", MessageWordsPrint)
if i >= queuesize:
printdbg("FileCount queue is now at %d, flushing to database",
(i,))
processed_jobs = self.__process_finished_jobs(job_pool,
write_cursor, db)
connection.commit()
i = i - processed_jobs
if processed_jobs < (queuesize / 5):
job_pool.join()
job_pool.join()
self.__process_finished_jobs(job_pool, write_cursor, db)
read_cursor.close()
connection.commit()
connection.close()
# This turns off the profiler and deletes its timings
profiler_stop("Running FileCount extension", delete=True)
def backout(self, repo, uri, db):
update_statement = """update scmlog
set file_count = NULL
where repository_id = ?"""
self._do_backout(repo, uri, db, update_statement)
register_extension("FileCount", FileCount)
# Cursor for reading from the database
cursor = cnn.cursor()
# Cursor for writing to the database
write_cursor = cnn.cursor()
cursor.execute("SELECT MIN(date) FROM scmlog")
minDate = cursor.fetchone()[0]
cursor.execute("SELECT MAX(date) FROM scmlog")
maxDate = cursor.fetchone()[0]
cursor.execute("DROP TABLE IF EXISTS months")
theMonthsTable = MonthsTable(db, cnn, repo)
firstMonth = minDate.year * 12 + minDate.month
lastMonth = maxDate.year * 12 + maxDate.month
for period in range(firstMonth, lastMonth + 1):
month = (period - 1) % 12 + 1
year = (period - 1) // 12
date = str(year) + "-" + str(month) + "-01"
theMonthsTable.add_pending_row((period, year, month, date))
theMonthsTable.insert_rows(write_cursor)
cnn.commit()
write_cursor.close()
cursor.close()
cnn.close()
# Register in the CVSAnalY extension system
register_extension("Months", Months)
icursor = ICursor(cursor, self.INTERVAL_SIZE)
icursor.execute(
statement(
"SELECT id, rev, composed_rev from scmlog where repository_id = ?",
db.place_holder), (repo_id, ))
rs = icursor.fetchmany()
while rs:
for commit_id, revision, composed_rev in rs:
if commit_id in commits:
continue
if composed_rev:
rev = revision.split("|")[0]
else:
rev = revision
p = DBPatch(None, commit_id, self.get_patch_for_commit(rev))
write_cursor.execute(
statement(DBPatch.__insert__, self.db.place_holder),
(p.id, p.commit_id, self.db.to_binary(p.patch)))
rs = icursor.fetchmany()
cnn.commit()
write_cursor.close()
cursor.close()
cnn.close()
register_extension("Patches", Patches)
job_pool.push (job)
n_blames += 1
if n_blames >= self.MAX_BLAMES:
processed_jobs = self.process_finished_jobs (job_pool, write_cursor)
n_blames -= processed_jobs
if processed_jobs<=self.MAX_BLAMES/5:
profiler_start("Joining unprocessed jobs")
job_pool.join()
profiler_stop("Joining unprocessed jobs", delete=True)
except NotValidHunkWarning as e:
printerr("Not a valid hunk: "+str(e))
finally:
file_rev = read_cursor.fetchone()
job_pool.join ()
self.process_finished_jobs (job_pool, write_cursor, True)
try:
self.__drop_cache(cnn)
except:
printdbg("Couldn't drop cache because of " + str(e))
read_cursor.close ()
write_cursor.close ()
cnn.close()
profiler_stop ("Running HunkBlame extension", delete = True)
register_extension ("HunkBlame", HunkBlame)
except Exception, e:
raise ExtensionRunError (str (e))
write_cursor = cnn.cursor ()
icursor = ICursor (cursor, self.INTERVAL_SIZE)
icursor.execute (statement ("SELECT id, rev, composed_rev from scmlog where repository_id = ?",
db.place_holder), (repo_id,))
rs = icursor.fetchmany ()
while rs:
for commit_id, revision, composed_rev in rs:
if commit_id in commits:
continue
if composed_rev:
rev = revision.split ("|")[0]
else:
rev = revision
p = DBPatch (None, commit_id, self.get_patch_for_commit (rev))
write_cursor.execute (statement (DBPatch.__insert__, self.db.place_holder),
(p.id, p.commit_id, self.db.to_binary (p.patch)))
rs = icursor.fetchmany ()
cnn.commit ()
write_cursor.close ()
cursor.close ()
cnn.close ()
register_extension ("Patches", Patches)
write_cursor = cnn.cursor()
rs = cursor.fetchmany()
while rs:
commit_list = []
for commit_id, revision, composed_rev in rs:
if commit_id in commits:
continue
if composed_rev:
rev = revision.split("|")[0]
else:
rev = revision
(added, removed) = counter.get_lines_for_revision(revision)
commit_list.append(DBCommitLines(None, commit_id, added, removed))
if commit_list:
commits_lines = [(commit.id, commit.commit_id, commit.added, commit.removed) for commit in commit_list]
write_cursor.executemany(statement(DBCommitLines.__insert__, self.db.place_holder), commits_lines)
rs = cursor.fetchmany()
cnn.commit()
write_cursor.close()
cursor.close()
cnn.close()
register_extension("CommitsLOC", CommitsLOC)
for commit_id, revision, composed_rev in rs:
if commit_id in commits:
continue
if composed_rev:
rev = revision.split("|")[0]
else:
rev = revision
(added, removed) = counter.get_lines_for_revision(revision)
commit_list.append(
DBCommitLines(None, commit_id, added, removed))
if commit_list:
commits_lines = [(commit.id, commit.commit_id, commit.added, \
commit.removed) for commit in commit_list]
write_cursor.executemany(
statement(DBCommitLines.__insert__, self.db.place_holder),
commits_lines)
rs = cursor.fetchmany()
cnn.commit()
write_cursor.close()
cursor.close()
cnn.close()
register_extension("CommitsLOC", CommitsLOC)
job = ContentJob(commit_id, file_id, rev, relative_path)
job_pool.push(job)
i = i + 1
if i >= queuesize:
printdbg("Content queue is now at %d, flushing to database", (i,))
processed_jobs = self.__process_finished_jobs(job_pool, connection, db)
i = i - processed_jobs
if processed_jobs < (queuesize / 5):
job_pool.join()
job_pool.join()
self.__process_finished_jobs(job_pool, connection, db)
read_cursor.close()
connection.close()
# This turns off the profiler and deletes it's timings
profiler_stop("Running content extension", delete=True)
def backout(self, repo, uri, db):
update_statement = """delete from content where
commit_id in (select id from scmlog s
where s.repository_id = ?)"""
self._do_backout(repo, uri, db, update_statement)
register_extension("Content", Content)
def patch_generator(repo, repo_uri, repo_id, db, cursor):
icursor = ICursor(cursor, self.INTERVAL_SIZE)
icursor.execute(
statement("SELECT id, rev, composed_rev " + "from scmlog where repository_id = ?", db.place_holder),
(repo_id,),
)
rs = icursor.fetchmany()
while rs:
for commit_id, revision, composed_rev in rs:
# Get the patch
pj = PatchJob(revision, commit_id)
path = uri_to_filename(repo_uri)
pj.run(repo, path or repo.get_uri())
# Yield the patch to hunks
yield (pj.commit_id, pj.data, pj.rev)
rs = icursor.fetchmany()
profiler_start("Running PatchesAndHunks extension")
hunks = Hunks()
hunks.get_patches = patch_generator
hunks.run(repo, uri, db)
register_extension("PatchesAndHunks", PatchesAndHunks)
+ "not exists (select id from file_links where parent_id = a.file_id) "
+ "and f.repository_id = ? group by fid, fname"
)
cursor.execute(statement(query, db.place_holder), (repo_id,))
write_cursor = cnn.cursor()
rs = cursor.fetchmany()
while rs:
types = []
for file_id, file_name in rs:
if file_id in files:
continue
type = guess_file_type(file_name)
types.append(DBFileType(None, type, file_id))
if types:
file_types = [(type.id, type.file_id, type.type) for type in types]
write_cursor.executemany(statement(DBFileType.__insert__, self.db.place_holder), file_types)
rs = cursor.fetchmany()
cnn.commit()
write_cursor.close()
cursor.close()
cnn.close()
register_extension("FileTypes", FileTypes)
p.patch is not NULL""", db.place_holder), (repo_id, ))
nr_records = cursor.fetchone()[0]
progress = Progress("[Extension PatchesLOC]", nr_records)
patches = self.get_patches(repo, path or repo.get_uri(), repo_id, db,
cursor)
for commit_id, file_id, patch_content, rev in patches:
(added, removed) = self.count_lines(patch_content)
insert = """insert into patch_lines(file_id, commit_id,
added, removed)
values(?,?,?,?)"""
execute_statement(statement(insert, db.place_holder),
(file_id, commit_id, added, removed),
cursor,
db,
"Couldn't insert patch, dup record?",
exception=ExtensionRunError)
connection.commit()
progress.finished_one()
cursor.close()
connection.commit()
connection.close()
progress.done()
profiler_stop("Running PatchLOC extension", delete=True)
register_extension("PatchLOC", PatchLOC)
# Cursor for reading from the database
cursor = cnn.cursor()
# Cursor for writing to the database
write_cursor = cnn.cursor()
cursor.execute("SELECT MIN(date) FROM scmlog")
minDate = cursor.fetchone()[0]
cursor.execute("SELECT MAX(date) FROM scmlog")
maxDate = cursor.fetchone()[0]
cursor.execute("DROP TABLE IF EXISTS months")
theMonthsTable = MonthsTable(db, cnn, repo)
firstMonth = minDate.year * 12 + minDate.month
lastMonth = maxDate.year * 12 + maxDate.month
for period in range(firstMonth, lastMonth + 1):
month = (period - 1) % 12 + 1
year = (period - 1) // 12
date = str(year) + "-" + str(month) + "-01"
theMonthsTable.add_pending_row((period, year, month, date))
theMonthsTable.insert_rows(write_cursor)
cnn.commit()
write_cursor.close()
cursor.close()
cnn.close()
# Register in the CVSAnalY extension system
register_extension("Months", Months)
cadded = cremoved = 0
if not theTableComLines.in_table(id):
(cadded, cremoved) = counter.get_lines_for_commit(commit)
theTableComLines.add_pending_row((None, id,
cadded, cremoved))
tadded = tremoved = 0
for path in counter.get_paths_for_commit(commit):
if not theTableComFilLines.in_table(str(id) + ',' + path):
(added, removed) = \
counter.get_lines_for_commit_file(commit, path)
theTableComFilLines.add_pending_row((None, id, path,
added, removed))
tadded += int(added)
tremoved += int(removed)
# Sanity check
if (cadded != tadded) or (cremoved != tremoved):
printerr("Sanity check failed: %d, %s, %d, %d, %d, %d" %
(id, commit, cadded, tadded, cremoved, tremoved))
printerr(counter.get_paths_for_commit(commit))
theTableComLines.insert_rows(write_cursor)
theTableComFilLines.insert_rows(write_cursor)
if not rows:
rows_left = False
cnn.commit()
write_cursor.close()
cursor.close()
cnn.close()
register_extension("CommitsLOCDet", CommitsLOCDet)
if person_id:
write_cursor.execute(
statement(
"UPDATE scmlog SET wordpress_author_id = ? WHERE id = ?",
db.place_holder), (person_id, scmlog_id))
rs = cursor.fetchmany()
cnn.commit()
write_cursor.close()
cursor.close()
cnn.close()
register_extension("WordPress", WordPress)
class TestWordPress(unittest.TestCase):
def setUp(self):
self.wordpress = WordPress()
def test_get_author_from_message(self):
data_provider = (
('asdf', None), ('props user', 'user'),
('Textile checkin. Props to Jaykul for prodding me into this.\n\n',
'jaykul'),
('Remove unnecessary call to generic_ping(). The publish_post action will take care of it. Props to hades. http://wordpress.org/support/4/6876\n\n',
'hades'),
('Docs: Clarify parameter and return descriptions in the DocBlock for `wp_set_all_user_settings()`.\n\nSee [32613]. See #30989.\n\nBuilt from https://develop.svn.wordpress.org/trunk@37263\n\n',
None),
printdbg("FileCount queue is now at %d, flushing to database",
(i, ))
processed_jobs = self.__process_finished_jobs(
job_pool, write_cursor, db)
connection.commit()
i = i - processed_jobs
if processed_jobs < (queuesize / 5):
job_pool.join()
job_pool.join()
self.__process_finished_jobs(job_pool, write_cursor, db)
read_cursor.close()
connection.commit()
connection.close()
# This turns off the profiler and deletes its timings
profiler_stop("Running FileCount extension", delete=True)
def backout(self, repo, uri, db):
update_statement = """update scmlog
set file_count = NULL
where repository_id = ?"""
self._do_backout(repo, uri, db, update_statement)
register_extension("FileCount", FileCount)
lastMonth = (maxDate.year - minDate.year) * 12 + maxDate.month - minDate.month
self.cursor.execute("SELECT id FROM branches")
branches = [row[0] for row in self.cursor.fetchall()]
# Fill in months, with months considered
theTableMonths = TableMonths(db, cnn, repo_id)
for period in range(0, lastMonth):
month = (minDate.month + period) % 12 + 1
year = minDate.year + (period + minDate.month) // 12
date = str(year) + "-" + str(month) + "-01"
theTableMonths.add_pending_row((None, date))
theTableMonths.insert_rows(write_cursor)
# Fill in metrics_evo, with the metrics for monthly snapshots
theTableMetricsEvo = TableMetricsEvo(db, cnn, repo_id)
for branch in branches:
for period in range(0, lastMonth):
month = (minDate.month + period) % 12 + 1
year = minDate.year + (period + minDate.month) // 12
date = str(year) + "-" + str(month) + "-01"
(loc, sloc, files) = self._metrics_period(branch, date)
theTableMetricsEvo.add_pending_row((None, branch, date, loc, sloc, files))
theTableMetricsEvo.insert_rows(write_cursor)
cnn.commit()
write_cursor.close()
self.cursor.close()
cnn.close()
register_extension("MetricsEvo", MetricsEvo)
if n_blames >= self.MAX_BLAMES:
processed_jobs = self.process_finished_jobs(
job_pool, write_cursor)
n_blames -= processed_jobs
if processed_jobs <= self.MAX_BLAMES / 5:
profiler_start("Joining unprocessed jobs")
job_pool.join()
profiler_stop("Joining unprocessed jobs", delete=True)
except NotValidHunkWarning as e:
printerr("Not a valid hunk: " + str(e))
finally:
file_rev = read_cursor.fetchone()
job_pool.join()
self.process_finished_jobs(job_pool, write_cursor, True)
try:
self.__drop_cache(cnn)
except:
printdbg("Couldn't drop cache because of " + str(e))
read_cursor.close()
write_cursor.close()
cnn.close()
profiler_stop("Running HunkBlame extension", delete=True)
register_extension("HunkBlame", HunkBlame)
p.patch is not NULL""",
db.place_holder), (repo_id,))
nr_records = cursor.fetchone()[0]
progress = Progress("[Extension PatchesLOC]", nr_records)
patches = self.get_patches(repo, path or repo.get_uri(), repo_id, db,
cursor)
for commit_id, file_id, patch_content, rev in patches:
(added, removed) = self.count_lines(patch_content)
insert = """insert into patch_lines(file_id, commit_id,
added, removed)
values(?,?,?,?)"""
execute_statement(statement(insert, db.place_holder),
(file_id, commit_id, added, removed),
cursor,
db,
"Couldn't insert patch, dup record?",
exception=ExtensionRunError)
connection.commit()
progress.finished_one()
cursor.close()
connection.commit()
connection.close()
progress.done()
profiler_stop("Running PatchLOC extension", delete=True)
register_extension("PatchLOC", PatchLOC)
