def go_go(self):
try:
# Make sure the tmp directory exists.
cmd_and_args = ['/bin/mkdir', '-p', self.cli_opts.tmppath,]
the_resp = misc.process_check_output(cmd_and_args)
self.assert_(not the_resp)
# Init. the stats
for svn_path in self.cli_opts.svnhier:
self.stats[svn_path] = 0
# Get the SVN log of the leafiest branch. Look for revisions from any
# of the branches in the hierarchy (starting at the leaf).
# MAYBE: In SVN, each revision no. corresponds to a checkin of files,
# but is there anyway to find out the base directory path of
# each revision, given just a revision no., without having to
# do trial and error?
if not debug_skip_trunk:
self.process_trunk()
# Our SVN history only extends back to March, 2010, when the code was
# made public -- and I can't find the original SVN trunk. So get
# releases from the releases folder to fill in the long ago history.
if not debug_skip_relss:
self.process_releases()
# Print out the results.
self.print_results()
# Generate the images.
self.generate_pngs()
# Copy the images to the htdocs exports/ directory. [lb] supposes
# clients won't care about these... but it's as good a place as any.
# Though maybe we can make some subdirectories.
self.install_exports()
except subprocess.CalledProcessError, e:
log.warning('CalledProcessError: %s' % (str(e),))
def release_get_rdat(self, relname, svn_path):
#
cmd_and_args = ['svn', 'info', svn_path,]
rel_info = misc.process_check_output(cmd_and_args)
#info_lines = rel_list.split('\n')
# Why do I default to using re.match, which matches at the start of the
# string. Use re.search if you want MULTILINE to work.
m = Log_Jammin.re_sinfo_author.search(rel_info)
committed_by = m.group(1)
#
m = Log_Jammin.re_sinfo_rev.search(rel_info)
rev_id = int(m.group(1))
#
m = Log_Jammin.re_sinfo_date.search(rel_info)
#rev_year = int(m.group(1))
#rev_month = int(m.group(2))
#rev_day = int(m.group(3))
#rev_date = datetime.date(rev_year, rev_month, rev_day)
#commit_date = rev_date
commit_date = m.group(1)
#
comment_lines = ''
#
rdat = (relname, rev_id, committed_by, commit_date, comment_lines,)
#
return rdat
def png_file_write_for(self, dat_path):
log.info('Creating gnuplot image for %s.' % (dat_path,))
#
cdiff_path = os.path.join(script_root, dat_path)
log.debug('Creating png file: %s' % (cdiff_path,))
# pyserver_glue chdired us, but our gnuplot uses paths relative to our
# dir, so chdir back.
was_dir = os.path.abspath(os.curdir)
os.chdir(script_root)
# process_check_output: resp:
# "/export/scratch/ccp/dev/cp_2628/scripts/dev/croc_diff.gnuplot",
# line 36: warning: Skipping unreadable file "croc_diff.dat"
#
# plot 'croc_diff.dat' using 2:xtic(1), for [i=3:4] '' using i
# ^
# "/export/scratch/ccp/dev/cp_2628/scripts/dev/croc_diff.gnuplot",
# line 36: ':' expected
cmd_and_args = ['gnuplot', cdiff_path,]
# process_check_output: resp: Could not find/open font when opening
# font arial, trying default
# "/export/scratch/ccp/dev/cp_2628/scripts/dev/croc_solo.gnuplot",
# line 69: warning: Skipping unreadable file "croc_solo.dat"
# process_check_output: resp: Cannot open load file 'croc_diff.gnuplot'
#cmd_and_args = ['gnuplot', dat_path,]
the_resp = misc.process_check_output(cmd_and_args)
log.verbose('the_resp: %s' % (the_resp,))
os.chdir(was_dir)
def process_releases(self):
cmd_and_args = ['svn', 'list', self.cli_opts.svnrels,]
rel_list = misc.process_check_output(cmd_and_args)
list_lines = rel_list.split('\n')
# Each line is a number followed by a slash, e.g., '51.1/'.
self.releases = [Decimal(x.rstrip('/')) for x in list_lines if x]
self.releases.sort()
log.info('Found %d releases under "%s".'
% (len(self.releases), os.path.basename(self.cli_opts.svnrels),))
self.checkout_and_analyze_releases()
def cleanup_co(self, co_path):
# Cleanup
self.assert_(co_path and (co_path != '/'))
self.assert_(len(co_path.split(os.path.sep)) > 2)
self.assert_(co_path.startswith(self.cli_opts.tmppath))
self.assert_(os.path.isdir(co_path))
if not debug_skip_checkout_remove:
cmd_and_args = ['/bin/rm', '-rf', co_path,]
log.debug('Removing checkout folder: %s' % (cmd_and_args,))
the_resp = misc.process_check_output(cmd_and_args)
def something_Croc_two(self, lhs_rdat, lhs_path, rhs_rdat, rhs_path):
# CLOC's yaml is, e.g.,
#
# same :
# - language : Bourne Again Shell
# files_count : 1
# blank : 0
# comment : 46
# code : 153
# same_total :
# sum_files : 963
# blank : 0
# comment : 41987
# code : 121671
# modified :
# - language : ...
# modified_total :
# sum_files : ...
# added :
# - language : ...
# added_total :
# sum_files : ...
# removed :
# - language : ...
# removed_total :
# sum_files : ...
cmd_and_args = [cloc_path,
'--diff',
'--yaml',
'--quiet',
'--progress-rate=0',
# FIXME: Why not use cloc_exclude_dirs?
'--exclude-dir=cloc-1.55',
]
#
exclude_switch = self.cloc_get_exclude_switch()
if exclude_switch:
cmd_and_args.append(exclude_switch)
#
cmd_and_args += [lhs_path, rhs_path,]
log.info('CLOCing diff: %d:%d' % (lhs_rdat[Log_Jammin.p_revid],
rhs_rdat[Log_Jammin.p_revid],))
log.debug('DIFFCLOCing: %s' % (cmd_and_args,))
cloc_yaml = misc.process_check_output(cmd_and_args)
log.verbose('CLOC cloc_yaml: %s' % (cloc_yaml,))
caml = yaml.load(cloc_yaml)
if not lhs_rdat[Log_Jammin.p_rname]:
self.croc_diff[lhs_rdat[Log_Jammin.p_revid]] = (lhs_rdat, caml,)
else:
self.rels_diff[lhs_rdat[Log_Jammin.p_revid]] = (lhs_rdat, caml,)
def process_trunk(self):
leafy_log_branch = self.cli_opts.svnhier[0]
log_cache = os.path.join(self.cli_opts.tmppath, 'log_jammin_svn.log')
if self.cli_opts.svnlogf:
# DEVS: This is for testing, if you want to use a log file you've
# generated.
log.info('Reading existing user-generated log: %s'
% (self.cli_opts.svnlogf,))
log_f = open(self.cli_opts.svnlogf, 'r')
the_log = log_f.read()
log_f.close()
elif not debug_skip_fetchlog:
# NORMALS: Normally, this script makes a fresh log file, which takes a
# few seconds.
log.info('Fetching fresh log for: %s' % (leafy_log_branch,))
cmd_and_args = ['svn', 'log', leafy_log_branch,]
## no? This hangs if the svn command resets the SSH connection and
## doesn't actually run. I.e., if you see
## Identity added: /home/misc00/landonb/.ssh/id_rsa (/home/misc00/..)
the_log = misc.process_check_output(cmd_and_args)
# DEVS: This script doesn't really keep to cache the log except unless
# you're debugging this script, but we cache it always anyway.
log_f = open(log_cache, 'w')
log_f.write(the_log)
log_f.close()
else:
# DEVS: This is for testing, if you want to use this script's log file
# that it generated the last time it ran.
log.info('Reading existing script-generated log: %s' % (log_cache,))
log_f = open(log_cache, 'r')
the_log = log_f.read()
log_f.close()
# Parse the log.
self.consume_log(the_log)
log.info('Found %d revisions for branch "%s".'
% (len(self.revisions), os.path.basename(leafy_log_branch),))
# Check out pairs of revisions and analyze the code.
self.checkout_and_analyze_revisions()
def try_checkout(self, co_rev, co_path, revision_id=None):
try:
cmd_and_args = ['svn', 'co', co_rev, co_path,]
the_resp = misc.process_check_output(cmd_and_args)
# The response is, e.g.,
# A /tmp/log_jammin/r26657/pyserver
# ...
# Checked out revision 26657.
log.verbose('the_resp: %s' % (the_resp,))
log.debug('the_resp: %s' % (the_resp[-28:],))
success = not the_resp.endswith("' doesn't exist\n")
if success:
if revision_id is not None:
# This is for the branch hierarchy, --svnhier.
self.assert_(the_resp.endswith('Checked out revision %d.\n'
% revision_id))
else:
# This is for the release branches, --svnrels.
m = re.search(Log_Jammin.re_checkout_rev, the_resp)
self.assert_(m is not None)
except subprocess.CalledProcessError:
success = False
return success
def something_Croc_one(self, rdat, co_path):
# FIXME: Neither exclude seems to work...
cmd_and_args = [cloc_path,
co_path,
'--quiet',
'--progress-rate=0',
]
#
exclude_switch = self.cloc_get_exclude_switch()
if exclude_switch:
cmd_and_args.append(exclude_switch)
if cloc_exclude_dirs:
cmd_and_args.append('--exclude-dir=%s' % (cloc_exclude_dirs,))
#
log.info('CLOCing revn: %d' % (rdat[Log_Jammin.p_revid],))
log.debug('CLOCing checkout: %s' % (cmd_and_args,))
cloc = misc.process_check_output(cmd_and_args)
log.verbose('CLOC results: %s' % (cloc,))
cloc_lines = cloc.split('\n')
counts = {}
state = 'reset1'
next_state = ''
for line in cloc_lines:
if state == 'reset1':
if line != '':
# E.g., <<CLOC line: Use of qw(...) as parentheses is deprecated
# at /export/scratch/ccp/dev/cp_2628/scripts/dev/cloc-1.55/
# cloc-1.55.pl line 1841.>>
log.error('CLOC line: %s' % (line,))
self.assert_(line == '')
state = 'reset2'
elif state == 'reset2':
if not line.startswith(Log_Jammin.cloc_outp_intro):
log.error('Unexpected cloc version: %s / %s'
% (line, Log_Jammin.cloc_outp_intro,))
self.assert_(False)
state = 'delim'
next_state = 'header'
elif state == 'delim':
m = Log_Jammin.re_cloc_delim.match(line)
self.assert_(m is not None)
state = next_state
next_state = ''
elif state == 'header':
m = Log_Jammin.re_cloc_header.match(line)
self.assert_(m is not None)
state = 'delim'
next_state = 'data'
elif state == 'data':
m = Log_Jammin.re_cloc_delim.match(line)
if m is not None:
state = 'footer'
else:
m = Log_Jammin.re_cloc_data.match(line)
self.assert_(m is not None)
self.proc_croc_data(m, counts)
# No state change.
elif state == 'footer':
m = Log_Jammin.re_cloc_footer.match(line)
self.proc_croc_footer(m, counts, rdat)
self.assert_(m is not None)
state = 'delim'
next_state = 'end'
elif state == 'end':
self.assert_(line == '')
state = 'eof'
elif state == 'eof':
self.assert_(False)
else:
self.assert_(False)