def examine_file(f):
"""Examine and copy a file if it needs copying."""
rval = 0
sfile = os.path.join(flag_source_dir, f)
if not os.path.exists(sfile):
u.warning("file %s does not exist in src dir -- skipping" % f)
return 0
dfile = os.path.join(flag_dest_dir, f)
docopy = False
if not os.path.exists(dfile):
u.verbose(1, "file %s does not exist in dest dir" % f)
docopy = True
else:
scksum = checksum_file(sfile)
dcksum = checksum_file(dfile)
if scksum != dcksum:
u.verbose(
1, "checksum mismatch (%s vs %s) "
"on file %s" % (scksum, dcksum, f))
docopy = True
if docopy:
if flag_dryrun:
u.verbose(0, "dryrun: cp %s %s" % (sfile, dfile))
else:
u.verbose(0, "cp %s %s" % (sfile, dfile))
u.docmd("cp %s %s" % (sfile, dfile))
u.docmd("chmod 0755 %s" % dfile)
rval = 1
return rval
def docmd(cmd):
"""Execute a command."""
if flag_dryrun or u.verbosity_level() > 0:
sys.stderr.write("executing: " + cmd + "\n")
if flag_dryrun:
return
u.docmd(cmd)
def download_blob(device, version, link):
"""Download a specific blob."""
# create location if needed
devdir = "%s/%d" % (flag_archive_dir, version)
if not os.path.isdir(devdir):
os.mkdir(devdir)
verdir = "%s/%s/%s" % (flag_archive_dir, version, device)
if not os.path.isdir(verdir):
os.mkdir(verdir)
# download file
base = os.path.basename(link)
path = "%s/%s" % (verdir, base)
if not os.path.exists(path):
print "... downloading %s => %s" % (link, path)
u.docmd("curl -L %s -o %s" % (link, path))
else:
print "... skipping %s blob %s (exists in archive already)" % (device, link)
# Update current version link
curlink = "%s/cur" % flag_archive_dir
if os.path.exists(curlink):
try:
os.remove(curlink)
except OSError as err:
u.error("unable to remove current version "
"link %s: %s" % (curlink, err))
try:
os.symlink("%d" % version, "%s/cur" % flag_archive_dir)
except OSError as err:
u.error("unable to update current version link %s" % curlink)
def capture_env_from_cmds(cmds, cachefile, errfile):
"""Capture the environment resulting from executing bash cmds."""
# Emit small bash script to execute
cmdfile = ".bashcmd"
with open(cmdfile, "w") as wf:
first = True
for c in cmds:
if first:
first = False
wf.write("%s 1> %s 2>&1\n" % (c, errfile))
else:
wf.write("%s 1>> %s 2>&1\n" % (c, errfile))
wf.write("if [ $? != 0 ]; then\n")
wf.write(" exit 1\n")
wf.write("fi\n")
wf.write("printenv > %s\n" % cachefile)
wf.write("exit 0\n")
wf.close()
rc = u.docmdnf("bash %s" % cmdfile)
if rc != 0:
u.warning("bash cmd failed")
u.warning("cmd script was:")
u.docmd("cat %s" % cmdfile)
u.warning("bash error output was:")
u.docmd("cat %s" % errfile)
raise Exception("command failed")
def mksnap_subcommand(volname, snapname):
"""Snapshot an existing BTRFS subvolume or snapshot."""
# Determine /ssd root
ssdroot = u.determine_btrfs_ssdroot(os.getcwd())
u.verbose(1, "ssdroot=%s" % ssdroot)
# Normalize snap name, volume name
volname = normalize(ssdroot, volname)
snapname = normalize(ssdroot, snapname)
# Existing volume should exist
oldvolume = "%s/%s" % (ssdroot, volname)
if not os.path.exists(oldvolume):
u.error("unable to locate existing subvolume %s" % oldvolume)
# Check to make sure the new snapshot doesn't already exist
newsnap = "%s/%s" % (ssdroot, snapname)
if os.path.exists(newsnap):
u.error("path %s already exists -- can't create" % newsnap)
# Here goes
u.docmd("sudo btrfs subvolume snapshot %s %s" % (oldvolume, newsnap))
# Repair ownership/permissions
repair(newsnap)
sys.stderr.write("... new snapshot %s created\n" % newsnap)
def examine_file(f):
"""Examine and copy a file if it needs copying."""
rval = 0
sfile = os.path.join(flag_source_dir, f)
if not os.path.exists(sfile):
u.warning("file %s does not exist in src dir -- skipping" % f)
return 0
dfile = os.path.join(flag_dest_dir, f)
docopy = False
if not os.path.exists(dfile):
u.verbose(1, "file %s does not exist in dest dir" % f)
docopy = True
else:
scksum = checksum_file(sfile)
dcksum = checksum_file(dfile)
if scksum != dcksum:
u.verbose(1, "checksum mismatch (%s vs %s) "
"on file %s" % (scksum, dcksum, f))
docopy = True
if docopy:
if flag_dryrun:
u.verbose(0, "dryrun: cp %s %s" % (sfile, dfile))
else:
u.verbose(0, "cp %s %s" % (sfile, dfile))
u.docmd("cp %s %s" % (sfile, dfile))
u.docmd("chmod 0755 %s" % dfile)
rval = 1
return rval
def mksnap_subcommand(volname, snapname):
"""Snapshot an existing BTRFS subvolume or snapshot."""
# Determine /ssd root
ssdroot = u.determine_btrfs_ssdroot(os.getcwd())
u.verbose(1, "ssdroot=%s" % ssdroot)
# Normalize snap name, volume name
volname = normalize(ssdroot, volname)
snapname = normalize(ssdroot, snapname)
# Existing volume should exist
oldvolume = "%s/%s" % (ssdroot, volname)
if not os.path.exists(oldvolume):
u.error("unable to locate existing subvolume %s" % oldvolume)
# Check to make sure the new snapshot doesn't already exist
newsnap = "%s/%s" % (ssdroot, snapname)
if os.path.exists(newsnap):
u.error("path %s already exists -- can't create" % newsnap)
# Here goes
u.docmd("sudo btrfs subvolume snapshot %s %s" % (oldvolume, newsnap))
# Repair ownership/permissions
repair(newsnap)
sys.stderr.write("... new snapshot %s created\n" % newsnap)
def save_temps(tfile, revision):
"""Save copy of temp file."""
scrubbed = scrub_filename(flag_target_file)
savepath = "/tmp/R%d.%s" % (revision, scrubbed)
u.docmd("cp %s %s" % (tfile.name, savepath))
u.verbose(0, "... saved revision %d copy of "
"%s into %s" % (revision, flag_target_file, savepath))
def docmd(cmd):
"""Execute a command."""
if flag_echo:
sys.stderr.write("executing: " + cmd + "\n")
if flag_dryrun:
return
u.docmd(cmd)
def docmd(cmd):
"""Execute a command."""
if flag_dryrun or u.verbosity_level() > 0:
sys.stderr.write("executing: " + cmd + "\n")
if flag_dryrun:
return
u.docmd(cmd)
def perform_diff():
"""Perform graphical diff."""
(modified, rightrev) = get_pred_revision_and_modified()
ltf = None
leftname = None
if flag_prev_revision:
rightrev = flag_prev_revision
if flag_revision_pair:
if modified:
u.warning("warning: working copy of %s is "
"modified" % flag_target_file)
leftrev = flag_revision_pair[0]
rightrev = flag_revision_pair[1]
ltf = tempfile.NamedTemporaryFile(mode="w",
prefix="svnpreddifftmp_REV%s_" % leftrev,
delete=True)
leftname = ltf.name
u.verbose(1, "left temp file is: %s" % leftname)
u.docmdout("svn cat -r%d %s" % (leftrev, flag_target_file), leftname)
if flag_save_temps:
save_temps(ltf, leftrev)
else:
u.verbose(1, "left file is: %s" % flag_target_file)
leftname = flag_target_file
rtf = tempfile.NamedTemporaryFile(mode="w",
prefix="svnpreddifftmp_REV%s_" % rightrev,
delete=True)
rightname = rtf.name
u.verbose(1, "right temp file is: %s" % rightname)
u.docmdout("svn cat -r%d %s" % (rightrev, flag_target_file), rightname)
if flag_save_temps:
save_temps(rtf, rightrev)
# Perform diff
u.docmd("%s %s %s" % (flag_diff_cmd, leftname, rightname))
def docmd(cmd):
"""Execute a command."""
if flag_echo:
sys.stderr.write("executing: " + cmd + "\n")
if flag_dryrun:
return
u.docmd(cmd)
def perform():
"""Main driver routine."""
# Volumes
volumes = {}
# Snapshots
snapshots = {}
# Key is vol, value is dictionary of subvolumes
voldict = defaultdict(lambda: defaultdict(int))
# Multiprocessing pool
nworkers = 8
pool = multiprocessing.Pool(processes=nworkers)
# Get info on volumes
collect_subvolumes_and_snapshots(volumes, snapshots, voldict)
# Kick off job for each volume, then snapshot
results = []
snapvols = []
for v in volumes:
u.verbose(1, "enqueue job for vol %s" % v)
r = pool.apply_async(process_volsnap, [v])
results.append(r)
snapvols.append(v)
if flag_shortrun:
break
for sv in snapshots:
u.verbose(1, "enqueue job for snap %s" % sv)
r = pool.apply_async(process_volsnap, [sv])
results.append(r)
snapvols.append(sv)
if flag_shortrun:
break
# Collect results
resdict = {}
nr = len(results)
for idx in range(0, nr):
r = results[idx]
v = snapvols[idx]
u.verbose(1, "waiting on result %d %s" % (idx, v))
pair = r.get(timeout=200)
resdict[v] = pair
# Emit results
for v in volumes:
if v in resdict:
emit(v, resdict, voldict, 0)
outf.close()
if flag_email_dest:
cmd = ("sendgmr --to=%s --body_file=%s "
"--subject='repo status "
"summary'" % (whoami, flag_outfile))
u.verbose(1, "email cmd is: %s" % cmd)
u.docmd(cmd)
def save_temps(tfile, revision):
"""Save copy of temp file."""
scrubbed = scrub_filename(flag_target_file)
savepath = "/tmp/R%d.%s" % (revision, scrubbed)
u.docmd("cp %s %s" % (tfile.name, savepath))
u.verbose(
0, "... saved revision %d copy of "
"%s into %s" % (revision, flag_target_file, savepath))
def doscmd(cmd):
"""Execute a command."""
if flag_echo:
sys.stderr.write("executing: " + cmd + "\n")
if flag_dryrun:
return
if flag_show_output:
u.docmd(cmd)
else:
u.doscmd(cmd)
def doscmd(cmd):
"""Execute a command."""
if flag_echo:
sys.stderr.write("executing: " + cmd + "\n")
if flag_dryrun:
return
if flag_show_output:
u.docmd(cmd)
else:
u.doscmd(cmd)
def perform():
"""Emit files."""
u.verbose(1, "emitting makefile.generated")
mf = open("makefile.generated", "w")
# divide instances into chunks by 10
chunks = flag_num_instances / 10
chunksize = 10
jj = 0
for ch in range(chunks):
mf.write("OBJECTS%s =" % ch)
for ii in range(chunksize):
mf.write(" i%d_generated.o i%d_usegen.o" % (jj, jj))
jj += 1
mf.write("\n")
# all objects
mf.write("OBJECTS =")
for ch in range(chunks):
mf.write(" $(OBJECTS%d)" % ch)
# emit a series of partial targets by chunk
mf.write("\n")
for kk in range(chunks):
mf.write("\ngenerated-%d-%d.so:" % (0, kk))
for ii in range(0, kk + 1):
mf.write(" $(OBJECTS%d)" % ii)
mf.write("\n")
mf.write("\t$(CXX) $(CXXFLAGS) $? -shared -o $@\n")
# the big one
mf.write("\n")
mf.write("generated.so: $(OBJECTS)\n")
mf.write("\t$(CXX) $(CXXFLAGS) $? -shared -o $@\n")
# and some pseudo-targets
mf.write("\n")
mf.write("\nallgen:")
kk = 0
for ii in range(chunks):
if kk + 1 > chunks:
kk = chunks - 1
mf.write(" generated-%d-%d.so" % (0, kk))
break
mf.write(" generated-%d-%d.so" % (0, kk))
kk += 2
mf.write("\n")
mf.write("\n")
mf.write("\nallobjs: $(OBJECTS)\n")
mf.close()
# the source code
for ii in range(flag_num_instances):
u.docmd("gencode_classes.py -c %s -C i%d_ -F "
"i%d\n" % (flag_num_classes, ii, ii))
def install_blob(blob, devdir):
"""Install a single blob."""
# Determine blob name
blobpath = "%s/%s" % (devdir, blob)
lines = u.docmdlines("tar tzf %s" % blobpath)
if len(lines) != 1:
u.error("error while examining blob %s: expected single file" % blob)
# Unpack
blobfile = lines[0]
u.verbose(1, "unpacking blob %s" % blob)
u.docmd("tar xzf %s" % blobpath)
# Invoke installer
u.docmd("blobinstall.py %s" % blobfile)
def install_blob(blob, devdir):
"""Install a single blob."""
# Determine blob name
blobpath = "%s/%s" % (devdir, blob)
lines = u.docmdlines("tar tzf %s" % blobpath)
if len(lines) != 1:
u.error("error while examining blob %s: expected single file" % blob)
# Unpack
blobfile = lines[0]
u.verbose(1, "unpacking blob %s" % blob)
u.docmd("tar xzf %s" % blobpath)
# Invoke installer
u.docmd("blobinstall.py %s" % blobfile)
def perform(repo, flags):
"""Patch specified repo."""
u.verbose(1, "repo: %s flags: '%s'" % (repo, flags))
rc = u.docmdnf("grep -q gccgoflags %s/src/cmd/dist/buildtool.go" % repo)
if rc == 0:
u.verbose(1, "src/cmd/dist/buildtool.go already patched")
return
# Remove any version file if it exists.
vfile = os.path.join(repo, "VERSION")
if os.path.exists(vfile):
os.unlink(vfile)
# Mangle build flags.
regex = re.compile(r"^.+gcflags=.+$")
oldf = "%s/src/cmd/dist/buildtool.go" % repo
newf = "%s/src/cmd/dist/buildtool.go.patched" % repo
try:
with open(newf, "w") as wf:
try:
with open(oldf, "r") as rf:
lines = rf.readlines()
for line in lines:
if regex.match(line):
comps = line.split()
newcomps = []
for c in comps:
if c == "\"-gcflags=-l\",":
u.verbose(0, "patching gcflags line\n")
newcomps.append("\"-gccgoflags=%s\", " %
flags)
newcomps.append("\"-p=8\", ")
if flag_addmx:
newcomps.append("\"-x\", ")
newcomps.append(c)
line = " ".join(newcomps)
line += "\n"
wf.write(line)
except IOError:
u.verbose(0, "open failed for %s" % oldf)
except IOError:
u.verbose(0, "open failed for %s" % newf)
u.verbose(1, "mv %s %s" % (newf, oldf))
u.docmd("mv %s %s" % (newf, oldf))
def collectem():
"""Locate and upload tombstones."""
# Need root access to get at /data/tombstones
u.doscmd("adb root")
# See what's available
lines = u.docmdlines("adb shell ls -l /data/tombstones")
# Things we found
fnames = []
# -rw------- system system 56656 2015-05-25 03:01 tombstone_09
matcher = re.compile(r"^\S+\s+\S+\s+\S+\s+\d+\s+(\S+)\s+(\S+)\s+(tomb\S+)\s*$")
for line in lines:
u.verbose(3, "line is: %s" % line)
m = matcher.match(line)
if m:
datestr = m.group(1)
timestr = m.group(2)
fname = m.group(3)
fnames.append(fname)
u.verbose(1, ("found tombstone %s date %s time %s" %
(fname, datestr, timestr)))
u.docmd("mkdir -p /tmp/tombstones/%s" % whichdev)
newname = ("/tmp/tombstones/%s/%s_%s_%s" %
(whichdev, fname, datestr, timestr))
u.docmd("adb pull /data/tombstones/%s %s_tmp" % (fname, newname))
if os.path.exists(newname):
# already there?
rc = u.docmdnf("cmp -s %s %s_tmp" % (newname, newname))
if rc == 0:
print "file %s already uploaded, skipping..." % fname
else:
print "overwriting existing %s with new version" % fname
else:
u.docmdnf("mv %s_tmp %s" % (newname, newname))
print "uploaded new tombstone to %s" % newname
# Anything there?
if not fnames:
print "No tombstones found... terminating."
def perform():
"""Main driver routine."""
# Check for existence of ninja log file
if not os.path.exists(".ninja_log"):
u.error("unable to access .ninja_log file")
# Generate json file from ninja log
if flag_dryrun or flag_echo:
u.verbose(0, "%s .ninja_log > trace.json" % flag_ninjatracing)
if not flag_dryrun:
u.docmdout("%s .ninja_log" % flag_ninjatracing, "trace.json")
# Generate trace.html file from json
cmd = ("%s/tracing/bin/trace2html trace.json "
"--output=%s" % (flag_catapult, flag_outfile))
if flag_dryrun or flag_echo:
u.verbose(0, cmd)
if not flag_dryrun:
u.docmd(cmd)
def perform(repo, flags):
"""Patch specified repo."""
u.verbose(1, "repo: %s flags: '%s'" % (repo, flags))
rc = u.docmdnf("grep -q gccgoflags %s/src/cmd/dist/buildtool.go" % repo)
if rc == 0:
u.verbose(1, "src/cmd/dist/buildtool.go already patched")
return
# Remove any version file if it exists.
vfile = os.path.join(repo, "VERSION")
if os.path.exists(vfile):
os.unlink(vfile)
# Mangle build flags.
regex = re.compile(r"^.+gcflags=.+$")
oldf = "%s/src/cmd/dist/buildtool.go" % repo
newf = "%s/src/cmd/dist/buildtool.go.patched" % repo
try:
with open(newf, "w") as wf:
try:
with open(oldf, "r") as rf:
lines = rf.readlines()
for line in lines:
if regex.match(line):
comps = line.split()
newcomps = []
for c in comps:
if c == "\"-gcflags=-l\",":
u.verbose(0, "patching gcflags line\n")
newcomps.append("\"-gccgoflags=%s\", " % flags)
newcomps.append("\"-p=8\", ")
if flag_addmx:
newcomps.append("\"-x\", ")
newcomps.append(c)
line = " ".join(newcomps)
line += "\n"
wf.write(line)
except IOError:
u.verbose(0, "open failed for %s" % oldf)
except IOError:
u.verbose(0, "open failed for %s" % newf)
u.verbose(1, "mv %s %s" % (newf, oldf))
u.docmd("mv %s %s" % (newf, oldf))
def mkvol_subcommand(volname):
"""Create a new btrfs subvolume."""
# Determine /ssd root
ssdroot = u.determine_btrfs_ssdroot(os.getcwd())
u.verbose(1, "ssdroot=%s" % ssdroot)
# Normalize snap name
volname = normalize(ssdroot, volname)
# Check to make sure the new volume doesn't already exist
newvolume = "%s/%s" % (ssdroot, volname)
if os.path.exists(newvolume):
u.error("path %s already exists -- can't create" % newvolume)
# Here goes
u.docmd("sudo btrfs subvolume create %s" % newvolume)
# Repair ownership/permissions
repair(newvolume)
sys.stderr.write("... new subvolume %s created\n" % newvolume)
def mkvol_subcommand(volname):
"""Create a new btrfs subvolume."""
# Determine /ssd root
ssdroot = u.determine_btrfs_ssdroot(os.getcwd())
u.verbose(1, "ssdroot=%s" % ssdroot)
# Normalize snap name
volname = normalize(ssdroot, volname)
# Check to make sure the new volume doesn't already exist
newvolume = "%s/%s" % (ssdroot, volname)
if os.path.exists(newvolume):
u.error("path %s already exists -- can't create" % newvolume)
# Here goes
u.docmd("sudo btrfs subvolume create %s" % newvolume)
# Repair ownership/permissions
repair(newvolume)
sys.stderr.write("... new subvolume %s created\n" % newvolume)
def perform():
"""Main driver routine."""
if run_javac_and_dx:
docmd("javac -g -source 1.7 -target 1.7 %s" % flag_progname)
docmd("%s -JXmx256m --debug --dex "
"--output=classes.dex %s.class" % (dxpath, flag_progbase))
doscmd("zip %s.jar classes.dex" % flag_progbase)
doscmd("adb push %s.jar /data/local/tmp" % flag_progbase)
if flag_nativelibs:
for lib in flag_nativelibs:
doscmd("adb push %s /data/local/tmp" % lib)
if flag_simpleperf_static:
doscmd("adb push %s/system/bin/simpleperf_static /data/local/tmp" %
apo)
emit_cmds()
if flag_dryrun:
u.verbose(0, "contents of cmd file:")
u.docmd("cat %s" % cmdfile)
rc = docmdnf("adb shell sh /data/local/tmp/%s" % cmdfile)
if rc != 0:
u.error("** command failed: adb shell sh "
"/data/local/tmp/%s (temp file left in .)" % cmdfile)
else:
if flag_preserve:
u.verbose(
0, "cmd files preserved: %s on "
"host and /data/local/tmp/%s on target" % (cmdfile, cmdfile))
else:
u.docmd("rm -f %s" % cmdfile)
docmd("adb shell rm -f /data/local/tmp/%s" % cmdfile)
if flag_strace:
docmd("adb pull /data/local/tmp/run-trace.txt .")
if flag_dex2oat:
docmd("adb pull /data/local/tmp/compile-trace.txt .")
if flag_simpleperf:
docmd("adb pull /data/local/tmp/perf.data .")
perform_symbolization()
if flag_simpleperf:
docmdout("simpleperf report --symfs %s/symbols" % apo, "report.txt")
def perform():
"""Main driver routine."""
# Emit temp file to use for pprof input
try:
scriptf = tempfile.NamedTemporaryFile(mode="w", delete=True)
outf = tempfile.NamedTemporaryFile(mode="w", delete=True)
ppo = open(scriptf.name, "w")
except IOError:
u.verbose(0, "open failed for %s" % outf.name)
u.verbose(1, "opened tempfile %s" % outf.name)
profiles = {"top15": "%s/top15.%s.txt" % (flag_outdir, flag_tag),
"top100": "%s/top100.%s.txt" % (flag_outdir, flag_tag),
"svg": "%s/graph.%s.svg" % (flag_outdir, flag_tag),
"tree": "%s/tree.%s.txt" % (flag_outdir, flag_tag),
"raw": "%s/raw.%s.txt" % (flag_outdir, flag_tag)}
# Write to file
for p, dest in profiles.iteritems():
ppo.write("%s > %s\n" % (p, dest))
ppo.write("quit\n")
ppo.close()
if u.verbosity_level() > 1:
u.verbose(0, "tempfile contents:")
u.docmd("cat %s" % scriptf.name)
# Execute
pcmd = "%s %s %s" % (flag_pprof_path, flag_binary,
" ".join(flag_infiles))
docmdinout(pcmd, scriptf.name, outf.name)
# Check to make sure the files turned up.
if not flag_dryrun:
for p, dest in profiles.iteritems():
if not os.path.exists(dest):
u.error("error: %s profile '%s' not present "
"after pprof run" % (p, dest))
def perform():
"""Main driver routine."""
if run_javac_and_dx:
docmd("javac -g -source 1.7 -target 1.7 %s" % flag_progname)
docmd("%s -JXmx256m --debug --dex "
"--output=classes.dex %s.class" % (dxpath, flag_progbase))
doscmd("zip %s.jar classes.dex" % flag_progbase)
doscmd("adb push %s.jar /data/local/tmp" % flag_progbase)
if flag_nativelibs:
for lib in flag_nativelibs:
doscmd("adb push %s /data/local/tmp" % lib)
if flag_simpleperf_static:
doscmd("adb push %s/system/bin/simpleperf_static /data/local/tmp" % apo)
emit_cmds()
if flag_dryrun:
u.verbose(0, "contents of cmd file:")
u.docmd("cat %s" % cmdfile)
rc = docmdnf("adb shell sh /data/local/tmp/%s" % cmdfile)
if rc != 0:
u.error("** command failed: adb shell sh "
"/data/local/tmp/%s (temp file left in .)" % cmdfile)
else:
if flag_preserve:
u.verbose(0, "cmd files preserved: %s on "
"host and /data/local/tmp/%s on target"
% (cmdfile, cmdfile))
else:
u.docmd("rm -f %s" % cmdfile)
docmd("adb shell rm -f /data/local/tmp/%s" % cmdfile)
if flag_strace:
docmd("adb pull /data/local/tmp/run-trace.txt .")
if flag_dex2oat:
docmd("adb pull /data/local/tmp/compile-trace.txt .")
if flag_simpleperf:
docmd("adb pull /data/local/tmp/perf.data .")
perform_symbolization()
if flag_simpleperf:
docmdout("simpleperf report --symfs %s/symbols" % apo, "report.txt")
def perform_diff():
"""Perform graphical diff."""
(modified, rightrev) = get_pred_revision_and_modified()
ltf = None
leftname = None
if flag_prev_revision:
rightrev = flag_prev_revision
if flag_revision_pair:
if modified:
u.warning("warning: working copy of %s is "
"modified" % flag_target_file)
leftrev = flag_revision_pair[0]
rightrev = flag_revision_pair[1]
ltf = tempfile.NamedTemporaryFile(mode="w",
prefix="svnpreddifftmp_REV%s_" %
leftrev,
delete=True)
leftname = ltf.name
u.verbose(1, "left temp file is: %s" % leftname)
u.docmdout("svn cat -r%d %s" % (leftrev, flag_target_file), leftname)
if flag_save_temps:
save_temps(ltf, leftrev)
else:
u.verbose(1, "left file is: %s" % flag_target_file)
leftname = flag_target_file
rtf = tempfile.NamedTemporaryFile(mode="w",
prefix="svnpreddifftmp_REV%s_" %
rightrev,
delete=True)
rightname = rtf.name
u.verbose(1, "right temp file is: %s" % rightname)
u.docmdout("svn cat -r%d %s" % (rightrev, flag_target_file), rightname)
if flag_save_temps:
save_temps(rtf, rightrev)
# Perform diff
u.docmd("%s %s %s" % (flag_diff_cmd, leftname, rightname))
def test_docmd_pass(self):
u.docmd("/bin/true")
def test_docmd_fail(self):
with self.assertRaises(Exception):
u.docmd("/bin/false")
def perform():
"""Main driver routine."""
vnames = ["alloc_space", "inuse_space", "alloc_objects", "inuse_objects"]
infixes = []
if flag_tag:
infixes = flag_tag
variants = []
if flag_memvariants:
for v in vnames:
infixes = [v]
if flag_tag:
infixes = [flag_tag, v]
variants.append([v, infixes])
else:
infixes = []
if flag_tag:
infixes = [flag_tag]
variants.append(["", infixes])
for v in variants:
ppopt = v[0]
infixes = v[1]
# Emit temp file to use for pprof input
try:
scriptf = tempfile.NamedTemporaryFile(mode="w", delete=True)
outf = tempfile.NamedTemporaryFile(mode="w", delete=True)
ppo = open(scriptf.name, "w")
except IOError:
u.verbose(0, "open failed for %s" % outf.name)
u.verbose(1, "opened tempfile %s" % outf.name)
profiles = {}
pnames = ["top15", "top100", "svg", "tree", "raw"]
for pn in pnames:
fn = "%s/%s" % (flag_outdir, pn)
tojoin = [fn]
tojoin.extend(infixes)
tojoin.append("txt")
fn = ".".join(tojoin)
profiles[pn] = fn
# Write to file
for p, dest in profiles.items():
ppo.write("%s > %s\n" % (p, dest))
ppo.write("quit\n")
ppo.close()
if u.verbosity_level() > 1:
u.verbose(0, "tempfile contents:")
u.docmd("cat %s" % scriptf.name)
# Execute
pcmd = "%s %s %s %s" % (flag_pprof_path, ppopt, flag_binary,
" ".join(flag_infiles))
docmdinout(pcmd, scriptf.name, outf.name)
# Check to make sure the files turned up.
if not flag_dryrun:
for p, dest in profiles.items():
if not os.path.exists(dest):
u.error("error: %s profile '%s' not present "
"after pprof run" % (p, dest))
def perform():
"""Main driver routine."""
vnames = ["alloc_space", "inuse_space", "alloc_objects", "inuse_objects"]
infixes = []
if flag_tag:
infixes = flag_tag
variants = []
if flag_memvariants:
for v in vnames:
infixes = [v]
if flag_tag:
infixes = [flag_tag, v]
variants.append([v, infixes])
else:
infixes = []
if flag_tag:
infixes = [flag_tag]
variants.append(["", infixes])
for v in variants:
ppopt = v[0]
infixes = v[1]
# Emit temp file to use for pprof input
try:
scriptf = tempfile.NamedTemporaryFile(mode="w", delete=True)
outf = tempfile.NamedTemporaryFile(mode="w", delete=True)
ppo = open(scriptf.name, "w")
except IOError:
u.verbose(0, "open failed for %s" % outf.name)
u.verbose(1, "opened tempfile %s" % outf.name)
profiles = {}
pnames = ["top15", "top100", "svg", "tree", "raw"]
for pn in pnames:
fn = "%s/%s" % (flag_outdir, pn)
tojoin = [fn]
tojoin.extend(infixes)
tojoin.append("txt")
fn = ".".join(tojoin)
profiles[pn] = fn
# Write to file
for p, dest in profiles.items():
ppo.write("%s > %s\n" % (p, dest))
ppo.write("quit\n")
ppo.close()
if u.verbosity_level() > 1:
u.verbose(0, "tempfile contents:")
u.docmd("cat %s" % scriptf.name)
# Execute
pcmd = "%s %s %s %s" % (flag_pprof_path, ppopt, flag_binary,
" ".join(flag_infiles))
docmdinout(pcmd, scriptf.name, outf.name)
# Check to make sure the files turned up.
if not flag_dryrun:
for p, dest in profiles.items():
if not os.path.exists(dest):
u.error("error: %s profile '%s' not present "
"after pprof run" % (p, dest))
def perform_filt(inf, outf):
"""Read inf and emit summary to outf."""
ncollections = 0
nroots = 0
collections = []
elist = []
addrsize = {}
# Read input
while True:
line = inf.readline()
if not line:
break
u.verbose(3, "line is %s" % line)
# Root collection start?
m1 = rcstartre.match(line)
if m1:
collections.append(elist)
elist = []
ncollections += 1
continue
# Root list entry?
m2 = rlere.match(line)
if m2:
nroots += 1
hexaddr = m2.group(1)
siz = m2.group(2)
addrsize[hexaddr] = int(siz)
elist.append(hexaddr)
continue
if elist:
collections.append(elist)
# Now that we've read everything, write GDB script.
if os.path.exists("gdb-cmds.txt"):
os.unlink("gdb-cmds.txt")
if os.path.exists("gdb-out.txt"):
os.unlink("gdb-out.txt")
try:
gf = open("gdb-cmds.txt", "wb")
except IOError as e:
u.error("unable to open output file 'gdbcmds.txt': %s" % e.strerror)
gf.write("set height 0\n")
gf.write("set width 0\n")
gf.write("set pagination off\n")
gf.write("set logging file gdb-out.txt\n")
gf.write("set logging on\n")
gf.write("file %s\n" % flag_module)
ncol = 0
for el in collections:
gf.write("print \"collection %d\"\n" % ncol)
ncol += 1
for hexaddr in el:
gf.write("print \"0x%x size %d\"\n" % (int(hexaddr, 16), addrsize[hexaddr]))
gf.write("info sym 0x%s\n" % hexaddr)
gf.close()
# Invoke GDB
u.docmd("gdb -batch -nh -x gdb-cmds.txt")
# Emit
try:
rf = open("gdb-out.txt", "r")
except IOError as e:
u.error("unable to open output file 'gdb-out.txt': %s" % e.strerror)
lines = rf.readlines()
rf.close()
for line in lines:
outf.write(line)
outf.close()
u.verbose(0, "processed %d roots in %d collections" % (nroots, ncollections))
def perform():
"""Main driver routine."""
vnames = ["alloc_space", "inuse_space", "alloc_objects", "inuse_objects"]
infixes = []
if flag_tag:
infixes = flag_tag
variants = []
if flag_memvariants:
for v in vnames:
infixes = [v]
if flag_tag:
infixes = [flag_tag, v]
variants.append([v, infixes])
else:
infixes = []
if flag_tag:
infixes = [flag_tag]
variants.append(["", infixes])
if flag_capture:
# Copy input files
docmd("cp %s %s" % (" ".join(flag_infiles), flag_outdir))
try:
with open("%s/runpprof.sh" % flag_outdir, "w") as wf:
wf.write("#/bin/sh\n")
wf.write("# run me from parent of outdir.\n")
wf.write("set -x\n")
except IOError:
u.verbose(0, "open for write failed for 'runprof.sh'")
for v in variants:
ppopt = v[0]
if ppopt:
ppopt = "-" + ppopt
infixes = v[1]
# Emit temp file to use for pprof input
try:
scriptf = tempfile.NamedTemporaryFile(mode="w",
delete=flag_cleantemps)
outf = tempfile.NamedTemporaryFile(mode="w",
delete=flag_cleantemps)
ppo = open(scriptf.name, "w")
except IOError:
u.verbose(0, "open failed for %s" % outf.name)
u.verbose(1, "opened tempfile %s" % outf.name)
profiles = {}
pnames = ["top15", "top100", "svg", "tree", "raw"]
for pn in pnames:
fn = "%s/%s" % (flag_outdir, pn)
tojoin = [fn]
tojoin.extend(infixes)
ext = "txt"
if pn == "svg":
ext = "svg"
tojoin.append(ext)
fn = ".".join(tojoin)
profiles[pn] = fn
# Write to file
for p, dest in profiles.items():
ppo.write("%s > %s\n" % (p, dest))
ppo.write("quit\n")
ppo.close()
if u.verbosity_level() > 1:
u.verbose(0, "tempfile contents:")
u.docmd("cat %s" % scriptf.name)
# Execute
lines = ""
if flag_dashlines:
lines = "-lines"
pcmd = "%s %s %s %s %s" % (flag_pprof_path, lines, ppopt, flag_binary,
" ".join(flag_infiles))
docmdinout(pcmd, scriptf.name, outf.name)
if flag_capture:
if ppopt:
cfile = "pprofcmds.%s.txt" % ppopt
else:
cfile = "pprofcmds.txt"
docmd("cp %s %s/%s" % (scriptf.name, flag_outdir, cfile))
wf = None
try:
with open("%s/runpprof.sh" % flag_outdir, "a") as wf:
wf.write(pcmd)
wf.write(" < %s/%s\n" % (flag_outdir, cfile))
except IOError:
u.verbose(0, "open for append failed for 'runprof.sh'")
# Check to make sure the files turned up.
if not flag_dryrun:
for p, dest in profiles.items():
if not os.path.exists(dest):
u.error("error: %s profile '%s' not present "
"after pprof run" % (p, dest))
def test_docmd_fail(self):
with self.assertRaises(Exception):
u.docmd("/bin/false")
def repair(newvolume):
"""Repair ownership/permissions for new snapshot/subvolume."""
u.docmd("sudo chown --reference=%s %s" % (flag_homedir, newvolume))
u.docmd("sudo chgrp --reference=%s %s" % (flag_homedir, newvolume))
u.docmd("chmod 0750 %s" % newvolume)
def repair(newvolume):
"""Repair ownership/permissions for new snapshot/subvolume."""
u.docmd("sudo chown --reference=%s %s" % (flag_homedir, newvolume))
u.docmd("sudo chgrp --reference=%s %s" % (flag_homedir, newvolume))
u.docmd("chmod 0750 %s" % newvolume)
def test_docmd_pass(self):
u.docmd("/bin/true")
def perform_filt(inf, outf):
"""Read inf and emit summary to outf."""
ncollections = 0
nroots = 0
collections = []
elist = []
addrsize = {}
# Read input
while True:
line = inf.readline()
if not line:
break
u.verbose(3, "line is %s" % line)
# Root collection start?
m1 = rcstartre.match(line)
if m1:
collections.append(elist)
elist = []
ncollections += 1
continue
# Root list entry?
m2 = rlere.match(line)
if m2:
nroots += 1
hexaddr = m2.group(1)
siz = m2.group(2)
addrsize[hexaddr] = int(siz)
elist.append(hexaddr)
continue
if elist:
collections.append(elist)
# Now that we've read everything, write GDB script.
if os.path.exists("gdb-cmds.txt"):
os.unlink("gdb-cmds.txt")
if os.path.exists("gdb-out.txt"):
os.unlink("gdb-out.txt")
try:
gf = open("gdb-cmds.txt", "wb")
except IOError as e:
u.error("unable to open output file 'gdbcmds.txt': %s" % e.strerror)
gf.write("set height 0\n")
gf.write("set width 0\n")
gf.write("set pagination off\n")
gf.write("set logging file gdb-out.txt\n")
gf.write("set logging on\n")
gf.write("file %s\n" % flag_module)
ncol = 0
for el in collections:
gf.write("print \"collection %d\"\n" % ncol)
ncol += 1
for hexaddr in el:
gf.write("print \"0x%x size %d\"\n" %
(int(hexaddr, 16), addrsize[hexaddr]))
gf.write("info sym 0x%s\n" % hexaddr)
gf.close()
# Invoke GDB
u.docmd("gdb -batch -nh -x gdb-cmds.txt")
# Emit
try:
rf = open("gdb-out.txt", "r")
except IOError as e:
u.error("unable to open output file 'gdb-out.txt': %s" % e.strerror)
lines = rf.readlines()
rf.close()
for line in lines:
outf.write(line)
outf.close()
u.verbose(0,
"processed %d roots in %d collections" % (nroots, ncollections))
def perform_extract(inf, outf):
"""Read inf and extract compile cmd to outf."""
global driver_count, workdir
regwrk = re.compile(r"^WORK=(\S+)$")
reggcg = re.compile(r"^(\S+/bin/gccgo)\s+(.+)$")
regglm = re.compile(r"^(\S+/bin/llvm-goc)\s+(.+)$")
reggc = re.compile(r"^(\S+/compile)\s+(.+)$")
regcd = re.compile(r"^cd (\S+)\s*$")
regar = re.compile(r"^ar .+$")
regcp = re.compile(r"^cp (\S+) (\S+)$")
regmkdir = re.compile(r"^mkdir .+$")
regcat = re.compile(r"^cat >\$WORK.*$")
regeof = re.compile(r"^EOF\s*$")
outf.write("#!/bin/sh\n")
outf.write("# [this file auto-generated via the command:]\n")
outf.write("# %s\n" % " ".join(sys.argv))
if flag_cpuprofile:
outf.write("pcount=0\n")
if flag_emitloop:
outf.write("ITER=0\n")
outf.write("if [ -z $LOOP_LIMIT ]; then\n")
outf.write(" LOOP_LIMIT=1\n")
outf.write("fi\n")
outf.write("while [ $ITER -lt $LOOP_LIMIT ];\ndo\n")
outf.write("ITER=`expr $ITER + 1`\n")
cdir = "."
while True:
line = inf.readline()
if not line:
break
dflavor = ""
u.verbose(2, "line is %s" % line.strip())
mwrk = regwrk.match(line)
if mwrk:
workdir = mwrk.group(1)
if flag_relocate:
u.verbose(0, "... relocating work dir %s to "
"%s" % (workdir, flag_relocate))
if os.path.exists(flag_relocate):
u.docmd("rm -rf %s" % flag_relocate)
u.docmd("cp -r %s %s" % (workdir, flag_relocate))
outf.write("WORK=%s\n" % flag_relocate)
else:
outf.write(line)
continue
mcd = regcd.match(line)
if mcd:
cdir = mcd.group(1)
outf.write("cd %s\n" % cdir)
continue
mmkdir = regmkdir.match(line)
if mmkdir:
outf.write(line)
continue
mcat = regcat.match(line)
if mcat:
while True:
line = inf.readline()
if not line:
u.error("fatal: end of file while looking for EOF tag")
u.verbose(2, "line is %s" % line.strip())
meof = regeof.match(line)
if meof:
u.verbose(2, "seen EOF ==")
break
continue
mar = regar.match(line)
if mar:
outf.write(line)
continue
mcp = regcp.match(line)
if mcp:
outf.write(line)
continue
mgc = reggc.match(line)
mgcg = reggcg.match(line)
mgcglm = regglm.match(line)
if not mgc and not mgcg and not mgcglm:
continue
if mgc:
driver = mgc.group(1)
dflavor = "gc"
argstring = mgc.group(2)
elif mgcglm:
driver = mgcglm.group(1)
dflavor = "gollvm"
argstring = mgcglm.group(2)
else:
driver = mgcg.group(1)
dflavor = "gccgo"
argstring = mgcg.group(2)
u.verbose(1, "matched: %s %s" % (driver, argstring))
driver_var = "DRIVER%d" % driver_count
if driver in drivers:
driver_var = drivers[driver]
else:
outf.write("%s=%s\n" % (driver_var, driver))
drivers[driver] = driver_var
driver_flavor[driver_var] = dflavor
driver_count += 1
if extract_line(outf, driver, driver_var, argstring, cdir) < 0:
break
if flag_emitloop:
outf.write("done\n")