def insert_before_if_not_already_present(mfile, insertloc, keyword, toadd):
"""Insert specified chunk of text to makefile if not already present."""
if not os.path.exists(mfile):
u.error("bad entry in munge makefile table-- %s "
"does not appear to exist" % mfile)
mfile_new = "%s.munged" % mfile
u.verbose(2, "examining %s in insert-before munge" % mfile)
with open(mfile, "r") as rf:
with open(mfile_new, "w") as wf:
lines = rf.readlines()
linecount = 0
for line in lines:
linecount += 1
# Already present?
linewords = line.split()
for word in linewords:
if word == keyword:
u.verbose(2, "found keyword %s at line %d "
"in %s" % (keyword, linecount, mfile))
return False
# At insertloc?
if line.strip() == insertloc:
u.verbose(2, "adding %s insert-before text at line %d "
"in %s " % (keyword, linecount, mfile))
wf.write("%s\n" % toadd)
wf.write(line)
else:
wf.write(line)
docmd("mv -f %s %s" % (mfile_new, mfile))
return True
def setup_go(targ):
"""Set up go-specific stuff."""
if os.path.exists("gofrontend"):
u.verbose(0, "... 'gofrontend' already exists, skipping clone")
return
docmd("git clone https://go.googlesource.com/gofrontend")
dochdir("gofrontend")
try:
with open("./.clang-format", "w") as wf:
wf.write(clang_format_contents)
wf.write("\n")
except IOError:
u.error("open/write failed for .clang-format")
dochdir("..")
dochdir(targ)
docmd("rm -rf gcc/go/gofrontend")
docmd("ln -s ../../../gofrontend/go gcc/go/gofrontend")
docmd("rm -rf libgo")
docmd("mkdir libgo")
if flag_dryrun:
u.verbose(0, "for f in GOFRONTEND/libgo/*; "
"do ln -s $f libgo/`basename $f`; done")
else:
libgo = "../gofrontend/libgo"
for item in os.listdir(libgo):
docmd("ln -s ../../gofrontend/libgo/%s libgo/%s" % (item, item))
dochdir("..")
def parse_args():
"""Command line argument parsing."""
global flag_dryrun, flag_mode, flag_noclean
try:
optlist, args = getopt.getopt(sys.argv[1:], "hdDSF:")
except getopt.GetoptError as err:
# unrecognized option
usage(str(err))
for opt, arg in optlist:
if opt == "-d":
u.increment_verbosity()
elif opt == "-h":
usage()
elif opt == "-D":
flag_dryrun = True
elif opt == "-S":
flag_noclean = True
elif opt == "-F":
if os.path.exists(arg):
u.verbose(0, "adding %s to cmakefiles dict" % arg)
cmakefiles[arg] = 1
else:
u.error("-F arg %s doesn't seem to exist" % arg)
# Check for mode
if len(args) != 1:
usage("supply a single mode argument (either 'pre' or 'post')")
if args[0] == "pre":
flag_mode = "pre"
elif args[0] == "post":
flag_mode = "post"
else:
usage("unknown mode argument %s" % args[0])
def mksnap_subcommand(volname, snapname):
"""Snapshot an existing BTRFS subvolume or snapshot."""
# Determine /ssd root
ssdroot = u.determine_btrfs_ssdroot(os.getcwd())
u.verbose(1, "ssdroot=%s" % ssdroot)
# Normalize snap name, volume name
volname = normalize(ssdroot, volname)
snapname = normalize(ssdroot, snapname)
# Existing volume should exist
oldvolume = "%s/%s" % (ssdroot, volname)
if not os.path.exists(oldvolume):
u.error("unable to locate existing subvolume %s" % oldvolume)
# Check to make sure the new snapshot doesn't already exist
newsnap = "%s/%s" % (ssdroot, snapname)
if os.path.exists(newsnap):
u.error("path %s already exists -- can't create" % newsnap)
# Here goes
u.docmd("sudo btrfs subvolume snapshot %s %s" % (oldvolume, newsnap))
# Repair ownership/permissions
repair(newsnap)
sys.stderr.write("... new snapshot %s created\n" % newsnap)
def dobuild():
"""Run the build and test."""
dochdir(flag_build_dir)
try:
with open("tmp.sh", "w") as wf:
wf.write("#/bin/sh\n")
scriptbody = """\
echo "make -j20 all 1> berr.txt 2>&1"
make -j20 all 1> berr.txt 2>&1
if [ $? != 0 ]; then
echo "** build failed, skipping tests"
emacs berr.txt &
exit 9
fi
echo "make -j20 check-go 1> terr.txt 2>&1"
make -j20 check-go 1> terr.txt 2>&1
if [ $? != 0 ]; then
echo "** test failed"
emacs berr.txt terr.txt &
exit 9
fi
echo "result: PASS"
emacs berr.txt terr.txt &"""
wf.write(scriptbody)
except IOError:
u.error("open failed for tmp.sh")
docmd("sh tmp.sh")
def patch_gmp_configure():
"""Ridiculous that this is needed..."""
if not flag_dryrun:
try:
with open("gmp/configure", "r") as rf:
lines = rf.readlines()
try:
with open("gmp/configure.hacked", "w") as wf:
matcher = re.compile(r"^\s+M4=m4\-not\-needed\s*$")
for line in lines:
m = matcher.match(line)
if m:
wf.write(" echo M4=m4-not-needed\n")
else:
wf.write(line)
wf.close()
docmd("mv gmp/configure gmp/configure.orig")
docmd("mv gmp/configure.hacked gmp/configure")
docmd("chmod 0755 gmp/configure")
except IOError:
u.error("open failed for gmp/configure.hacked")
except IOError:
u.error("open failed for gmp/configure")
else:
u.verbose(0, "<applying gmp configure hack>")
def collect_all_loadmodules():
"""Collect names of all interesting loadmodules."""
locations = None
if flag_filemode == "target":
locations = "%s/symbols/system" % apo
else:
locations = "%s/bin %s/lib64" % (aho, aho)
u.verbose(1, "collecting loadmodules from %s" % locations)
cmd = "find %s -type f -print" % locations
u.verbose(1, "find cmd: %s" % cmd)
cargs = shlex.split(cmd)
mypipe = subprocess.Popen(cargs, stdout=subprocess.PIPE)
pout, _ = mypipe.communicate()
if mypipe.returncode != 0:
u.error("command failed (rc=%d): cmd was %s" % (mypipe.returncode, cmd))
encoding = locale.getdefaultlocale()[1]
decoded = pout.decode(encoding)
lines = decoded.strip().split("\n")
u.verbose(1, "found a total of %d load modules" % len(lines))
for line in lines:
path = line.strip()
u.verbose(2, "adding LM %s" % path)
all_loadmodules[path] = 0
bn = os.path.basename(path)
pdict = base_to_paths[bn]
pdict[path] = 1
if flag_backward_slice:
for filearg in flag_input_files:
bn = os.path.basename(filearg)
if bn not in all_loadmodules:
u.warning("argument %s not found in all_loadmodules "
"-- unable to compute slice" % filearg)
def benchmark(repo, runscript, wrapcmd, tag):
"""Run benchmark build."""
f = "bench.%s.%s.sh" % (repo, tag)
u.verbose(1, "... running %s" % f)
if os.path.exists(f):
rmfile(f)
try:
with open(f, "w") as wf:
wf.write("#!/bin/sh\n")
wf.write("set -x\n")
if flag_gomaxprocs:
wf.write("export GOMAXPROCS=%s\n" % flag_gomaxprocs)
wf.write("export LD_LIBRARY_PATH=%s/lib64\n" % gccgo_install)
wf.write("go clean -cache\n")
wf.write("cd %s/src/cmd/compile\n" % repo)
wf.write("%s sh %s\n" % (wrapcmd, runscript))
wf.write("if [ $? != 0 ]; then\n")
wf.write(" echo '*** FAIL ***'\n")
wf.write(" exit 1\n")
wf.write("fi\n")
wf.write("exit 0\n")
except IOError:
u.error("unable to open %s for writing" % f)
outfile = "err.benchrun.%s.%s.txt" % (repo, tag)
docmderrout("sh %s" % f, outfile)
def restore_mtimes():
"""Restore mtimes from tokenfile."""
u.verbose(1, "reading token file %s" % flag_tokenfile)
restored = 0
try:
with open(flag_tokenfile, "r") as tf:
pat = re.compile(r"^\s*(\S+)\s+(\d+)\s+(\d+)\s*$")
lines = tf.readlines()
for line in lines:
m = pat.match(line)
if not m:
u.error("pattern match failed on token file line %s" % line)
f = m.group(1)
st = os.stat(f)
u.verbose(2, "before restore for %s, at=%d "
"mt=%d" % (f, st.st_atime, st.st_mtime))
mt = int(m.group(2))
at = int(m.group(3))
newtimes = (at, mt)
os.utime(f, newtimes)
u.verbose(2, "restoring at=%d mt=%d for %s" % (at, mt, f))
st = os.stat(f)
u.verbose(2, "after restore for %s, at=%d "
"mt=%d" % (f, st.st_atime, st.st_mtime))
restored += 1
except IOError:
u.error("unable to read token file %s" % flag_tokenfile)
return restored
def download_blob(device, version, link):
"""Download a specific blob."""
# create location if needed
devdir = "%s/%d" % (flag_archive_dir, version)
if not os.path.isdir(devdir):
os.mkdir(devdir)
verdir = "%s/%s/%s" % (flag_archive_dir, version, device)
if not os.path.isdir(verdir):
os.mkdir(verdir)
# download file
base = os.path.basename(link)
path = "%s/%s" % (verdir, base)
if not os.path.exists(path):
print "... downloading %s => %s" % (link, path)
u.docmd("curl -L %s -o %s" % (link, path))
else:
print "... skipping %s blob %s (exists in archive already)" % (device, link)
# Update current version link
curlink = "%s/cur" % flag_archive_dir
if os.path.exists(curlink):
try:
os.remove(curlink)
except OSError as err:
u.error("unable to remove current version "
"link %s: %s" % (curlink, err))
try:
os.symlink("%d" % version, "%s/cur" % flag_archive_dir)
except OSError as err:
u.error("unable to update current version link %s" % curlink)
def parse_args():
"""Command line argument parsing."""
global flag_dryrun, flag_mode, flag_noclean
try:
optlist, args = getopt.getopt(sys.argv[1:], "hdDSF:")
except getopt.GetoptError as err:
# unrecognized option
usage(str(err))
for opt, arg in optlist:
if opt == "-d":
u.increment_verbosity()
elif opt == "-h":
usage()
elif opt == "-D":
flag_dryrun = True
elif opt == "-S":
flag_noclean = True
elif opt == "-F":
if os.path.exists(arg):
u.verbose(0, "adding %s to cmakefiles dict" % arg)
cmakefiles[arg] = 1
else:
u.error("-F arg %s doesn't seem to exist" % arg)
# Check for mode
if len(args) != 1:
usage("supply a single mode argument (either 'pre' or 'post')")
if args[0] == "pre":
flag_mode = "pre"
elif args[0] == "post":
flag_mode = "post"
else:
usage("unknown mode argument %s" % args[0])
def restore_mtimes():
"""Restore mtimes from tokenfile."""
u.verbose(1, "reading token file %s" % flag_tokenfile)
restored = 0
try:
with open(flag_tokenfile, "r") as tf:
pat = re.compile(r"^\s*(\S+)\s+(\d+)\s+(\d+)\s*$")
lines = tf.readlines()
for line in lines:
m = pat.match(line)
if not m:
u.error("pattern match failed on token file line %s" %
line)
f = m.group(1)
st = os.stat(f)
u.verbose(
2, "before restore for %s, at=%d "
"mt=%d" % (f, st.st_atime, st.st_mtime))
mt = int(m.group(2))
at = int(m.group(3))
newtimes = (at, mt)
os.utime(f, newtimes)
u.verbose(2, "restoring at=%d mt=%d for %s" % (at, mt, f))
st = os.stat(f)
u.verbose(
2, "after restore for %s, at=%d "
"mt=%d" % (f, st.st_atime, st.st_mtime))
restored += 1
except IOError:
u.error("unable to read token file %s" % flag_tokenfile)
return restored
def expand_die(lines):
"""Expand/explode specified DIE."""
# grab abbrev and tag from first line
m1 = bdiere.match(lines[0])
abbrev = 0
tag = "undefined"
if not m1:
u.error("can'r apply bdiere match to %s" % lines[0])
abbrev = m1.group(2)
tag = m1.group(3)
attrs = {}
# Process remaining lines
for line in lines[1:]:
m2 = indiere.match(line)
if not m2:
u.error("can't apply indiere match to %s" % line)
attr = m2.group(4)
val = m2.group(6).strip()
attrs[attr] = val
# return results
return abbrev, tag, attrs
def parse_args():
"""Command line argument parsing."""
global whichdev
try:
optlist, args = getopt.getopt(sys.argv[1:], "d")
except getopt.GetoptError as err:
# unrecognized option
usage(str(err))
for opt, _ in optlist:
if opt == "-d":
u.increment_verbosity()
if args:
usage("unrecognized arg")
# Check to make sure we can run adb
u.doscmd("which adb")
# Collect device flavor
lines = u.docmdlines("whichdevice.sh")
if len(lines) != 1:
u.error("unexpected output from whichdevice.sh")
whichdev = lines[0].strip()
u.verbose(1, "device: %s" % whichdev)
def dobuild():
"""Run the build and test."""
dochdir(flag_build_dir)
try:
with open("tmp.sh", "w") as wf:
wf.write("#/bin/sh\n")
scriptbody = """\
echo "make -j20 all 1> berr.txt 2>&1"
make -j20 all 1> berr.txt 2>&1
if [ $? != 0 ]; then
echo "** build failed, skipping tests"
emacs berr.txt &
exit 9
fi
echo "make -j20 check-go 1> terr.txt 2>&1"
make -j20 check-go 1> terr.txt 2>&1
if [ $? != 0 ]; then
echo "** test failed"
emacs berr.txt terr.txt &
exit 9
fi
echo "result: PASS"
emacs berr.txt terr.txt &"""
wf.write(scriptbody)
except IOError:
u.error("open failed for tmp.sh")
docmd("sh tmp.sh")
def emitdump(passname, funcname, looplab, lines):
"""Emit single dump for module/pass or fn/pass."""
u.verbose(2, "emitdump(%s,%s,%s,lines=%d)" % (passname, funcname, looplab, len(lines)))
if not lines:
return
tag = funcname
if not funcname:
tag = "__module__"
if looplab:
dump = "%s:L%s:%s" % (tag, looplab, passname)
else:
dump = "%s:%s" % (tag, passname)
dumpver = dumps[dump]
dumps[dump] += 1
dumpname = "%s:%d" % (dump, dumpver)
ofname = os.path.join(flag_outdir, dumpname)
try:
with open(ofname, "w") as wf:
for line in lines:
wf.write(line)
except IOError:
u.error("open failed for %s" % ofname)
u.verbose(1, "emitted dump %d of %d "
"lines to %s" % (dumpver, len(lines), ofname))
# book-keeping
dumpidx = len(alldumps)
alldumps.append(dumpname)
if funcname:
funcdumps[funcname].append(dumpidx)
return dumpname
def expand_die(lines):
"""Expand/explode specified DIE."""
# grab abbrev and tag from first line
m1 = bdiere.match(lines[0])
abbrev = 0
tag = "undefined"
if not m1:
u.error("can'r apply bdiere match to %s" % lines[0])
abbrev = m1.group(2)
tag = m1.group(3)
attrs = {}
# Process remaining lines
for line in lines[1:]:
m2 = indiere.match(line)
if not m2:
u.error("can't apply indiere match to %s" % line)
attr = m2.group(4)
val = m2.group(6).strip()
attrs[attr] = val
# return results
return abbrev, tag, attrs
def remove_from_file_if_present(mfile, todel):
"""Remove specified line from makefile if present."""
if not os.path.exists(mfile):
u.error("bad entry in munge makefile table-- %s "
"does not appear to exist" % mfile)
mfile_new = "%s.munged" % mfile
found = False
u.verbose(2, "examining %s in remove munge" % mfile)
with open(mfile, "r") as rf:
with open(mfile_new, "w") as wf:
lines = rf.readlines()
linecount = 0
for line in lines:
linecount += 1
sline = line.strip()
if sline == todel:
found = True
u.verbose(2, "found todel %s at line %d "
"in %s" % (todel, linecount, mfile))
continue
wf.write(line)
if found:
docmd("mv -f %s %s" % (mfile_new, mfile))
return True
return False
def bootstrap(repo, goroot, variant):
"""Build a go repo with a specific go root."""
f = "build.%s.sh" % repo
if os.path.exists(f):
rmfile(f)
try:
with open(f, "w") as wf:
wf.write("#!/bin/sh\n")
wf.write("set -x\n")
wf.write("export PATH=%s/bin:$PATH\n" % goroot)
wf.write("export GOROOT_BOOTSTRAP=%s\n" % goroot)
vcomp = variants[variant]["compiler"]
if vcomp == "gccgo" or vcomp == "gollvm":
wf.write("export LD_LIBRARY_PATH=%s/lib64\n" % goroot)
wf.write("cd %s/src\n" % repo)
wf.write("export GOOS=linux\n")
wf.write("export GOARCH=amd64\n")
wf.write("bash make.bash -v\n")
wf.write("if [ $? != 0 ]; then\n")
wf.write(" echo '*** FAIL ***'\n")
wf.write(" exit 1\n")
wf.write("fi\n")
wf.write("# Hack: copy bootstrap compiler into correct place\n")
wf.write("cd ../pkg\n")
wf.write("rm -f tool/linux_amd64/compile\n")
wf.write("mv bootstrap/bin/compile tool/linux_amd64/compile\n")
wf.write("if [ $? != 0 ]; then\n")
wf.write(" echo '*** FAIL copy ***'\n")
wf.write(" exit 1\n")
wf.write("fi\n")
except IOError:
u.error("unable to open %s for writing" % f)
outfile = "err.%s.txt" % repo
docmderrout("sh %s" % f, outfile)
docmd("touch %s/token.txt" % repo)
def do_setup_cmake(targdir):
"""Run cmake in each of the bin dirs."""
dochdir(ssdroot)
dochdir(targdir)
pool = None
if flag_parallel:
nworkers = len(cmake_flavors)
pool = multiprocessing.Pool(processes=nworkers)
results = []
for flav in cmake_flavors:
docmd("mkdir build.%s" % flav)
dochdir("build.%s" % flav)
emit_rebuild_scripts(flav, targdir)
cmake_cmd = emit_cmake_cmd_script(flav, targdir)
if flag_parallel and not flag_dryrun:
u.verbose(0, "...kicking off cmake for %s in parallel..." % flav)
builddir = "%s/%s/build.%s" % (ssdroot, targdir, flav)
r = pool.apply_async(run_cmake, [builddir, cmake_cmd])
results.append(r)
else:
doscmd(cmake_cmd)
dochdir("..")
nr = len(results)
rc = 0
for idx in range(0, nr):
r = results[idx]
u.verbose(1, "waiting on result %d" % idx)
res = r.get(timeout=600)
if res != 0:
rc = 1
if rc:
u.error("one or more cmake cmds failed")
def emitdump(passname, funcname, looplab, lines):
"""Emit single dump for module/pass or fn/pass."""
u.verbose(
2, "emitdump(%s,%s,%s,lines=%d)" %
(passname, funcname, looplab, len(lines)))
if not lines:
return
tag = funcname
if not funcname:
tag = "__module__"
if looplab:
dump = "%s:L%s:%s" % (tag, looplab, passname)
else:
dump = "%s:%s" % (tag, passname)
dumpver = dumps[dump]
dumps[dump] += 1
dumpname = "%s:%d" % (dump, dumpver)
ofname = os.path.join(flag_outdir, dumpname)
try:
with open(ofname, "w") as wf:
for line in lines:
wf.write(line)
except IOError:
u.error("open failed for %s" % ofname)
u.verbose(
1, "emitted dump %d of %d "
"lines to %s" % (dumpver, len(lines), ofname))
# book-keeping
dumpidx = len(alldumps)
alldumps.append(dumpname)
if funcname:
funcdumps[funcname].append(dumpidx)
return dumpname
def perform():
"""Main driver routine."""
lines = u.docmdlines("usb-devices")
dmatch = re.compile(r"^\s*T:\s*Bus\s*=\s*(\d+)\s+.*\s+Dev#=\s*(\d+).*$")
smatch = re.compile(r"^\s*S:\s*SerialNumber=(.*)$")
device = None
found = False
for line in lines:
m = dmatch.match(line)
if m:
p1 = int(m.group(1))
p2 = int(m.group(2))
device = "/dev/bus/usb/%03d/%03d" % (p1, p2)
u.verbose(1, "setting device: %s" % device)
continue
m = smatch.match(line)
if m:
ser = m.group(1)
if ser == flag_serial:
u.verbose(1, "matched serial, invoking reset")
issue_ioctl_to_device(device)
found = True
break
if not found:
u.error("unable to locate device with serial number %s" % flag_serial)
def mksnap_subcommand(volname, snapname):
"""Snapshot an existing BTRFS subvolume or snapshot."""
# Determine /ssd root
ssdroot = u.determine_btrfs_ssdroot(os.getcwd())
u.verbose(1, "ssdroot=%s" % ssdroot)
# Normalize snap name, volume name
volname = normalize(ssdroot, volname)
snapname = normalize(ssdroot, snapname)
# Existing volume should exist
oldvolume = "%s/%s" % (ssdroot, volname)
if not os.path.exists(oldvolume):
u.error("unable to locate existing subvolume %s" % oldvolume)
# Check to make sure the new snapshot doesn't already exist
newsnap = "%s/%s" % (ssdroot, snapname)
if os.path.exists(newsnap):
u.error("path %s already exists -- can't create" % newsnap)
# Here goes
u.docmd("sudo btrfs subvolume snapshot %s %s" % (oldvolume, newsnap))
# Repair ownership/permissions
repair(newsnap)
sys.stderr.write("... new snapshot %s created\n" % newsnap)
def perform():
"""Top level driver routine."""
try:
with open(flag_infile, "r") as rf:
process(rf)
except IOError:
u.error("open failed for %s" % flag_infile)
emitstats()
def perform():
"""Top level driver routine."""
try:
with open(flag_infile, "r") as rf:
process(rf)
except IOError:
u.error("open failed for %s" % flag_infile)
emitstats()
def dochdir(thedir):
"""Switch to dir."""
if flag_echo or flag_dryrun:
sys.stderr.write("cd " + thedir + "\n")
try:
os.chdir(thedir)
except OSError as err:
u.error("chdir failed: %s" % err)
def dochdir(thedir):
"""Switch to dir."""
if flag_echo or flag_dryrun:
sys.stderr.write("cd " + thedir + "\n")
try:
os.chdir(thedir)
except OSError as err:
u.error("chdir failed: %s" % err)
def select_cmake_type(flav):
"""Return cmake type for build."""
fd = cmake_flavors[flav]
if "cmflav" not in fd:
u.error("internal error: build flavor %s has no cmflav setting" % flav)
cmflav = fd["cmflav"]
if not cmflav:
cmflav = flag_cmake_type
return cmflav
def check_btrfs(rdir):
"""Check to make sure that 'rdir' is a BTRFS filesystem."""
outlines = u.docmdlines("stat -f --printf=%%T %s" % rdir)
if not outlines:
u.error("internal error-- could not determine FS type for dir %s" % rdir)
if outlines[0] != "btrfs":
u.error("FS type for %s is %s, not btrfs (can't "
"proceed)" % (rdir, outlines[0]))
def collect_file_size(afile):
"""Collect file size in bytes."""
if flag_dryrun:
return 1
try:
fsiz = os.path.getsize(afile)
except os.error as oe:
u.error("unable to collect file size for %s: %s" % (afile, oe))
return fsiz
def perform():
"""Top level driver routine."""
if os.path.exists("config.log"):
do_gccgo_clean()
elif os.path.exists("CMakeCache.txt"):
do_gollvm_clean()
else:
u.error("no 'config.log' or 'CMakeCache.txt' here -- "
"needs to be run in gccgo or gollvm build dir")
def perform():
"""Top level driver routine."""
if os.path.exists("config.log"):
do_gccgo_clean()
elif os.path.exists("CMakeCache.txt"):
do_gollvm_clean()
else:
u.error("no 'config.log' or 'CMakeCache.txt' here -- "
"needs to be run in gccgo or gollvm build dir")
def collect_file_size(afile):
"""Collect file size in bytes."""
if flag_dryrun:
return 1
try:
fsiz = os.path.getsize(afile)
except os.error as oe:
u.error("unable to collect file size for %s: %s" % (afile, oe))
return fsiz
def perform():
"""Main driver routine."""
#tf = tempfile.NamedTemporaryFile(mode="w", delete=True)
lines = u.docmdlines("git status -sb")
if not lines:
u.error("empty output from git status -sb")
brnreg = re.compile(r"^## (\S+)\.\.(\S+) \[ahead (\d+)\]\s*$")
m = brnreg.match(lines[0])
if not m:
u.error("can't pattern match output of git status -sb: %s" % lines[0])
branchname = m.group(1).strip(".")
commits = int(m.group(3))
u.verbose(1, "branch is: %s commits: %d" % (branchname, commits))
# Grab info on commits
lines = u.docmdlines("git log --oneline -%d" % commits)
if not lines:
u.error("empty output from 'git log --oneline'")
# Process commits in reverse order
firsthash = None
lasthash = None
creg = re.compile(r"^(\S+) (\S.+)$")
lines.reverse()
idx = 0
for cl in lines:
idx += 1
m = creg.match(cl)
if not m:
u.error("can't pattern match git log output: %s" % cl)
githash = m.group(1)
lasthash = githash
if not firsthash:
firsthash = githash
comment = m.group(2)
u.verbose(1, "processing hash %s comment %s" % (githash, comment))
process_commit(idx, branchname, githash, comment)
# Emit index file
n = len(files_emitted) + 1
fn = "/tmp/item%d.branch=%s.index.txt" % (n, branchname)
try:
outf = open(fn, "w")
except IOError as e:
u.error("unable to open %s: %s" % (fn, e.strerror))
outf.write("Files emitted:\n\n")
outf.write(" ".join(files_emitted))
outf.write("\n\nBranch log:\n\n")
u.verbose(1, "index diff cmd hashes: %s %s" % (firsthash, lasthash))
lines = u.docmdlines("git log --name-only -%d HEAD" % len(files_emitted))
for line in lines:
outf.write(line)
outf.write("\n")
outf.close()
u.verbose(0, "... index file emitted to %s\n" % fn)
def perform():
"""Main driver routine."""
#tf = tempfile.NamedTemporaryFile(mode="w", delete=True)
lines = u.docmdlines("git status -sb")
if not lines:
u.error("empty output from git status -sb")
brnreg = re.compile(r"^## (\S+)\.\.(\S+) \[ahead (\d+)\]\s*$")
m = brnreg.match(lines[0])
if not m:
u.error("can't pattern match output of git status -sb: %s" % lines[0])
branchname = m.group(1).strip(".")
commits = int(m.group(3))
u.verbose(1, "branch is: %s commits: %d" % (branchname, commits))
# Grab info on commits
lines = u.docmdlines("git log --oneline -%d" % commits)
if not lines:
u.error("empty output from 'git log --oneline'")
# Process commits in reverse order
firsthash = None
lasthash = None
creg = re.compile(r"^(\S+) (\S.+)$")
lines.reverse()
idx = 0
for cl in lines:
idx += 1
m = creg.match(cl)
if not m:
u.error("can't pattern match git log output: %s" % cl)
githash = m.group(1)
lasthash = githash
if not firsthash:
firsthash = githash
comment = m.group(2)
u.verbose(1, "processing hash %s comment %s" % (githash, comment))
process_commit(idx, branchname, githash, comment)
# Emit index file
n = len(files_emitted) + 1
fn = "/tmp/item%d.branch=%s.index.txt" % (n, branchname)
try:
outf = open(fn, "w")
except IOError as e:
u.error("unable to open %s: %s" % (fn, e.strerror))
outf.write("Files emitted:\n\n")
outf.write(" ".join(files_emitted))
outf.write("\n\nBranch log:\n\n")
u.verbose(1, "index diff cmd hashes: %s %s" % (firsthash, lasthash))
lines = u.docmdlines("git log --name-only -%d HEAD" % len(files_emitted))
for line in lines:
outf.write(line)
outf.write("\n")
outf.close()
u.verbose(0, "... index file emitted to %s\n" % fn)
def copy_file(srcf, destf):
"""Copy a file."""
if not os.path.exists(srcf):
u.error("unable to copy src file %s: doesn't exist" % srcf)
ddir = os.path.dirname(destf)
if not os.path.exists(ddir):
docmd("mkdir -p %s" % ddir)
if os.path.exists(destf):
os.chmod(destf, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH)
docmd("cp %s %s" % (srcf, destf))
def perform_pre():
"""Driver for 'pre' mode."""
u.verbose(1, "starting 'pre' mode")
find_cmakefiles()
if not cmakefiles:
u.error("no cmake files found -- nothing to do")
create_token()
u.verbose(1, "'pre' mode complete")
u.verbose(0, "... %d cmake files archived, state "
"written to %s" % (len(cmakefiles), flag_tokenfile))
def check_btrfs(rdir):
"""Check to make sure that 'rdir' is a BTRFS filesystem."""
outlines = u.docmdlines("stat -f --printf=%%T %s" % rdir)
if not outlines:
u.error("internal error-- could not determine FS type for dir %s" %
rdir)
if outlines[0] != "btrfs":
u.error("FS type for %s is %s, not btrfs (can't "
"proceed)" % (rdir, outlines[0]))
def copy_file(srcf, destf):
"""Copy a file."""
if not os.path.exists(srcf):
u.error("unable to copy src file %s: doesn't exist" % srcf)
ddir = os.path.dirname(destf)
if not os.path.exists(ddir):
docmd("mkdir -p %s" % ddir)
if os.path.exists(destf):
os.chmod(destf, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH)
docmd("cp %s %s" % (srcf, destf))
def do_check(subdir):
"""Make sure this repo has the master branch checked out."""
here = os.getcwd()
dn = os.path.dirname(subdir)
dochdir(dn)
lines = u.docmdlines("git rev-parse --abbrev-ref HEAD")
if lines[0] != "master":
u.error("error: repo at %s not on master "
"branch (on '%s' instead" % (dn, lines[0]))
dochdir(here)
def check_inputs():
"""Check that setup and inputs are legal."""
global flag_toolchain_repo
check_repo(flag_ndk_repo)
if flag_toolchain_repo:
check_repo(flag_toolchain_repo)
else:
flag_toolchain_repo = "%s/toolchain" % flag_ndk_repo
if not check_dir(flag_workdir):
u.error("can't access workdir %s" % flag_workdir)
def do_check(subdir):
"""Make sure this repo has the master branch checked out."""
here = os.getcwd()
dn = os.path.dirname(subdir)
dochdir(dn)
lines = u.docmdlines("git rev-parse --abbrev-ref HEAD")
if lines[0] != "master":
u.error("error: repo at %s not on master "
"branch (on '%s' instead" % (dn, lines[0]))
dochdir(here)
def parse_args():
"""Command line argument parsing."""
global flag_offset_to_find, flag_loadmodule, flag_objdump
try:
optlist, args = getopt.getopt(sys.argv[1:], "dm:x:T:")
except getopt.GetoptError as err:
# unrecognized option
usage(str(err))
if args:
usage("unknown extra args")
for opt, arg in optlist:
if opt == "-d":
u.increment_verbosity()
elif opt == "-T":
flag_objdump = arg
elif opt == "-x":
r = re.compile(r"^0x(\S+)$")
m = r.match(arg)
if not m:
usage(
"supply argument of the form 0x<hexliteral> to -x option")
hexdigits = m.group(1)
try:
v = int(hexdigits, 16)
except ValueError:
usage(
"supply argument of the form 0x<hexliteral> to -x option")
u.verbose(
1, "restricting output to compunit "
"containing DIE offset %x\n" % v)
flag_offset_to_find = v
elif opt == "-m":
if not os.path.exists(arg):
usage("argument '%s' to -m option does not exist" % arg)
flag_loadmodule = arg
# Make sure at least one function, loadmodule
if not flag_loadmodule:
usage("specify loadmodule -m")
if not flag_offset_to_find:
usage("specify offset to find with -x")
# Pick objdump variant based on Os.
if not flag_objdump:
lines = u.docmdlines("uname")
if not lines:
u.error("unable to run/interpret 'uname'")
if lines[0] == "Darwin":
flag_objdump = "gobjdump"
else:
flag_objdump = "objdump"
def normalize(ssdroot, volsnapname):
"""Remove initial /ssdroot, check for bad name."""
sr = ssdroot + "/"
vsn = volsnapname
if volsnapname.startswith(sr):
srl = len(sr)
vsn = volsnapname[srl:]
if vsn.find("/") != -1:
u.error("illegal volume or snapshot name %s "
"(must refer to top level dir)" % volsnapname)
return vsn
def checksum_file(f):
"""Return md5sum for contents of file."""
m = hashlib.md5()
try:
with open(f, "r") as rf:
lines = rf.readlines()
for line in lines:
m.update(line.encode("utf-8"))
except IOError:
u.error("open failed for %s" % f)
return m.hexdigest()
def perform_pre():
"""Driver for 'pre' mode."""
u.verbose(1, "starting 'pre' mode")
find_cmakefiles()
if not cmakefiles:
u.error("no cmake files found -- nothing to do")
create_token()
u.verbose(1, "'pre' mode complete")
u.verbose(
0, "... %d cmake files archived, state "
"written to %s" % (len(cmakefiles), flag_tokenfile))
def issue_ioctl_to_device(device):
"""Issue USB reset ioctl to device."""
try:
fd = open(device, "wb")
except IOError as e:
u.error("unable to open device %s: "
"%s" % (device, e.strerror))
u.verbose(1, "issuing USBDEVFS_RESET ioctl() to %s" % device)
fcntl.ioctl(fd, USBDEVFS_RESET, 0)
fd.close()
def checksum_file(f):
"""Return md5sum for contents of file."""
m = hashlib.md5()
try:
with open(f, "r") as rf:
lines = rf.readlines()
for line in lines:
m.update(line.encode("utf-8"))
except IOError:
u.error("open failed for %s" % f)
return m.hexdigest()
def normalize(ssdroot, volsnapname):
"""Remove initial /ssdroot, check for bad name."""
sr = ssdroot + "/"
vsn = volsnapname
if volsnapname.startswith(sr):
srl = len(sr)
vsn = volsnapname[srl:]
if vsn.find("/") != -1:
u.error("illegal volume or snapshot name %s "
"(must refer to top level dir)" % volsnapname)
return vsn
def form_golibargs(driver):
"""Form correct go library args."""
ddir = os.path.dirname(driver)
bdir = os.path.dirname(ddir)
cmd = "find %s/lib64 -name runtime.gox -print" % bdir
lines = u.docmdlines(cmd)
if not lines:
u.error("no output from %s -- bad gccgo install dir?" % cmd)
line = lines[0]
rdir = os.path.dirname(line)
u.verbose(1, "libdir is %s" % rdir)
return ["-L", rdir]
def dormdir(thedir):
"""Remove dir."""
if flag_echo:
sys.stderr.write("rm -r " + thedir + "\n")
if flag_dryrun:
return
if not os.path.exists(thedir):
return
try:
rmdir(thedir)
except OSError as err:
u.error("rmdir(%s) failed: %s" % (thedir, err))
def rmvolsnap(volsnapname, which):
"""Remove an existing btrfs snapshot or subvolume."""
# Determine /ssd root
ssdroot = u.determine_btrfs_ssdroot(os.getcwd())
u.verbose(1, "ssdroot=%s" % ssdroot)
# Normalize snap name
volsnapname = normalize(ssdroot, volsnapname)
# Check for existence
oldvolsnap = "%s/%s" % (ssdroot, volsnapname)
if not os.path.exists(oldvolsnap):
u.error("unable to locate existing %s %s" % (which, oldvolsnap))
# Determine whether there is a parent uuid
isvol = -1
showlines = u.docmdlines("sudo btrfs subvolume show %s" % oldvolsnap)
if not showlines:
u.error("unable to get subvolume info for %s" % oldvolsnap)
matcher = re.compile(r"^\s*Parent uuid\:\s+(\S+).*$")
for line in showlines:
m = matcher.match(line)
if m:
puid = m.group(1)
if puid == "-":
isvol = 1
else:
isvol = 0
u.verbose(2, "isvol=%d for %s" % (isvol, oldvolsnap))
if isvol == -1:
u.warning("unable to determine snapshot/subvolume status for %s" %
oldvolsnap)
elif isvol == 0:
if which == "volume":
u.warning("%s appears to be snapshot, not subvolume" % oldvolsnap)
else:
if which == "snapshot":
u.warning("%s appears to be subvolume, not snapshot" % oldvolsnap)
# Here goes
rc = u.docmdnf("sudo btrfs subvolume delete %s" % oldvolsnap)
if rc != 0:
# Couldn't delete the subvolume. Suggest running lsof
sys.stderr.write(
"** deletion failed -- trying to determine open file:\n")
sys.stderr.write(" lsof +D %s\n" % oldvolsnap)
u.docmdnf("lsof +D %s\n" % oldvolsnap)
exit(1)
sys.stderr.write("... %s %s deleted\n" % (which, oldvolsnap))
def create_token():
"""Deposit token file."""
u.verbose(0, "creating token %s" % flag_tokenfile)
try:
with open(flag_tokenfile, "w") as tf:
for f in sorted(cmakefiles):
st = os.stat(f)
u.verbose(2, "storing %s at=%d mt=%d "
"to token" % (f, st.st_atime, st.st_mtime))
tf.write("%s %d %d\n" % (f, st.st_atime, st.st_mtime))
except IOError:
u.error("unable to write to %s" % flag_tokenfile)
def form_golibargs(driver):
"""Form correct go library args."""
ddir = os.path.dirname(driver)
bdir = os.path.dirname(ddir)
cmd = "find %s/lib64 -name runtime.gox -print" % bdir
lines = u.docmdlines(cmd)
if not lines:
u.error("no output from %s -- bad gccgo install dir?" % cmd)
line = lines[0]
rdir = os.path.dirname(line)
u.verbose(1, "libdir is %s" % rdir)
return ["-L", rdir]
def dormdir(thedir):
"""Remove dir."""
if flag_echo:
sys.stderr.write("rm -r " + thedir + "\n")
if flag_dryrun:
return
if not os.path.exists(thedir):
return
try:
rmdir(thedir)
except OSError as err:
u.error("rmdir(%s) failed: %s" % (thedir, err))
def create_token():
"""Deposit token file."""
u.verbose(0, "creating token %s" % flag_tokenfile)
try:
with open(flag_tokenfile, "w") as tf:
for f in sorted(cmakefiles):
st = os.stat(f)
u.verbose(
2, "storing %s at=%d mt=%d "
"to token" % (f, st.st_atime, st.st_mtime))
tf.write("%s %d %d\n" % (f, st.st_atime, st.st_mtime))
except IOError:
u.error("unable to write to %s" % flag_tokenfile)
def create_or_check_link(src, dst):
"""Create or check a symbolic link."""
if not os.path.exists(dst):
u.verbose(0, "... creating link %s -> %s" % (dst, src))
os.symlink(src, dst)
else:
u.verbose(0, "... verifying link %s -> %s" % (dst, src))
if not os.path.islink(dst):
u.error("can't proceed: %s exists but is not a link" % dst)
ltarget = os.readlink(dst)
if ltarget != src:
u.error("can't proceed: %s exists but points to %s "
"instead of %s" % (dst, ltarget, src))
