def AddAndCommit(cfg, commit_title):
"""Add everything, and commit locally with |commit_title|"""
log("Creating local commit %s" % commit_title)
cfg.chdir_to_ffmpeg_home()
if IsWorkingDirectoryClean():
log("No files to commit to %s" % commit_title)
return
if call(["git", "add", "-u"]):
raise Exception("Could not add files")
if call(["git", "commit", "-m", commit_title]):
raise Exception("Could create commit")
def FetchMacSDK(robo_configuration):
"""Download the 10.10 MacOSX sdk."""
log("Installing Mac OSX sdk")
robo_configuration.chdir_to_chrome_src()
sdk_base = "build/win_files/Xcode.app"
if not os.path.exists(sdk_base):
os.makedirs(sdk_base)
os.chdir(sdk_base)
if call("gsutil.py cat gs://chrome-mac-sdk/toolchain-8E2002-3.tgz | tar xzvf -",
shell=True):
raise Exception("Cannot download and extract Mac SDK")
def InstallUbuntuPackage(robo_configuration, package):
"""Install |package|.
Args:
robo_configuration: current RoboConfiguration.
package: package name.
"""
log("Installing package %s" % package)
if robo_configuration.Call(["sudo", "apt-get", "install", package]):
raise Exception("Could not install %s" % package)
def AddAndCommit(cfg, commit_title):
"""Add everything, and commit locally with |commit_title|"""
log("Creating local commit %s" % commit_title)
cfg.chdir_to_ffmpeg_home();
if IsWorkingDirectoryClean():
log("No files to commit to %s" % commit_title)
return
# TODO: Ignore this file, for the "comment out autorename exception" thing.
if cfg.Call(["git", "add", "-u"]):
raise Exception("Could not add files")
if cfg.Call(["git", "commit", "-m", commit_title]):
raise Exception("Could create commit")
def EnsureUpstreamRemote(robo_configuration):
"""Make sure that the upstream remote is defined."""
remotes = subprocess.check_output(["git", "remote", "-v"]).split()
if "upstream" in remotes:
log("Upstream remote found")
return
log("Adding upstream remote")
if robo_configuration.Call([
"git", "remote", "add", "upstream",
"git://source.ffmpeg.org/ffmpeg.git"
]):
raise Exception("Failed to add git remote")
def MergeUpstreamToSushiBranchIfNeeded(cfg):
"""Start a merge if we've not started one before, or do nothing successfully
if the merge is complete. If it's half done, then get mad and exit."""
if IsMergeCommitOnThisBranch(cfg):
log("Merge commit already marked as complete")
return
# See if a merge is in progress. "git merge HEAD" will do nothing if it
# succeeds, but will fail if a merge is in progress.
if cfg.Call(["git", "merge", "HEAD"]):
raise UserInstructions(
"Merge is in progress -- please resolve conflicts and complete it.")
# There is no merge on this branch, and none is in progress. Start a merge.
MergeUpstreamToSushiBranch(cfg)
def IsUploadedForReview(robo_configuration):
"""Check if the local branch is already uploaded."""
robo_configuration.chdir_to_ffmpeg_home();
if not HasGerritIssueNumber(robo_configuration):
log("No Gerrit issue number exsts.")
return False
if not IsWorkingDirectoryClean():
log("Working directory is not clean -- commit changes and update CL");
return False
# Has been uploaded for review. Might or might not have been landed yet.
return True
def IsUploadedForReviewAndLanded(robo_configuration):
"""Check if the local sushi branch has been uploaded for review, and has also
been landed."""
robo_configuration.chdir_to_ffmpeg_home();
if not IsUploadedForReview(robo_configuration):
log("Is not uploaded for review")
return False
# See if origin/sushi and local/sushi are the same. This check by itself
# isn't sufficient, since it would return true any time the two are in sync.
diff = check_output(["git", "diff",
"origin/" + robo_configuration.sushi_branch_name(),
robo_configuration.sushi_branch_name()]).strip()
return not diff
def ImportFFmpegConfigsIntoChromium(robo_configuration):
"""Import all FFmpeg configs that have been built so far and build gn files.
Args:
robo_configuration: RoboConfiguration.
"""
robo_configuration.chdir_to_ffmpeg_home()
log("Copying FFmpeg configs")
if call(["./chromium/scripts/copy_config.sh"]):
raise Exception("FFmpeg copy_config.sh failed")
log("Generating GN config for all ffmpeg versions")
if call(["./chromium/scripts/generate_gn.py"]):
raise Exception("FFmpeg generate_gn.sh failed")
def EnsureGClientTargets(robo_configuration):
"""Make sure that we've got the right sdks if we're on a linux host."""
if not robo_configuration.host_operating_system() == "linux":
log("Not changing gclient target_os list on a non-linux host")
return
log("Checking gclient target_os list")
gclient_filename = os.path.join(robo_configuration.chrome_src(), "..",
".gclient")
# Ensure that target_os include 'android' and 'win'
scope = {}
try:
exec(FileRead(gclient_filename), scope)
except SyntaxError, e:
raise Exception("Unable to read %s" % gclient_filename)
def EnsureLLVMSymlinks(robo_configuration):
"""Create some symlinks to clang and friends, since that changes their
behavior somewhat."""
log("Creating symlinks to compiler tools if needed")
os.chdir(os.path.join(robo_configuration.chrome_src(), "third_party", "llvm-build", "Release+Asserts", "bin"))
def EnsureSymlink(source, link_name):
if not os.path.exists(link_name):
os.symlink(source, link_name)
# For windows.
EnsureSymlink("clang", "clang-cl")
EnsureSymlink("clang", "clang++")
EnsureSymlink("lld", "ld.lld")
EnsureSymlink("lld", "lld-link")
# For mac.
EnsureSymlink("lld", "ld64.lld")
def UpdateChromiumReadmeWithUpstream(robo_configuration):
"""Update the upstream info in README.chromium."""
log("Updating merge info in README.chromium")
merge_sha1 = FindUpstreamMergeParent(robo_configuration)
robo_configuration.chdir_to_ffmpeg_home()
with open("README.chromium", "r+") as f:
readme = f.read()
last_upstream_merge = "Last Upstream Merge:"
merge_date = check_output([
"git", "log", "-1", "--date=format:%b %d %Y", "--format=%cd",
merge_sha1
])
readme = re.sub(r"(Last Upstream Merge:).*\n",
r"\1 %s, %s" % (merge_sha1, merge_date), readme)
with open("README.chromium", "w") as f:
f.write(readme)
def ImportFFmpegConfigsIntoChromium(robo_configuration, write_git_file=False):
"""Import all FFmpeg configs that have been built so far and build gn files.
Args:
robo_configuration: RoboConfiguration.
write_git_file: if true, then we'll ask generate_gn.py to write a script
with the appropriate git commands to add / rm autorenames.
"""
robo_configuration.chdir_to_ffmpeg_home()
log("Copying FFmpeg configs")
if call(["./chromium/scripts/copy_config.sh"]):
raise Exception("FFmpeg copy_config.sh failed")
log("Generating GN config for all ffmpeg versions")
generate_cmd = ["./chromium/scripts/generate_gn.py"]
if write_git_file:
generate_cmd += ["-i", robo_configuration.autorename_git_file()]
if call(generate_cmd):
raise Exception("FFmpeg generate_gn.sh failed")
def ConfigureAndBuildFFmpeg(robo_configuration, platform, architecture):
"""Run FFmpeg's configure script, and build ffmpeg.
Args:
robo_configuration: RoboConfiguration.
platform: platform name (e.g., "linux")
architecture: (optional) arch name (e.g., "ia32"). If omitted, then we
build all of them for |platform|.
"""
log("Generating FFmpeg config and building for %s %s" %
(platform, architecture))
log("Starting FFmpeg build for %s %s" % (platform, architecture))
robo_configuration.chdir_to_ffmpeg_home()
command = ["./chromium/scripts/build_ffmpeg.py", platform]
if architecture:
command.append(architecture)
if call(command):
raise Exception("FFmpeg build failed for %s %s" %
(platform, architecture))
def TryFakeDepsRoll(robo_configuration):
"""Start a deps roll against the sushi branch, and -1 it."""
log("Considering starting a fake deps roll")
# Make sure that we've landed the sushi commits. Note that this can happen if
# somebody re-runs robosushi after we upload the commits to Gerrit, but before
# they've been reviewed and landed. This way, we provide a meaningful error.
if not IsUploadedForReviewAndLanded(robo_configuration):
raise Exception("Cannot start a fake deps roll until gerrit review lands!")
robo_configuration.chdir_to_ffmpeg_home();
sha1 = check_output("git", "show", "-1", "--format=%P").strip()
if not sha1:
raise Exception("Cannot get sha1 of HEAD for fakes dep roll")
robo_configuration.chdir_to_chrome_src()
# TODO: make sure that there's not a deps roll in progress, else we'll keep
# doing this every time we're run.
# TODO: get mad otherwise.
check_output(["roll-deps.py", "third_party/ffmpeg", sha1])
def FetchMacSDKs(robo_configuration):
"""Download the 10.10 MacOSX sdk."""
log("Installing Mac OSX sdk")
robo_configuration.chdir_to_chrome_src()
sdk_base = "build/win_files/Xcode.app"
if not os.path.exists(sdk_base):
os.makedirs(sdk_base)
os.chdir(sdk_base)
if robo_configuration.Call(
"gsutil.py cat gs://chrome-mac-sdk/toolchain-8E2002-3.tgz | tar xzvf -",
shell=True):
raise Exception("Cannot download and extract Mac SDK")
# TODO: Once the 11.0 SDK is out of beta, it should be used for both
# arm64 and intel builds.
log("Installing Mac OSX 11.0 beta sdk for arm64")
robo_configuration.chdir_to_chrome_src()
if robo_configuration.Call(
"build/mac_toolchain.py --xcode-version xcode_12_beta",
shell=True):
raise Exception("Cannot download and extract Mac beta SDK")
def EnsureASANDirWorks(robo_configuration):
"""Create the asan out dir and config for ninja builds.
Args:
robo_configuration: current RoboConfiguration.
"""
robo_configuration.chdir_to_chrome_src()
directory_name = robo_configuration.absolute_asan_directory()
if os.path.exists(directory_name):
return
# Dir doesn't exist, so make it and generate the gn files. Note that we
# have no idea what state the ffmpeg config is, but that's okay. gn will
# re-build it when we run ninja later (after importing the ffmpeg config)
# if it's changed.
log("Creating asan build dir %s" % directory_name)
os.mkdir(directory_name)
# TODO(liberato): ffmpeg_branding Chrome vs ChromeOS. also add arch
# flags, etc. Step 28.iii, and elsewhere.
opts = ("is_debug=false", "is_clang=true", "proprietary_codecs=true",
"media_use_libvpx=true", "media_use_ffmpeg=true",
'ffmpeg_branding="Chrome"', "use_goma=true", "is_asan=true",
"dcheck_always_on=true")
print opts
with open(os.path.join(directory_name, "args.gn"), "w") as f:
for opt in opts:
print opt
f.write("%s\n" % opt)
# Ask gn to generate build files.
log("Running gn on %s" % directory_name)
if robo_configuration.Call(
["gn", "gen",
robo_configuration.relative_asan_directory()]):
raise Exception("Unable to gn gen %s" %
robo_configuration.local_asan_directory())
def EnsureChromiumNasm(robo_configuration):
"""Make sure that chromium's nasm is built, so we can use it. apt-get's is
too old."""
os.chdir(robo_configuration.chrome_src())
# nasm in the LLVM bin directory that we already added to $PATH. Note that we
# put it there so that configure can find is as "nasm", rather than us having
# to give it the full path. I think the full path would affect the real
# build. That's not good.
llvm_nasm_path = os.path.join(robo_configuration.llvm_path(), "nasm")
if os.path.exists(llvm_nasm_path):
log("nasm already installed in llvm bin directory")
return
# Make sure nasm is built, and copy it to the llvm bin directory.
chromium_nasm_path = os.path.join(
robo_configuration.absolute_asan_directory(), "nasm")
if not os.path.exists(chromium_nasm_path):
log("Building Chromium's nasm")
if robo_configuration.Call([
"ninja", "-j5000", "-C",
robo_configuration.relative_asan_directory(),
"third_party/nasm"
]):
raise Exception("Failed to build nasm")
# Verify that it exists now, for sanity.
if not os.path.exists(chromium_nasm_path):
raise Exception("Failed to find nasm even after building it")
# Copy it
log("Copying Chromium's nasm to llvm bin directory")
if shutil.copy(chromium_nasm_path, llvm_nasm_path):
raise Exception("Could not copy %s into %s" %
(chromium_nasm_path, llvm_nasm_path))
def CreateAndCheckoutDatedSushiBranch(cfg):
"""Create a dated branch from origin/master and check it out."""
now = datetime.now()
branch_name=cfg.sushi_branch_prefix() + now.strftime("%Y-%m-%d-%H-%M-%S")
log("Creating dated branch %s" % branch_name)
# Fetch the latest from origin
if cfg.Call(["git", "fetch", "origin"]):
raise Exception("Could not fetch from origin")
# Create the named branch
# Note that we definitely do not want this branch to track origin/master; that
# would eventually cause 'git cl upload' to push the merge commit, assuming
# that the merge commit is pushed to origin/sushi-branch. One might argue
# that we should push the merge to origin/master, which would make this okay.
# For now, we leave the branch untracked to make sure that the user doesn't
# accidentally do the wrong thing. I think that with an automatic deps roll,
# we'll want to stage things on origin/sushi-branch.
#
# We don't want to push anything to origin yet, though, just to keep from
# making a bunch of sushi branches. We can do it later just as easily.
if cfg.Call(["git",
"branch",
"--no-track",
branch_name,
"origin/master"]):
raise Exception("Could not create branch")
# NOTE: we could push the remote branch back to origin and start tracking it
# now, and not worry about tracking later. However, until the scripts
# actually work, i don't want to push a bunch of branches to origin.
# Check out the branch. On failure, delete the new branch.
if cfg.Call(["git", "checkout", branch_name]):
cfg.Call(["git", "branch", "-D", branch_name])
raise Exception("Could not checkout branch")
cfg.SetBranchName(branch_name)
def PushToOriginWithoutReviewAndTrackIfNeeded(cfg):
"""Push the local branch to origin/ if we haven't yet."""
cfg.chdir_to_ffmpeg_home();
# If the tracking branch is unset, then assume that we haven't done this yet.
if IsTrackingBranchSet(cfg):
log("Already have local tracking branch")
return
log("Pushing merge to origin without review")
cfg.Call(["git", "push", "origin", cfg.sushi_branch_name()])
log("Setting tracking branch")
cfg.Call(["git", "branch", "--set-upstream-to=origin/%s" %
cfg.sushi_branch_name()])
# Sanity check. We don't want to start pushing other commits without review.
if not IsTrackingBranchSet(cfg):
raise Exception("Tracking branch is not set, but I just set it!")
def RunSteps(cfg, step_names):
for step_name in step_names:
if not step_name in steps:
raise Exception("Unknown step %s" % step_name)
log("Step %s" % step_name)
step = steps[step_name]
try:
if "pre_fn" in step:
raise Exception("pre_fn not supported yet")
if "skip_fn" in step:
if step["skip_fn"](cfg):
log("Step %s not needed, skipping" % step_name)
continue
step["do_fn"](cfg)
except Exception, e:
log("Step %s failed" % step_name)
raise e
def BuildAndRunChromeTargetASAN(robo_configuration, target, platform,
architecture):
"""Build and run a Chromium asan target.
Args:
robo_configuration: RoboConfiguration.
target: chrome target to build (e.g., "media_unittests")
platform: platform to build it for, which should probably be the host's.
architecture: arch to build it for (e.g., "x64").
"""
log("Building and running %s" % target)
BuildChromeTargetASAN(robo_configuration, target, platform, architecture)
# TODO: we should be smarter about running things on android, for example.
log("Running %s" % target)
robo_configuration.chdir_to_chrome_src()
if call([os.path.join(robo_configuration.absolute_asan_directory(), target)]):
raise Exception("%s didn't complete successfully" % target)
log("%s ran successfully" % target)
def WritePatchesReadme(cfg):
"""Write the chromium patches file."""
log("Generating CHROMIUM.patches file")
cfg.chdir_to_ffmpeg_home()
with open(os.path.join("chromium", "patches", "README"), "w+") as f:
find_patches.write_patches_file("HEAD", f)
def EnsureGClientTargets(robo_configuration):
"""Make sure that we've got the right sdks if we're on a linux host."""
if not robo_configuration.host_operating_system() == "linux":
log("Not changing gclient target_os list on a non-linux host")
return
log("Checking gclient target_os list")
gclient_filename = os.path.join(robo_configuration.chrome_src(), "..",
".gclient")
# Ensure that we've got our target_os line
for line in open(gclient_filename, "r"):
if "target_os" in line:
if (not "'android'" in line) or (not "'win'" in line):
log("Missing 'android' and/or 'win' in target_os, which goes at the")
log("end of .gclient, OUTSIDE of the solutions = [] section")
log("Example line:")
log("target_os = [ 'android', 'win' ]")
raise Exception("Please add 'android' and 'win' to target_os in %s" %
gclient_filename)
break
# Sync regardless of whether we changed the config.
log("Running gclient sync")
robo_configuration.chdir_to_chrome_src()
if call(["gclient", "sync"]):
raise Exception("gclient sync failed")
if not robo_configuration.host_operating_system() == "linux":
log("Not changing gclient target_os list on a non-linux host")
return
log("Checking gclient target_os list")
gclient_filename = os.path.join(robo_configuration.chrome_src(), "..",
".gclient")
# Ensure that target_os include 'android' and 'win'
scope = {}
try:
exec(FileRead(gclient_filename), scope)
except SyntaxError, e:
raise Exception("Unable to read %s" % gclient_filename)
if 'target_os' not in scope:
log("Missing 'target_os', which goes at the end of .gclient,")
log("OUTSIDE of the solutions = [] section")
log("Example line:")
log("target_os = [ 'android', 'win' ]")
raise UserInstructions("Please add target_os to %s" % gclient_filename)
if ('android' not in scope['target_os']) or ('win'
not in scope['target_os']):
log("Missing 'android' and/or 'win' in target_os, which goes at the")
log("end of .gclient, OUTSIDE of the solutions = [] section")
log("Example line:")
log("target_os = [ 'android', 'win' ]")
raise UserInstructions(
"Please add 'android' and 'win' to target_os in %s" %
gclient_filename)
def main(argv):
robo_configuration = robo_lib.RoboConfiguration()
robo_configuration.chdir_to_ffmpeg_home()
parsed, remaining = getopt.getopt(argv, "",
["setup", "test", "build", "auto-merge"])
for opt, arg in parsed:
if opt == "--setup":
robo_setup.InstallPrereqs(robo_configuration)
robo_setup.EnsureToolchains(robo_configuration)
robo_setup.EnsureASANDirWorks(robo_configuration)
robo_setup.EnsureChromiumNasm(robo_configuration)
elif opt == "--test":
robo_build.BuildAndImportFFmpegConfigForHost(robo_configuration)
robo_build.RunTests(robo_configuration)
elif opt == "--build":
# Unconditionally build all the configs and import them.
robo_build.BuildAndImportAllFFmpegConfigs(robo_configuration)
elif opt == "--auto-merge":
# Start a branch (if needed), merge (if needed), and try to verify it.
# TODO: Verify that the working directory is clean.
robo_branch.CreateAndCheckoutDatedSushiBranchIfNeeded(
robo_configuration)
robo_branch.MergeUpstreamToSushiBranchIfNeeded(robo_configuration)
# We want to push the merge and make the local branch track it, so that
# future 'git cl upload's don't try to review the merge commit, and spam
# the ffmpeg committers.
robo_branch.PushToOriginWithoutReviewAndTrackIfNeeded(
robo_configuration)
# Try to get everything to build if we haven't committed the configs yet.
# Note that the only time we need to do this again is if some change makes
# different files added / deleted to the build, or if ffmpeg configure
# changes. We don't need to do this if you just edit ffmpeg sources;
# those will be built with the tests if they've changed since last time.
#
# So, if you're just editing ffmpeg sources to get tests to pass, then you
# probably don't need to do this step again.
#
# TODO: Add a way to override this. I guess just edit out the config
# commit with a rebase for now.
if robo_branch.IsCommitOnThisBranch(
robo_configuration, robo_configuration.gn_commit_title()):
log("Skipping config build since already committed")
else:
robo_build.BuildAndImportAllFFmpegConfigs(robo_configuration)
# Run sanity checks on the merge before we commit.
robo_branch.CheckMerge(robo_configuration)
# Write the config changes to help the reviewer.
robo_branch.WriteConfigChangesFile(robo_configuration)
# TODO(liberato): Add the 'autodetect' regex too.
# Handle autorenames last, so that we don't stage things and then fail.
# While it's probably okay, it's nicer if we don't.
robo_branch.HandleAutorename(robo_configuration)
robo_branch.AddAndCommit(robo_configuration,
robo_configuration.gn_commit_title())
# Update the patches file.
if robo_branch.IsCommitOnThisBranch(
robo_configuration,
robo_configuration.patches_commit_title()):
log("Skipping patches file since already committed")
else:
robo_branch.WritePatchesReadme(robo_configuration)
robo_branch.AddAndCommit(
robo_configuration,
robo_configuration.patches_commit_title())
# Run the tests. Note that this will re-run ninja from chromium/src,
# which will rebuild any changed ffmpeg sources as it normally would.
robo_build.RunTests(robo_configuration)
# TODO: Start a fake deps roll. To do this, we would:
# Create new remote branch from the current remote sushi branch.
# Create and check out a new local branch at the current local branch.
# Make the new local branch track the new remote branch.
# Push to origin/new remote branch.
# Start a fake deps roll CL that runs the *san bots.
# Switch back to original local branch.
# For extra points, include a pointer to the fake deps roll CL in the
# local branch, so that when it's pushed for review, it'll point the
# reviewer at it.
# TODO: git cl upload for review.
else:
raise Exception("Unknown option '%s'" % opt)