def iter_notice_files():
"""Yields (library_name, notice_file_name) tuples."""
root = os.path.join(project_dir, "Assets/ThirdParty")
if not os.path.exists(root):
raise BuildFailed("Cannot generate NOTICE: missing %s" % root)
for r, _, fs in os.walk(root):
for f in fs:
if f.lower() in (
"notice",
"notice.txt",
"notice.tiltbrush",
"notice.md",
):
yield (os.path.basename(r), os.path.join(r, f))
root = os.path.join(project_dir, "Assets/ThirdParty/NuGet/Packages")
if not os.path.exists(root):
raise BuildFailed("Cannot generate NOTICE: missing %s" % root)
for r, _, fs in os.walk(root):
for f in fs:
if f.lower() in ("notice", "notice.md", "notice.txt"):
m = re.match(r"\D+", os.path.basename(r))
if m:
name = m.group(0).rstrip(".")
if name[-2:] == ".v" or name[-2:] == ".V":
name = name[:-2]
yield (name, os.path.join(r, f))
def get_unity_exe(version, lenient=True):
"""Returns a Unity executable of the same major version.
version - a (major, minor, point) tuple. Strings.
lenient - if True, allow the micro version to be higher.
"""
exes = sorted(iter_editors_and_versions(), reverse=True)
if len(exes) == 0:
raise BuildFailed("Cannot find any Unity versions (want %s)" % (version,))
for (found_exe, found_version) in exes:
if found_version == version:
return found_exe
if lenient:
# Compatible is defined as same major and minor version
compatible = [(exe, ver) for (exe, ver) in exes if ver[0:2] == version[0:2]]
if len(compatible) > 0:
def int_version(version):
(major, minor, micro) = version
return (int(major), int(minor), int(micro))
def by_int_version(xxx_todo_changeme):
(exe, ver) = xxx_todo_changeme
return (int_version(ver), exe)
found_exe, found_version = max(compatible, key=by_int_version)
if int_version(found_version) >= int_version(version):
return found_exe
raise BuildFailed("Cannot find desired Unity version (want %s)" % (version,))
def analyze_unity_failure(exitcode, log):
"""Raise BuildFailed with as much information about the failure as possible."""
# Build exceptions look like this:
# BuildFailedException: <<Build sanity checks failed:
# This is a dummy error>>
# at BuildTiltBrush.DoBuild (BuildOptions options, BuildTarget target, System.String location, SdkMode vrSdk, Boolean isExperimental, System.String stamp) [0x0026a] in C:\src\tb\Assets\Editor\BuildTiltBrush.cs:430
# at BuildTiltBrush.CommandLine () [0x001de] in C:\src\tb\Assets\Editor\BuildTiltBrush.cs:259
build_failed_pat = re.compile(
r"""BuildFailedException:\ <<(?P<description>.*?)>>
(?P<traceback> (\n\ \ at\ [^\n]+)* )""",
re.DOTALL | re.MULTILINE | re.VERBOSE,
)
m = build_failed_pat.search(log)
if m is not None:
raise BuildFailed(
"C# raised BuildFailedException\n%s\n| ---\n%s"
% (
indent("| ", m.group("traceback").strip()),
indent("| ", m.group("description").strip()),
)
)
internal_error_pat = re.compile(
r"^executeMethod method (?P<methodname>.*) threw exception\.", re.MULTILINE
)
m = internal_error_pat.search(log)
if m is not None:
exception_pat = re.compile(
r"^[A-Z][A-Za-z0-9]+(Exception|Error):", re.MULTILINE
)
start = search_backwards(log, m.start(0), m.start(0) - 1024, exception_pat)
end = m.end(0)
suspicious_portion = log[start:end]
raise BuildFailed(
"""Build script '%s' had an internal error.
Suspect log portion:
%s"""
% (m.group("methodname"), indent("| ", suspicious_portion))
)
# Check for BuildTiltBrush.Die()
btb_die_pat = re.compile(
r"_btb_ Abort <<(?P<description>.*?)>>", re.DOTALL | re.MULTILINE
)
m = btb_die_pat.search(log)
if m is not None:
raise BuildFailed("C# called Die %s '%s'" % (exitcode, m.group("description")))
if exitcode is None:
raise BuildFailed("Unity build seems to have been terminated prematurely")
raise BuildFailed(
"""Unity build failed with exit code %s but no errors seen
This probably means the project is already open in Unity"""
% exitcode
)
def sanity_check_build(build_dir):
# We've had issues with Unity dying(?) or exiting(?) before emitting an exe
exes = []
for pat in ("*.app", "*.exe", "*.apk"):
exes.extend(glob.glob(os.path.join(build_dir, pat)))
if len(exes) == 0:
raise BuildFailed("Cannot find any executables in %s" % build_dir)
def check_compile_output(log):
"""Raises BuildFailed if compile errors are found.
Spews to stderr if compile warnings are found."""
dcts = list(iter_compiler_output(log))
compiler_output = "\n".join(
stuff.strip() for dct in dcts for stuff in [dct["stderr"], dct["stdout"]]
)
if any(dct["compilationhadfailure"] for dct in dcts):
# Mono puts it in stderr; Roslyn puts it in stdout.
# But! Unity 2018 also gives us a good build report, so we might be able to
# get the compiler failures from the build report instead of this ugly parsing
# through Unity's log file.
raise BuildFailed("Compile\n%s" % indent("| ", compiler_output))
if compiler_output != "":
print("Compile warnings:\n%s" % indent("| ", compiler_output), file=sys.stderr)
def main(
args=None,
): # pylint: disable=too-many-statements,too-many-branches,too-many-locals
unitybuild.utils.msys_control_c_workaround()
if sys.platform == "cygwin":
raise UserError("Running under cygwin python is not supported.")
args = parse_args(args)
if args.push:
num = len(args.platforms) * len(args.vrsdks) * len(args.configs)
if num != 1:
raise UserError("Must specify exactly one build to push")
vcs = vcs_create()
project_dir = find_project_dir()
print("Project dir:", os.path.normpath(project_dir))
if args.jenkins:
# Jenkins does not allow building outside of the source tree.
build_dir = os.path.normpath(os.path.join(project_dir, "Builds"))
else:
# Local build setup.
build_dir = os.path.normpath(os.path.join(project_dir, "..", "Builds"))
# TODO(pld): maybe faster to call CommandLine() multiple times in the same
# Unity rather than to start up Unity multiple times. OTOH it requires faith
# in Unity's stability.
try:
tmp_dir = None
try:
revision = vcs.get_build_stamp(project_dir)
except LookupError as e:
print("WARN: no build stamp (%s). Continue?" % (e,))
if not input("(y/n) > ").strip().lower().startswith("y"):
raise UserError("Aborting: no stamp") from e
revision = "nostamp"
create_notice_file(project_dir)
for (platform, vrsdk, config) in iter_builds(args):
stamp = revision + ("-exp" if args.experimental else "")
print(
"Building %s %s %s exp:%d signed:%d il2cpp:%d"
% (
platform,
vrsdk,
config,
args.experimental,
args.for_distribution,
args.il2cpp,
)
)
sdk = vrsdk
if sdk == "Oculus" and platform == "Android":
sdk = "OculusMobile"
dirname = "%s_%s_%s%s%s%s%s_FromCli" % (
sdk,
"Release",
EXE_BASE_NAME,
"_Experimental" if args.experimental else "",
"_Il2cpp" if args.il2cpp else "",
"", # GuiAutoProfile
"_signed" if args.for_distribution and platform != "Windows" else "",
)
tmp_dir = os.path.join(build_dir, "tmp_" + dirname)
output_dir = os.path.join(build_dir, dirname)
if args.for_distribution and platform == "Android" and sys.stdin.isatty():
try:
maybe_prompt_and_set_version_code(project_dir)
except Exception as e: # pylint: disable=broad-except
print("Error prompting for version code: %s" % e)
tmp_dir = build(
stamp,
tmp_dir,
project_dir,
EXE_BASE_NAME,
experimental=args.experimental,
platform=platform,
il2cpp=args.il2cpp,
vrsdk=vrsdk,
config=config,
for_distribution=args.for_distribution,
is_jenkins=args.jenkins,
)
output_dir = finalize_build(tmp_dir, output_dir)
sanity_check_build(output_dir)
if args.for_distribution and platform == "Android":
set_android_version_code(project_dir, "increment")
if args.for_distribution and vrsdk == "Oculus":
# .pdb files violate VRC.PC.Security.3 and ovr-platform-utils rejects the submission
to_remove = []
for (r, _, fs) in os.walk(output_dir):
for f in fs:
if f.endswith(".pdb"):
to_remove.append(os.path.join(r, f))
if to_remove:
print(
"Removing from submission:\n%s"
% ("\n".join(os.path.relpath(f, output_dir) for f in to_remove))
)
list(map(os.unlink, to_remove))
if platform == "iOS":
# TODO: for iOS, invoke xcode to create ipa. E.g.:
# $ cd tmp_dir/TiltBrush
# $ xcodebuild -scheme Unity-iPhone archive -archivePath ARCHIVE_DIR
# $ xcodebuild -exportArchive -exportFormat ipa -archivePath ARCHIVE_DIR -exportPath IPA
print(
"iOS build must be completed from Xcode (%s)"
% (
os.path.join(
output_dir, EXE_BASE_NAME, "Unity-iPhone.xcodeproj"
)
)
)
continue
if args.push:
with open(os.path.join(output_dir, "build_stamp.txt")) as inf:
embedded_stamp = inf.read().strip()
description = "%s %s | %s@%s" % (
config,
embedded_stamp,
getpass.getuser(),
platform_node(),
)
if args.branch is not None:
description += " to %s" % args.branch
if vrsdk == "SteamVR":
if platform not in ("Windows",):
raise BuildFailed(
"Unsupported platform for push to Steam: %s" % platform
)
unitybuild.push.push_open_brush_to_steam(
output_dir,
description,
args.user or "tiltbrush_build",
args.branch,
)
elif vrsdk == "Oculus":
if platform not in ("Windows", "Android"):
raise BuildFailed(
"Unsupported platform for push to Oculus: %s" % platform
)
release_channel = args.branch
if release_channel is None:
release_channel = "ALPHA"
print(
(
"No release channel specified for Oculus: using %s"
% release_channel
)
)
unitybuild.push.push_open_brush_to_oculus(
output_dir, release_channel, description
)
except BadVersionCode as e:
if isinstance(e, BadVersionCode):
set_android_version_code(project_dir, e.desired_version_code)
print(
(
"\n\nVersion code has been auto-updated to %s.\nPlease retry your build."
% e.desired_version_code
)
)
if tmp_dir:
print("\nSee %s" % os.path.join(tmp_dir, "build_log.txt"))
sys.exit(1)
except Error as e:
print("\n%s: %s" % ("ERROR", e))
if tmp_dir:
print("\nSee %s" % os.path.join(tmp_dir, "build_log.txt"))
sys.exit(1)
except KeyboardInterrupt:
print("Aborted.")
sys.exit(2)
def build(
stamp,
output_dir,
project_dir,
exe_base_name, # pylint: disable=too-many-statements,too-many-branches,too-many-locals,too-many-arguments
experimental,
platform,
il2cpp,
vrsdk,
config,
for_distribution,
is_jenkins,
):
"""Create a build of Tilt Brush.
Pass:
stamp - string describing the version+build; will be embedded into the build somehow.
output_dir - desired output directory name
project_dir - directory name
project_name - name of the executable to create (sans extension)
experimental - boolean
platform - one of (Windows, OSX, Linux, Android, iOS)
il2cpp - boolean
vrsdk - Config.SdkMode; valid values are (Oculus, SteamVR, Monoscopic)
config - one of (Debug, Release)
for_distribution - boolean. Enables android signing, version code bump, removal of pdb files.
is_jenkins - boolean; used to customize stdout logging
Returns:
the actual output directory used
"""
def get_exe_name(platform, exe_base_name):
# This is a manually maintained duplicate of App.cs
if "Windows" in platform:
return "%s.exe" % exe_base_name
if "OSX" in platform:
return "%s.app" % exe_base_name
if "Linux" in platform:
return "%s" % exe_base_name
if "Android" in platform:
return "com.%s.%s.apk" % (VENDOR_NAME, exe_base_name)
if "iOS" in platform:
return "%s" % exe_base_name
raise InternalError("Don't know executable name for %s" % platform)
try:
unitybuild.utils.destroy(output_dir)
except Exception as e: # pylint: disable=broad-except
print("WARN: could not use %s: %s" % (output_dir, e))
output_dir = make_unused_directory_name(output_dir)
print("WARN: using %s intead" % output_dir)
unitybuild.utils.destroy(output_dir)
os.makedirs(output_dir)
logfile = os.path.join(output_dir, "build_log.txt")
exe_name = os.path.join(output_dir, get_exe_name(platform, exe_base_name))
cmd_env = os.environ.copy()
cmdline = [
get_unity_exe(get_project_unity_version(project_dir), lenient=is_jenkins),
"-logFile",
logfile,
"-batchmode",
# '-nographics', Might be needed on OSX if running w/o window server?
"-projectPath",
project_dir,
"-executeMethod",
"BuildTiltBrush.CommandLine",
"-btb-target",
PLATFORM_TO_UNITYTARGET[platform],
"-btb-out",
exe_name,
"-btb-display",
vrsdk,
]
if experimental:
cmdline.append("-btb-experimental")
if il2cpp:
cmdline.append("-btb-il2cpp")
# list of tuples:
# - the name of the credential in the environment (for Jenkins)
# - the name of the credential in the keystore (for interactive use)
required_credentials = []
if for_distribution and platform == "Android":
if vrsdk != "Oculus":
raise BuildFailed("Signing is currently only implemented for Oculus Quest")
keystore = os.path.abspath(
os.path.join(project_dir, "Support/Keystores/TiltBrush.keystore")
)
keystore = keystore.replace("/", "\\")
if not os.path.exists(keystore):
raise BuildFailed("To sign you need %s.\n" % keystore)
cmdline.extend(
[
"-btb-keystore-name",
keystore,
"-btb-keyalias-name",
"oculusquest",
]
)
required_credentials.extend(
[
("BTB_KEYSTORE_PASS", "Tilt Brush keystore password"),
("BTB_KEYALIAS_PASS", "Tilt Brush Oculus Quest signing key password"),
]
)
cmdline.extend(["-btb-stamp", stamp])
if config == "Debug":
cmdline.extend(
[
"-btb-bopt",
"Development",
"-btb-bopt",
"AllowDebugging",
]
)
cmdline.append("-quit")
full_version = "%s-%s" % (get_end_user_version(project_dir), stamp)
# Populate environment with secrets just before calling subprocess
for (env_var, credential_name) in required_credentials:
if env_var not in cmd_env:
if is_jenkins:
# TODO(pld): Look into Jenkins plugins to get at these credentials
raise BuildFailed(
'Credential "%s" is missing from Jenkins build environment'
% env_var
)
cmd_env[env_var] = (
get_credential(credential_name).get_secret().encode("ascii")
)
proc = subprocess.Popen(cmdline, stdout=sys.stdout, stderr=sys.stderr, env=cmd_env)
del cmd_env
with unitybuild.utils.ensure_terminate(proc):
with LogTailer(logfile, disabled=is_jenkins):
with open(os.path.join(output_dir, "build_stamp.txt"), "w") as outf:
outf.write(full_version)
# Use wait() instead of communicate() because Windows can't
# interrupt the thread joins that communicate() uses.
proc.wait()
with open(logfile) as inf:
log = inf.read().replace("\r", "")
check_compile_output(log)
if proc.returncode != 0:
analyze_unity_failure(proc.returncode, log)
# sanity-checking since we've been seeing bad Oculus builds
if platform == "Windows":
required_files = []
for f in required_files:
if not os.path.exists(os.path.join(output_dir, f)):
raise BuildFailed(
"""Build is missing the file '%s'
This is a known Unity bug and the only thing to do is try the build
over and over again until it works"""
% f
)
return output_dir