def installQt(self):
qt5InstallPath = self.getQt5InstallPath()
if not os.path.isdir(qt5InstallPath):
print('Downloading Qt from AWS')
dest, tail = os.path.split(qt5InstallPath)
url = 'NOT DEFINED'
if platform.system() == 'Windows':
url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-windows3.tar.gz'
elif platform.system() == 'Darwin':
url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-macos3.tar.gz'
elif platform.system() == 'Linux':
if platform.linux_distribution()[1][:3] == '16.':
url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-ubuntu-16.04.tar.gz'
elif platform.linux_distribution()[1][:3] == '18.':
url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-ubuntu-18.04.tar.gz'
else:
print('UNKNOWN LINUX VERSION!!!')
else:
print('UNKNOWN OPERATING SYSTEM!!!')
print('Extracting ' + url + ' to ' + dest)
hifi_utils.downloadAndExtract(url, dest)
else:
print('Qt has already been downloaded')
def bootstrap(self):
if self.upToDate():
return
self.clean()
downloadVcpkg = False
if self.args.force_bootstrap:
print("Forcing bootstrap")
downloadVcpkg = True
if not downloadVcpkg and not os.path.isfile(self.exe):
print("Missing executable, boostrapping")
downloadVcpkg = True
# Make sure we have a vcpkg executable
testFile = os.path.join(self.path, '.vcpkg-root')
if not downloadVcpkg and not os.path.isfile(testFile):
print("Missing {}, bootstrapping".format(testFile))
downloadVcpkg = True
if downloadVcpkg:
print("Fetching vcpkg from {} to {}".format(
self.vcpkgUrl, self.path))
hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path,
self.vcpkgHash)
print("Replacing port files")
portsPath = os.path.join(self.path, 'ports')
if (os.path.islink(portsPath)):
os.unlink(portsPath)
if (os.path.isdir(portsPath)):
shutil.rmtree(portsPath, ignore_errors=True)
shutil.copytree(self.sourcePortsPath, portsPath)
def setupAndroidDependencies(self):
# vcpkg prebuilt
if not os.path.isdir(
os.path.join(self.path, 'installed', 'arm64-android')):
dest = os.path.join(self.path, 'installed')
url = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-arm64-android.tar.gz"
# FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash
#hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3"
#hifi_utils.downloadAndExtract(url, dest, hash)
hifi_utils.downloadAndExtract(url, dest)
print("Installing additional android archives")
androidPackages = hifi_android.getPlatformPackages()
for packageName in androidPackages:
package = androidPackages[packageName]
dest = os.path.join(self.androidPackagePath, packageName)
if os.path.isdir(dest):
continue
url = hifi_android.getPackageUrl(package)
zipFile = package['file'].endswith('.zip')
print("Android archive {}".format(package['file']))
hifi_utils.downloadAndExtract(url,
dest,
isZip=zipFile,
hash=package['checksum'],
hasher=hashlib.md5())
def installQt(self):
qt5InstallPath = self.getQt5InstallPath()
if not os.path.isdir(qt5InstallPath):
print ('Downloading Qt from AWS')
dest, tail = os.path.split(qt5InstallPath)
url = 'NOT DEFINED'
if platform.system() == 'Windows':
url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-windows2.tar.gz'
elif platform.system() == 'Darwin':
url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-macos2.tar.gz'
elif platform.system() == 'Linux':
if platform.linux_distribution()[1][:3] == '16.':
url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-ubuntu-16.04.tar.gz'
elif platform.linux_distribution()[1][:3] == '18.':
url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-ubuntu-18.04.tar.gz'
else:
print('UNKNOWN LINUX VERSION!!!')
else:
print('UNKNOWN OPERATING SYSTEM!!!')
print('Extracting ' + url + ' to ' + dest)
hifi_utils.downloadAndExtract(url, dest)
else:
print ('Qt has already been downloaded')
def bootstrap(self):
if self.upToDate():
return
self.clean()
downloadVcpkg = False
if self.args.force_bootstrap:
print("Forcing bootstrap")
downloadVcpkg = True
if not downloadVcpkg and not os.path.isfile(self.exe):
print("Missing executable, boostrapping")
downloadVcpkg = True
# Make sure we have a vcpkg executable
testFile = os.path.join(self.path, '.vcpkg-root')
if not downloadVcpkg and not os.path.isfile(testFile):
print("Missing {}, bootstrapping".format(testFile))
downloadVcpkg = True
if downloadVcpkg:
print("Fetching vcpkg from {} to {}".format(self.vcpkgUrl, self.path))
hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path, self.vcpkgHash)
print("Replacing port files")
portsPath = os.path.join(self.path, 'ports')
if (os.path.islink(portsPath)):
os.unlink(portsPath)
if (os.path.isdir(portsPath)):
shutil.rmtree(portsPath, ignore_errors=True)
shutil.copytree(self.sourcePortsPath, portsPath)
def setupDependencies(self, qt=None):
if self.prebuiltArchive:
if not os.path.isfile(self.prebuildTagFile):
print('Extracting ' + self.prebuiltArchive + ' to ' + self.path)
hifi_utils.downloadAndExtract(self.prebuiltArchive, self.path)
self.writePrebuildTag()
return
if qt is not None:
self.buildEnv['QT_CMAKE_PREFIX_PATH'] = qt
# Special case for android, grab a bunch of binaries
# FIXME remove special casing for android builds eventually
if self.args.android:
print("Installing Android binaries")
self.setupAndroidDependencies()
print("Installing host tools")
if (self.vcpkgBuildType):
self.copyTripletForBuildType(self.hostTriplet)
self.run(['install', '--triplet', self.getTripletWithBuildType(self.hostTriplet), 'hifi-host-tools'])
# If not android, install the hifi-client-deps libraries
if not self.args.android:
print("Installing build dependencies")
if (self.vcpkgBuildType):
self.copyTripletForBuildType(self.triplet)
self.run(['install', '--triplet', self.getTripletWithBuildType(self.triplet), 'hifi-client-deps'])
def installQt(self):
if not os.path.isdir(self.fullPath):
print('Downloading Qt from AWS')
print('Extracting ' + self.qtUrl + ' to ' + self.path)
hifi_utils.downloadAndExtract(self.qtUrl, self.path)
else:
print('Qt has already been downloaded')
def setupAndroidDependencies(self):
# vcpkg prebuilt
if not os.path.isdir(
os.path.join(self.path, 'installed', 'arm64-android')):
dest = os.path.join(self.path, 'installed')
url = self.readVar('EXTERNAL_VCPKG_ANDROID_URLS').split(';')
# FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash
#hash = self.readVar(('EXTERNAL_VCPKG_ANDROID_SHA512')
#hifi_utils.downloadAndExtract(url, dest, hash)
hifi_utils.downloadAndExtract(url, dest)
print("Installing additional android archives")
androidPackages = hifi_android.getPlatformPackages()
for packageName in androidPackages:
package = androidPackages[packageName]
dest = os.path.join(self.androidPackagePath, packageName)
if os.path.isdir(dest):
continue
url = hifi_android.getPackageUrl(package)
zipFile = package['file'].endswith('.zip')
print("Android archive {}".format(package['file']))
hifi_utils.downloadAndExtract(url,
dest,
isZip=zipFile,
hash=package['checksum'],
hasher=hashlib.md5())
def setupAndroidDependencies(self):
# vcpkg prebuilt
if not os.path.isdir(os.path.join(self.path, 'installed', 'arm64-android')):
dest = os.path.join(self.path, 'installed')
url = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-arm64-android.tar.gz"
# FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash
#hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3"
#hifi_utils.downloadAndExtract(url, dest, hash)
hifi_utils.downloadAndExtract(url, dest)
def setupAndroidDependencies(self):
# vcpkg prebuilt
if not os.path.isdir(
os.path.join(self.path, 'installed', 'arm64-android')):
dest = os.path.join(self.path, 'installed')
url = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-arm64-android.tar.gz"
# FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash
#hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3"
#hifi_utils.downloadAndExtract(url, dest, hash)
hifi_utils.downloadAndExtract(url, dest)
def bootstrap(self):
if self.upToDate():
self.copyEnv()
return
if self.prebuiltArchive is not None:
return
self.clean()
downloadVcpkg = False
if self.args.force_bootstrap:
print("Forcing bootstrap")
downloadVcpkg = True
if not downloadVcpkg and not os.path.isfile(self.exe):
print("Missing executable, boot-strapping")
downloadVcpkg = True
# Make sure we have a vcpkg executable
testFile = os.path.join(self.path, '.vcpkg-root')
if not downloadVcpkg and not os.path.isfile(testFile):
print("Missing {}, bootstrapping".format(testFile))
downloadVcpkg = True
if downloadVcpkg:
if "HIFI_VCPKG_BOOTSTRAP" in os.environ:
print("Cloning vcpkg from github to {}".format(self.path))
hifi_utils.executeSubprocess([
'git', 'clone', 'https://github.com/microsoft/vcpkg',
self.path
])
print("Bootstrapping vcpkg")
hifi_utils.executeSubprocess(self.bootstrapCmds,
folder=self.path,
env=self.bootstrapEnv)
else:
print("Fetching vcpkg from {} to {}".format(
self.vcpkgUrl, self.path))
hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path,
self.vcpkgSha512)
print("Replacing port files")
portsPath = os.path.join(self.path, 'ports')
if (os.path.islink(portsPath)):
os.unlink(portsPath)
if (os.path.isdir(portsPath)):
shutil.rmtree(portsPath, ignore_errors=True)
shutil.copytree(self.sourcePortsPath, portsPath)
self.copyEnv()
def setupAndroidDependencies(self):
# vcpkg prebuilt
if not os.path.isdir(os.path.join(self.path, 'installed', 'arm64-android')):
dest = os.path.join(self.path, 'installed')
url = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-arm64-android.tar.gz"
# FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash
#hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3"
#hifi_utils.downloadAndExtract(url, dest, hash)
hifi_utils.downloadAndExtract(url, dest)
print("Installing additional android archives")
androidPackages = hifi_android.getPlatformPackages()
for packageName in androidPackages:
package = androidPackages[packageName]
dest = os.path.join(self.androidPackagePath, packageName)
if os.path.isdir(dest):
continue
url = hifi_android.getPackageUrl(package)
zipFile = package['file'].endswith('.zip')
print("Android archive {}".format(package['file']))
hifi_utils.downloadAndExtract(url, dest, isZip=zipFile, hash=package['checksum'], hasher=hashlib.md5())
def main():
# Fixup env variables. Leaving `USE_CCACHE` on will cause scribe to fail to build
# VCPKG_ROOT seems to cause confusion on Windows systems that previously used it for
# building OpenSSL
removeEnvVars = ['VCPKG_ROOT', 'USE_CCACHE']
for var in removeEnvVars:
if var in os.environ:
del os.environ[var]
args = parse_args()
nsis_urls = hifi_utils.readEnviromentVariableFromFile(
args.build_root, 'EXTERNAL_NSIS_HIFI_PLUGINS_URLS').split(';')
if args.ci_build:
logging.basicConfig(datefmt='%H:%M:%S',
format='%(asctime)s %(guid)s %(message)s',
level=logging.INFO)
logger.info('start')
# OS dependent information
system = platform.system()
if 'Windows' == system and 'CI_BUILD' in os.environ and os.environ[
"CI_BUILD"] == "Github":
logger.info("Downloading NSIS")
with timer('NSIS'):
hifi_utils.downloadAndExtract(nsis_urls, "C:/Program Files (x86)")
qtInstallPath = None
# If not android, install our Qt build
if not args.android:
qt = hifi_qt.QtDownloader(args)
qtInstallPath = qt.cmakePath
if qtInstallPath is not None:
# qtInstallPath is None when we're doing a system Qt build
print("cmake path: " + qtInstallPath)
with hifi_singleton.Singleton(qt.lockFile) as lock:
with timer('Qt'):
qt.installQt()
qt.writeConfig()
else:
if (os.environ["VIRCADIA_USE_SYSTEM_QT"]):
print("System Qt selected")
else:
raise Exception(
"Internal error: System Qt not selected, but hifi_qt.py failed to return a cmake path"
)
pm = hifi_vcpkg.VcpkgRepo(args)
if qtInstallPath is not None:
pm.writeVar('QT_CMAKE_PREFIX_PATH', qtInstallPath)
# Only allow one instance of the program to run at a time
if qtInstallPath is not None:
pm.writeVar('QT_CMAKE_PREFIX_PATH', qtInstallPath)
# Only allow one instance of the program to run at a time
with hifi_singleton.Singleton(pm.lockFile) as lock:
with timer('Bootstraping'):
if not pm.upToDate():
pm.bootstrap()
# Always write the tag, even if we changed nothing. This
# allows vcpkg to reclaim disk space by identifying directories with
# tags that haven't been touched in a long time
pm.writeTag()
# Grab our required dependencies:
# * build host tools, like spirv-cross and scribe
# * build client dependencies like openssl and nvtt
with timer('Setting up dependencies'):
pm.setupDependencies(qt=qtInstallPath)
# wipe out the build directories (after writing the tag, since failure
# here shouldn't invalidate the vcpkg install)
with timer('Cleaning builds'):
pm.cleanBuilds()
# If we're running in android mode, we also need to grab a bunch of additional binaries
# (this logic is all migrated from the old setupDependencies tasks in gradle)
if args.android:
# Find the target location
appPath = hifi_utils.scriptRelative('android/apps/' + args.android)
# Copy the non-Qt libraries specified in the config in hifi_android.py
hifi_android.copyAndroidLibs(pm.androidPackagePath, appPath)
# Determine the Qt package path
qtPath = os.path.join(pm.androidPackagePath, 'qt')
hifi_android.QtPackager(appPath, qtPath).bundle()
# Fixup the vcpkg cmake to not reset VCPKG_TARGET_TRIPLET
pm.fixupCmakeScript()
if not args.vcpkg_skip_clean:
# Cleanup downloads and packages folders in vcpkg to make it smaller for CI
pm.cleanupDevelopmentFiles()
# Write the vcpkg config to the build directory last
with timer('Writing configuration'):
pm.writeConfig()
logger.info('end')
def installQt(self):
if not os.path.isdir(self.fullPath):
print("Fetching Qt from {} to {}".format(self.qtUrl, self.path))
hifi_utils.downloadAndExtract(self.qtUrl, self.path)
else:
print('Qt has already been downloaded')
def main():
# Fixup env variables. Leaving `USE_CCACHE` on will cause scribe to fail to build
# VCPKG_ROOT seems to cause confusion on Windows systems that previously used it for
# building OpenSSL
removeEnvVars = ['VCPKG_ROOT', 'USE_CCACHE']
for var in removeEnvVars:
if var in os.environ:
del os.environ[var]
args = parse_args()
if args.ci_build:
logging.basicConfig(datefmt='%H:%M:%S', format='%(asctime)s %(guid)s %(message)s', level=logging.INFO)
logger.info('sha=%s' % headSha())
logger.info('start')
# OS dependent information
system = platform.system()
if 'Windows' == system and 'CI_BUILD' in os.environ and os.environ["CI_BUILD"] == "Github":
logger.info("Downloading NSIS")
with timer('NSIS'):
hifi_utils.downloadAndExtract('https://hifi-public.s3.amazonaws.com/dependencies/NSIS-hifi-plugins-1.0.tgz', "C:/Program Files (x86)")
qtInstallPath = ''
# If not android, install our Qt build
if not args.android:
qt = hifi_qt.QtDownloader(args)
qtInstallPath = qt.cmakePath
with hifi_singleton.Singleton(qt.lockFile) as lock:
with timer('Qt'):
qt.installQt()
qt.writeConfig()
# Only allow one instance of the program to run at a time
pm = hifi_vcpkg.VcpkgRepo(args)
with hifi_singleton.Singleton(pm.lockFile) as lock:
with timer('Bootstraping'):
if not pm.upToDate():
pm.bootstrap()
# Always write the tag, even if we changed nothing. This
# allows vcpkg to reclaim disk space by identifying directories with
# tags that haven't been touched in a long time
pm.writeTag()
# Grab our required dependencies:
# * build host tools, like spirv-cross and scribe
# * build client dependencies like openssl and nvtt
with timer('Setting up dependencies'):
pm.setupDependencies(qt=qtInstallPath)
# wipe out the build directories (after writing the tag, since failure
# here shouldn't invalidte the vcpkg install)
with timer('Cleaning builds'):
pm.cleanBuilds()
# If we're running in android mode, we also need to grab a bunch of additional binaries
# (this logic is all migrated from the old setupDependencies tasks in gradle)
if args.android:
# Find the target location
appPath = hifi_utils.scriptRelative('android/apps/' + args.android)
# Copy the non-Qt libraries specified in the config in hifi_android.py
hifi_android.copyAndroidLibs(pm.androidPackagePath, appPath)
# Determine the Qt package path
qtPath = os.path.join(pm.androidPackagePath, 'qt')
hifi_android.QtPackager(appPath, qtPath).bundle()
# Write the vcpkg config to the build directory last
with timer('Writing configuration'):
pm.writeConfig()
logger.info('end')
def bootstrap(self):
if self.upToDate():
return
if self.prebuiltArchive is not None:
return
self.clean()
downloadVcpkg = False
if self.args.force_bootstrap:
print("Forcing bootstrap")
downloadVcpkg = True
if not downloadVcpkg and not os.path.isfile(self.exe):
print("Missing executable, boot-strapping")
downloadVcpkg = True
# Make sure we have a vcpkg executable
testFile = os.path.join(self.path, '.vcpkg-root')
if not downloadVcpkg and not os.path.isfile(testFile):
print("Missing {}, bootstrapping".format(testFile))
downloadVcpkg = True
if downloadVcpkg:
# print("Cloning vcpkg from github to {}".format(self.path))
# hifi_utils.executeSubprocess(['git', 'clone', 'https://github.com/microsoft/vcpkg.git', self.path])
print("Download vcpkg from GitHub to {}".format(self.path))
hifi_utils.downloadAndExtract(
"https://codeload.github.com/microsoft/vcpkg/zip/" +
self.vcpkgVersion,
self.path,
isZip=True)
vcpkg_extract_dir = os.path.join(self.path,
"vcpkg-" + self.vcpkgVersion)
for filename in os.listdir(vcpkg_extract_dir):
shutil.move(os.path.join(vcpkg_extract_dir, filename),
os.path.join(self.path, filename))
os.rmdir(vcpkg_extract_dir)
if platform.system() != "Windows":
hifi_utils.executeSubprocess([
"chmod", "+x",
os.path.join(self.path, "bootstrap-vcpkg.sh")
])
print("Bootstrapping vcpkg")
hifi_utils.executeSubprocess(self.bootstrapCmds,
folder=self.path,
env=self.bootstrapEnv)
print("Replacing port files")
portsPath = os.path.join(self.path, 'ports')
if (os.path.islink(portsPath)):
os.unlink(portsPath)
if (os.path.isdir(portsPath)):
shutil.rmtree(portsPath, ignore_errors=True)
shutil.copytree(self.sourcePortsPath, portsPath)
