def installQt(self): qt5InstallPath = self.getQt5InstallPath() if not os.path.isdir(qt5InstallPath): print('Downloading Qt from AWS') dest, tail = os.path.split(qt5InstallPath) url = 'NOT DEFINED' if platform.system() == 'Windows': url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-windows3.tar.gz' elif platform.system() == 'Darwin': url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-macos3.tar.gz' elif platform.system() == 'Linux': if platform.linux_distribution()[1][:3] == '16.': url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-ubuntu-16.04.tar.gz' elif platform.linux_distribution()[1][:3] == '18.': url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-ubuntu-18.04.tar.gz' else: print('UNKNOWN LINUX VERSION!!!') else: print('UNKNOWN OPERATING SYSTEM!!!') print('Extracting ' + url + ' to ' + dest) hifi_utils.downloadAndExtract(url, dest) else: print('Qt has already been downloaded')
def bootstrap(self): if self.upToDate(): return self.clean() downloadVcpkg = False if self.args.force_bootstrap: print("Forcing bootstrap") downloadVcpkg = True if not downloadVcpkg and not os.path.isfile(self.exe): print("Missing executable, boostrapping") downloadVcpkg = True # Make sure we have a vcpkg executable testFile = os.path.join(self.path, '.vcpkg-root') if not downloadVcpkg and not os.path.isfile(testFile): print("Missing {}, bootstrapping".format(testFile)) downloadVcpkg = True if downloadVcpkg: print("Fetching vcpkg from {} to {}".format( self.vcpkgUrl, self.path)) hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path, self.vcpkgHash) print("Replacing port files") portsPath = os.path.join(self.path, 'ports') if (os.path.islink(portsPath)): os.unlink(portsPath) if (os.path.isdir(portsPath)): shutil.rmtree(portsPath, ignore_errors=True) shutil.copytree(self.sourcePortsPath, portsPath)
def setupAndroidDependencies(self): # vcpkg prebuilt if not os.path.isdir( os.path.join(self.path, 'installed', 'arm64-android')): dest = os.path.join(self.path, 'installed') url = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-arm64-android.tar.gz" # FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash #hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3" #hifi_utils.downloadAndExtract(url, dest, hash) hifi_utils.downloadAndExtract(url, dest) print("Installing additional android archives") androidPackages = hifi_android.getPlatformPackages() for packageName in androidPackages: package = androidPackages[packageName] dest = os.path.join(self.androidPackagePath, packageName) if os.path.isdir(dest): continue url = hifi_android.getPackageUrl(package) zipFile = package['file'].endswith('.zip') print("Android archive {}".format(package['file'])) hifi_utils.downloadAndExtract(url, dest, isZip=zipFile, hash=package['checksum'], hasher=hashlib.md5())
def installQt(self): qt5InstallPath = self.getQt5InstallPath() if not os.path.isdir(qt5InstallPath): print ('Downloading Qt from AWS') dest, tail = os.path.split(qt5InstallPath) url = 'NOT DEFINED' if platform.system() == 'Windows': url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-windows2.tar.gz' elif platform.system() == 'Darwin': url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-macos2.tar.gz' elif platform.system() == 'Linux': if platform.linux_distribution()[1][:3] == '16.': url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-ubuntu-16.04.tar.gz' elif platform.linux_distribution()[1][:3] == '18.': url = 'https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/qt5-install-5.12.3-ubuntu-18.04.tar.gz' else: print('UNKNOWN LINUX VERSION!!!') else: print('UNKNOWN OPERATING SYSTEM!!!') print('Extracting ' + url + ' to ' + dest) hifi_utils.downloadAndExtract(url, dest) else: print ('Qt has already been downloaded')
def bootstrap(self): if self.upToDate(): return self.clean() downloadVcpkg = False if self.args.force_bootstrap: print("Forcing bootstrap") downloadVcpkg = True if not downloadVcpkg and not os.path.isfile(self.exe): print("Missing executable, boostrapping") downloadVcpkg = True # Make sure we have a vcpkg executable testFile = os.path.join(self.path, '.vcpkg-root') if not downloadVcpkg and not os.path.isfile(testFile): print("Missing {}, bootstrapping".format(testFile)) downloadVcpkg = True if downloadVcpkg: print("Fetching vcpkg from {} to {}".format(self.vcpkgUrl, self.path)) hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path, self.vcpkgHash) print("Replacing port files") portsPath = os.path.join(self.path, 'ports') if (os.path.islink(portsPath)): os.unlink(portsPath) if (os.path.isdir(portsPath)): shutil.rmtree(portsPath, ignore_errors=True) shutil.copytree(self.sourcePortsPath, portsPath)
def setupDependencies(self, qt=None): if self.prebuiltArchive: if not os.path.isfile(self.prebuildTagFile): print('Extracting ' + self.prebuiltArchive + ' to ' + self.path) hifi_utils.downloadAndExtract(self.prebuiltArchive, self.path) self.writePrebuildTag() return if qt is not None: self.buildEnv['QT_CMAKE_PREFIX_PATH'] = qt # Special case for android, grab a bunch of binaries # FIXME remove special casing for android builds eventually if self.args.android: print("Installing Android binaries") self.setupAndroidDependencies() print("Installing host tools") if (self.vcpkgBuildType): self.copyTripletForBuildType(self.hostTriplet) self.run(['install', '--triplet', self.getTripletWithBuildType(self.hostTriplet), 'hifi-host-tools']) # If not android, install the hifi-client-deps libraries if not self.args.android: print("Installing build dependencies") if (self.vcpkgBuildType): self.copyTripletForBuildType(self.triplet) self.run(['install', '--triplet', self.getTripletWithBuildType(self.triplet), 'hifi-client-deps'])
def installQt(self): if not os.path.isdir(self.fullPath): print('Downloading Qt from AWS') print('Extracting ' + self.qtUrl + ' to ' + self.path) hifi_utils.downloadAndExtract(self.qtUrl, self.path) else: print('Qt has already been downloaded')
def setupAndroidDependencies(self): # vcpkg prebuilt if not os.path.isdir( os.path.join(self.path, 'installed', 'arm64-android')): dest = os.path.join(self.path, 'installed') url = self.readVar('EXTERNAL_VCPKG_ANDROID_URLS').split(';') # FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash #hash = self.readVar(('EXTERNAL_VCPKG_ANDROID_SHA512') #hifi_utils.downloadAndExtract(url, dest, hash) hifi_utils.downloadAndExtract(url, dest) print("Installing additional android archives") androidPackages = hifi_android.getPlatformPackages() for packageName in androidPackages: package = androidPackages[packageName] dest = os.path.join(self.androidPackagePath, packageName) if os.path.isdir(dest): continue url = hifi_android.getPackageUrl(package) zipFile = package['file'].endswith('.zip') print("Android archive {}".format(package['file'])) hifi_utils.downloadAndExtract(url, dest, isZip=zipFile, hash=package['checksum'], hasher=hashlib.md5())
def setupAndroidDependencies(self): # vcpkg prebuilt if not os.path.isdir(os.path.join(self.path, 'installed', 'arm64-android')): dest = os.path.join(self.path, 'installed') url = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-arm64-android.tar.gz" # FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash #hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3" #hifi_utils.downloadAndExtract(url, dest, hash) hifi_utils.downloadAndExtract(url, dest)
def setupAndroidDependencies(self): # vcpkg prebuilt if not os.path.isdir( os.path.join(self.path, 'installed', 'arm64-android')): dest = os.path.join(self.path, 'installed') url = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-arm64-android.tar.gz" # FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash #hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3" #hifi_utils.downloadAndExtract(url, dest, hash) hifi_utils.downloadAndExtract(url, dest)
def bootstrap(self): if self.upToDate(): self.copyEnv() return if self.prebuiltArchive is not None: return self.clean() downloadVcpkg = False if self.args.force_bootstrap: print("Forcing bootstrap") downloadVcpkg = True if not downloadVcpkg and not os.path.isfile(self.exe): print("Missing executable, boot-strapping") downloadVcpkg = True # Make sure we have a vcpkg executable testFile = os.path.join(self.path, '.vcpkg-root') if not downloadVcpkg and not os.path.isfile(testFile): print("Missing {}, bootstrapping".format(testFile)) downloadVcpkg = True if downloadVcpkg: if "HIFI_VCPKG_BOOTSTRAP" in os.environ: print("Cloning vcpkg from github to {}".format(self.path)) hifi_utils.executeSubprocess([ 'git', 'clone', 'https://github.com/microsoft/vcpkg', self.path ]) print("Bootstrapping vcpkg") hifi_utils.executeSubprocess(self.bootstrapCmds, folder=self.path, env=self.bootstrapEnv) else: print("Fetching vcpkg from {} to {}".format( self.vcpkgUrl, self.path)) hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path, self.vcpkgSha512) print("Replacing port files") portsPath = os.path.join(self.path, 'ports') if (os.path.islink(portsPath)): os.unlink(portsPath) if (os.path.isdir(portsPath)): shutil.rmtree(portsPath, ignore_errors=True) shutil.copytree(self.sourcePortsPath, portsPath) self.copyEnv()
def setupAndroidDependencies(self): # vcpkg prebuilt if not os.path.isdir(os.path.join(self.path, 'installed', 'arm64-android')): dest = os.path.join(self.path, 'installed') url = "https://hifi-public.s3.amazonaws.com/dependencies/vcpkg/vcpkg-arm64-android.tar.gz" # FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash #hash = "832f82a4d090046bdec25d313e20f56ead45b54dd06eee3798c5c8cbdd64cce4067692b1c3f26a89afe6ff9917c10e4b601c118bea06d23f8adbfe5c0ec12bc3" #hifi_utils.downloadAndExtract(url, dest, hash) hifi_utils.downloadAndExtract(url, dest) print("Installing additional android archives") androidPackages = hifi_android.getPlatformPackages() for packageName in androidPackages: package = androidPackages[packageName] dest = os.path.join(self.androidPackagePath, packageName) if os.path.isdir(dest): continue url = hifi_android.getPackageUrl(package) zipFile = package['file'].endswith('.zip') print("Android archive {}".format(package['file'])) hifi_utils.downloadAndExtract(url, dest, isZip=zipFile, hash=package['checksum'], hasher=hashlib.md5())
def main(): # Fixup env variables. Leaving `USE_CCACHE` on will cause scribe to fail to build # VCPKG_ROOT seems to cause confusion on Windows systems that previously used it for # building OpenSSL removeEnvVars = ['VCPKG_ROOT', 'USE_CCACHE'] for var in removeEnvVars: if var in os.environ: del os.environ[var] args = parse_args() nsis_urls = hifi_utils.readEnviromentVariableFromFile( args.build_root, 'EXTERNAL_NSIS_HIFI_PLUGINS_URLS').split(';') if args.ci_build: logging.basicConfig(datefmt='%H:%M:%S', format='%(asctime)s %(guid)s %(message)s', level=logging.INFO) logger.info('start') # OS dependent information system = platform.system() if 'Windows' == system and 'CI_BUILD' in os.environ and os.environ[ "CI_BUILD"] == "Github": logger.info("Downloading NSIS") with timer('NSIS'): hifi_utils.downloadAndExtract(nsis_urls, "C:/Program Files (x86)") qtInstallPath = None # If not android, install our Qt build if not args.android: qt = hifi_qt.QtDownloader(args) qtInstallPath = qt.cmakePath if qtInstallPath is not None: # qtInstallPath is None when we're doing a system Qt build print("cmake path: " + qtInstallPath) with hifi_singleton.Singleton(qt.lockFile) as lock: with timer('Qt'): qt.installQt() qt.writeConfig() else: if (os.environ["VIRCADIA_USE_SYSTEM_QT"]): print("System Qt selected") else: raise Exception( "Internal error: System Qt not selected, but hifi_qt.py failed to return a cmake path" ) pm = hifi_vcpkg.VcpkgRepo(args) if qtInstallPath is not None: pm.writeVar('QT_CMAKE_PREFIX_PATH', qtInstallPath) # Only allow one instance of the program to run at a time if qtInstallPath is not None: pm.writeVar('QT_CMAKE_PREFIX_PATH', qtInstallPath) # Only allow one instance of the program to run at a time with hifi_singleton.Singleton(pm.lockFile) as lock: with timer('Bootstraping'): if not pm.upToDate(): pm.bootstrap() # Always write the tag, even if we changed nothing. This # allows vcpkg to reclaim disk space by identifying directories with # tags that haven't been touched in a long time pm.writeTag() # Grab our required dependencies: # * build host tools, like spirv-cross and scribe # * build client dependencies like openssl and nvtt with timer('Setting up dependencies'): pm.setupDependencies(qt=qtInstallPath) # wipe out the build directories (after writing the tag, since failure # here shouldn't invalidate the vcpkg install) with timer('Cleaning builds'): pm.cleanBuilds() # If we're running in android mode, we also need to grab a bunch of additional binaries # (this logic is all migrated from the old setupDependencies tasks in gradle) if args.android: # Find the target location appPath = hifi_utils.scriptRelative('android/apps/' + args.android) # Copy the non-Qt libraries specified in the config in hifi_android.py hifi_android.copyAndroidLibs(pm.androidPackagePath, appPath) # Determine the Qt package path qtPath = os.path.join(pm.androidPackagePath, 'qt') hifi_android.QtPackager(appPath, qtPath).bundle() # Fixup the vcpkg cmake to not reset VCPKG_TARGET_TRIPLET pm.fixupCmakeScript() if not args.vcpkg_skip_clean: # Cleanup downloads and packages folders in vcpkg to make it smaller for CI pm.cleanupDevelopmentFiles() # Write the vcpkg config to the build directory last with timer('Writing configuration'): pm.writeConfig() logger.info('end')
def installQt(self): if not os.path.isdir(self.fullPath): print("Fetching Qt from {} to {}".format(self.qtUrl, self.path)) hifi_utils.downloadAndExtract(self.qtUrl, self.path) else: print('Qt has already been downloaded')
def main(): # Fixup env variables. Leaving `USE_CCACHE` on will cause scribe to fail to build # VCPKG_ROOT seems to cause confusion on Windows systems that previously used it for # building OpenSSL removeEnvVars = ['VCPKG_ROOT', 'USE_CCACHE'] for var in removeEnvVars: if var in os.environ: del os.environ[var] args = parse_args() if args.ci_build: logging.basicConfig(datefmt='%H:%M:%S', format='%(asctime)s %(guid)s %(message)s', level=logging.INFO) logger.info('sha=%s' % headSha()) logger.info('start') # OS dependent information system = platform.system() if 'Windows' == system and 'CI_BUILD' in os.environ and os.environ["CI_BUILD"] == "Github": logger.info("Downloading NSIS") with timer('NSIS'): hifi_utils.downloadAndExtract('https://hifi-public.s3.amazonaws.com/dependencies/NSIS-hifi-plugins-1.0.tgz', "C:/Program Files (x86)") qtInstallPath = '' # If not android, install our Qt build if not args.android: qt = hifi_qt.QtDownloader(args) qtInstallPath = qt.cmakePath with hifi_singleton.Singleton(qt.lockFile) as lock: with timer('Qt'): qt.installQt() qt.writeConfig() # Only allow one instance of the program to run at a time pm = hifi_vcpkg.VcpkgRepo(args) with hifi_singleton.Singleton(pm.lockFile) as lock: with timer('Bootstraping'): if not pm.upToDate(): pm.bootstrap() # Always write the tag, even if we changed nothing. This # allows vcpkg to reclaim disk space by identifying directories with # tags that haven't been touched in a long time pm.writeTag() # Grab our required dependencies: # * build host tools, like spirv-cross and scribe # * build client dependencies like openssl and nvtt with timer('Setting up dependencies'): pm.setupDependencies(qt=qtInstallPath) # wipe out the build directories (after writing the tag, since failure # here shouldn't invalidte the vcpkg install) with timer('Cleaning builds'): pm.cleanBuilds() # If we're running in android mode, we also need to grab a bunch of additional binaries # (this logic is all migrated from the old setupDependencies tasks in gradle) if args.android: # Find the target location appPath = hifi_utils.scriptRelative('android/apps/' + args.android) # Copy the non-Qt libraries specified in the config in hifi_android.py hifi_android.copyAndroidLibs(pm.androidPackagePath, appPath) # Determine the Qt package path qtPath = os.path.join(pm.androidPackagePath, 'qt') hifi_android.QtPackager(appPath, qtPath).bundle() # Write the vcpkg config to the build directory last with timer('Writing configuration'): pm.writeConfig() logger.info('end')
def bootstrap(self): if self.upToDate(): return if self.prebuiltArchive is not None: return self.clean() downloadVcpkg = False if self.args.force_bootstrap: print("Forcing bootstrap") downloadVcpkg = True if not downloadVcpkg and not os.path.isfile(self.exe): print("Missing executable, boot-strapping") downloadVcpkg = True # Make sure we have a vcpkg executable testFile = os.path.join(self.path, '.vcpkg-root') if not downloadVcpkg and not os.path.isfile(testFile): print("Missing {}, bootstrapping".format(testFile)) downloadVcpkg = True if downloadVcpkg: # print("Cloning vcpkg from github to {}".format(self.path)) # hifi_utils.executeSubprocess(['git', 'clone', 'https://github.com/microsoft/vcpkg.git', self.path]) print("Download vcpkg from GitHub to {}".format(self.path)) hifi_utils.downloadAndExtract( "https://codeload.github.com/microsoft/vcpkg/zip/" + self.vcpkgVersion, self.path, isZip=True) vcpkg_extract_dir = os.path.join(self.path, "vcpkg-" + self.vcpkgVersion) for filename in os.listdir(vcpkg_extract_dir): shutil.move(os.path.join(vcpkg_extract_dir, filename), os.path.join(self.path, filename)) os.rmdir(vcpkg_extract_dir) if platform.system() != "Windows": hifi_utils.executeSubprocess([ "chmod", "+x", os.path.join(self.path, "bootstrap-vcpkg.sh") ]) print("Bootstrapping vcpkg") hifi_utils.executeSubprocess(self.bootstrapCmds, folder=self.path, env=self.bootstrapEnv) print("Replacing port files") portsPath = os.path.join(self.path, 'ports') if (os.path.islink(portsPath)): os.unlink(portsPath) if (os.path.isdir(portsPath)): shutil.rmtree(portsPath, ignore_errors=True) shutil.copytree(self.sourcePortsPath, portsPath)