def _InstallSdk(buildroot, buildout, buildos, sdk): """Install the SDk into the RCP zip files(s). Args: buildroot: the boot of the build output buildout: the location of the ant build output buildos: the OS the build is running under sdk: the name of the zipped up sdk """ print '_InstallSdk({0}, {1}, {2}, {3})'.format(buildroot, buildout, buildos, sdk) tmp_dir = os.path.join(buildroot, 'tmp') unzip_dir = os.path.join(tmp_dir, 'unzip_sdk') if not os.path.exists(unzip_dir): os.makedirs(unzip_dir) sdk_zip = ziputils.ZipUtil(sdk, buildos) sdk_zip.UnZip(unzip_dir) files = _FindRcpZipFiles(buildout) for f in files: dart_zip_path = os.path.join(buildout, f) print('_installSdk: before ' '{0} is {1}'.format(dart_zip_path, os.path.getsize(dart_zip_path))) dart_zip = ziputils.ZipUtil(dart_zip_path, buildos) dart_zip.AddDirectoryTree(unzip_dir, 'dart') print('_installSdk: after ' '{0} is {1}'.format(dart_zip_path, os.path.getsize(dart_zip_path)))
def RunEditorTests(buildout, buildos): StartBuildStep('run_tests') for editorArchive in _GetTestableRcpArchives(buildout): with utils.TempDir('editor_') as tempDir: print 'Running tests for %s...' % editorArchive zipper = ziputils.ZipUtil(join(buildout, editorArchive), buildos) zipper.UnZip(tempDir) # before we run the editor, suppress any 'restore windows' dialogs if sys.platform == 'darwin': args = [ 'defaults', 'write', 'org.eclipse.eclipse.savedState', 'NSQuitAlwaysKeepsWindows', '-bool', 'false' ] subprocess.call(args, shell=IsWindows()) editorExecutable = GetEditorExecutable(join(tempDir, 'dart')) args = [ editorExecutable, '--test', '--auto-exit', '-data', join(tempDir, 'workspace') ] # Issue 12638. Enable this as soon as we can run editor tests in xvfb # again. ##if sys.platform == 'linux2': ## args = ['xvfb-run', '-a'] + args # this can hang if a 32 bit jvm is not available on windows... if subprocess.call(args, shell=IsWindows()): BuildStepFailure()
def add_download_scripts(zipFile, arch): shell_ending = { 'win': '.bat', 'linux': '.sh', 'mac': '.sh', }[SYSTEM] # We don't have 64-bit versions of dartium/content_shell for # macos/windows. if SYSTEM in ['mac', 'win']: arch = '32' namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RELEASE) # We're adding download scripts to the chromium directory. # The directory tree will look like this after that: # dart/dart-sdk/bin/dart{,.exe} # /chromium/download_contentshell.{sh,bat} # /chromium/download_dartium_debug.{sh,bat} # /chromium/download_file.dart # Add download_file.dart helper utility to the zip file. f = ziputils.ZipUtil(zipFile, buildos) f.AddFile(join(DART_DIR, 'tools', 'dartium', 'download_file.dart'), 'dart/chromium/download_file.dart') # Add content shell download script contentshell_name = namer.dartium_variant_zipfilename( 'content_shell', SYSTEM, arch, 'release') contentshell_download_script = join(scratch_dir, 'download_contentshell') instantiate_download_script_template( contentshell_download_script, { 'VAR_DESTINATION': contentshell_name, 'VAR_DOWNLOAD_URL': ("http://dartlang.org/editor/update/channels/%s/%s/dartium/%s" % (CHANNEL, REVISION, contentshell_name)), }) f.AddFile(contentshell_download_script, 'dart/chromium/download_contentshell%s' % shell_ending) # Add dartium debug download script dartium_debug_name = namer.dartium_variant_zipfilename( 'dartium', SYSTEM, arch, 'debug') dartium_download_script = join(scratch_dir, 'download_dartium_debug') instantiate_download_script_template( dartium_download_script, { 'VAR_DESTINATION': dartium_debug_name, 'VAR_DOWNLOAD_URL': ("http://dartlang.org/editor/update/channels/%s/%s/dartium/%s" % (CHANNEL, REVISION, dartium_debug_name)), }) f.AddFile(dartium_download_script, 'dart/chromium/download_dartium_debug%s' % shell_ending)
def InstallSdk(buildroot, buildout, buildos, sdk_dir): """Install the SDK into the RCP zip files. Args: buildroot: the boot of the build output buildout: the location of the ant build output buildos: the OS the build is running under sdk_dir: the directory containing the built SDKs """ print 'InstallSdk(%s, %s, %s, %s)' % (buildroot, buildout, buildos, sdk_dir) tmp_dir = os.path.join(buildroot, 'tmp') unzip_dir_32 = os.path.join(tmp_dir, 'unzip_sdk_32') if not os.path.exists(unzip_dir_32): os.makedirs(unzip_dir_32) unzip_dir_64 = os.path.join(tmp_dir, 'unzip_sdk_64') if not os.path.exists(unzip_dir_64): os.makedirs(unzip_dir_64) sdk_zip = ziputils.ZipUtil(join(sdk_dir, "dartsdk-%s-32.zip" % buildos), buildos) sdk_zip.UnZip(unzip_dir_32) sdk_zip = ziputils.ZipUtil(join(sdk_dir, "dartsdk-%s-64.zip" % buildos), buildos) sdk_zip.UnZip(unzip_dir_64) files = _FindRcpZipFiles(buildout) for f in files: dart_zip_path = os.path.join(buildout, f) dart_zip = ziputils.ZipUtil(dart_zip_path, buildos) # dart-editor-macosx.cocoa.x86_64.zip if '_64.zip' in f: dart_zip.AddDirectoryTree(unzip_dir_64, 'dart') else: dart_zip.AddDirectoryTree(unzip_dir_32, 'dart')
def _InstallArtifacts(buildout, buildos, extra_artifacts): """Install extra build artifacts into the RCP zip files. Args: buildout: the location of the ant build output buildos: the OS the build is running under extra_artifacts: the directory containing the extra artifacts """ print '_InstallArtifacts({%s}, {%s}, {%s})' % (buildout, buildos, extra_artifacts) files = _FindRcpZipFiles(buildout) for f in files: dart_zip_path = os.path.join(buildout, f) dart_zip = ziputils.ZipUtil(dart_zip_path, buildos) dart_zip.AddDirectoryTree(extra_artifacts, 'dart')
def _InstallArtifacts(buildout, buildos, extra_artifacts): """Install the SDk into the RCP zip files(s). Args: buildout: the location of the ant build output buildos: the OS the build is running under extra_artifacts: the directoryt he extra artifacts are in """ print '_InstallArtifacts({0}, {1}, {2})'.format(buildout, buildos, extra_artifacts) files = _FindRcpZipFiles(buildout) for f in files: dart_zip_path = os.path.join(buildout, f) dart_zip = ziputils.ZipUtil(dart_zip_path, buildos) dart_zip.AddDirectoryTree(extra_artifacts, 'dart')
def PostProcessEditorBuilds(out_dir, buildos): """Post-process the created RCP builds""" with utils.TempDir('editor_scratch') as scratch_dir: def instantiate_download_script_template(destination, replacements): """Helper function for replacing variables in the tools/dartium/download_shellscript_templates.{sh,bat} scripts. It will write the final download script to [destination] after doing all replacements given in the [replacements] dictionary.""" template_location = { 'win': join(DART_DIR, 'tools', 'dartium', 'download_shellscript_template.bat'), 'linux': join(DART_DIR, 'tools', 'dartium', 'download_shellscript_template.sh'), 'mac': join(DART_DIR, 'tools', 'dartium', 'download_shellscript_template.sh'), }[SYSTEM] with open(template_location) as fd: content = fd.read() for key in replacements: content = content.replace(key, replacements[key]) with open(destination, 'w') as fd: fd.write(content) # Make it executable if we are not on windows if SYSTEM != 'win': os.chmod(destination, os.stat(destination).st_mode | stat.S_IEXEC) def add_download_scripts(zipFile, arch): shell_ending = { 'win': '.bat', 'linux': '.sh', 'mac': '.sh', }[SYSTEM] # We don't have 64-bit versions of dartium/content_shell for # macos/windows. if SYSTEM in ['mac', 'win']: arch = '32' namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RELEASE) # We're adding download scripts to the chromium directory. # The directory tree will look like this after that: # dart/dart-sdk/bin/dart{,.exe} # /chromium/download_contentshell.{sh,bat} # /chromium/download_dartium_debug.{sh,bat} # /chromium/download_file.dart # Add download_file.dart helper utility to the zip file. f = ziputils.ZipUtil(zipFile, buildos) f.AddFile(join(DART_DIR, 'tools', 'dartium', 'download_file.dart'), 'dart/chromium/download_file.dart') # Add content shell download script contentshell_name = namer.dartium_variant_zipfilename( 'content_shell', SYSTEM, arch, 'release') contentshell_download_script = join(scratch_dir, 'download_contentshell') instantiate_download_script_template( contentshell_download_script, { 'VAR_DESTINATION': contentshell_name, 'VAR_DOWNLOAD_URL': ("http://dartlang.org/editor/update/channels/%s/%s/dartium/%s" % (CHANNEL, REVISION, contentshell_name)), }) f.AddFile(contentshell_download_script, 'dart/chromium/download_contentshell%s' % shell_ending) # Add dartium debug download script dartium_debug_name = namer.dartium_variant_zipfilename( 'dartium', SYSTEM, arch, 'debug') dartium_download_script = join(scratch_dir, 'download_dartium_debug') instantiate_download_script_template( dartium_download_script, { 'VAR_DESTINATION': dartium_debug_name, 'VAR_DOWNLOAD_URL': ("http://dartlang.org/editor/update/channels/%s/%s/dartium/%s" % (CHANNEL, REVISION, dartium_debug_name)), }) f.AddFile(dartium_download_script, 'dart/chromium/download_dartium_debug%s' % shell_ending) # Create a editor.properties editor_properties = os.path.join(scratch_dir, 'editor.properties') with open(editor_properties, 'w') as fd: fd.write("com.dart.tools.update.core.url=http://dartlang.org" "/editor/update/channels/%s/\n" % CHANNEL) for zipFile in _FindRcpZipFiles(out_dir): basename = os.path.basename(zipFile) is_64bit = basename.endswith('-64.zip') print(' processing %s' % basename) readme_file = join('dart', 'README') if (basename.startswith('darteditor-win32-')): seven_zip = os.path.join(DART_DIR, 'third_party', '7zip', '7za.exe') bot_utils.run([seven_zip, 'd', zipFile, readme_file], env=os.environ) else: bot_utils.run(['zip', '-d', zipFile, readme_file], env=os.environ) # If we're on -dev/-stable build: add an editor.properties file # pointing to the correct update location of the editor for the channel # we're building for. if CHANNEL != 'be': f = ziputils.ZipUtil(zipFile, buildos) f.AddFile(editor_properties, 'dart/editor.properties') # Add a shell/bat script to download contentshell and dartium debug. # (including the necessary tools/dartium/download_file.dart helper). add_download_scripts(zipFile, '64' if is_64bit else '32') # adjust memory params for 64 bit versions if is_64bit: if (basename.startswith('darteditor-macos-')): inifile = join('dart', 'DartEditor.app', 'Contents', 'MacOS', 'DartEditor.ini') else: inifile = join('dart', 'DartEditor.ini') if (basename.startswith('darteditor-win32-')): f = zipfile.ZipFile(zipFile) f.extract(inifile.replace('\\', '/')) f.close() else: bot_utils.run(['unzip', zipFile, inifile], env=os.environ) Modify64BitDartEditorIni(inifile) if (basename.startswith('darteditor-win32-')): seven_zip = os.path.join(DART_DIR, 'third_party', '7zip', '7za.exe') bot_utils.run([seven_zip, 'd', zipFile, inifile], env=os.environ) bot_utils.run([seven_zip, 'a', zipFile, inifile], env=os.environ) else: bot_utils.run(['zip', '-d', zipFile, inifile], env=os.environ) bot_utils.run(['zip', '-q', zipFile, inifile], env=os.environ) os.remove(inifile) # post-process the info.plist file if (basename.startswith('darteditor-macos-')): infofile = join('dart', 'DartEditor.app', 'Contents', 'Info.plist') bot_utils.run(['unzip', zipFile, infofile], env=os.environ) ReplaceInFiles([infofile], [( '<dict>', '<dict>\n\t<key>NSHighResolutionCapable</key>\n\t\t<true/>' )]) bot_utils.run(['zip', '-q', zipFile, infofile], env=os.environ) os.remove(infofile)
def InstallDartiumFromDartArchive(buildroot, buildout, buildos, gsu): """Install Dartium into the RCP zip files. Args: buildroot: the boot of the build output buildout: the location of the ant build output buildos: the OS the build is running under gsu: the gsutil wrapper Raises: Exception: if no dartium files can be found """ print 'InstallDartium(%s, %s, %s)' % (buildroot, buildout, buildos) tmp_dir = os.path.join(buildroot, 'tmp') revision = 'latest' if CHANNEL == 'be' else REVISION rcpZipFiles = _FindRcpZipFiles(buildout) for rcpZipFile in rcpZipFiles: print ' found rcp: %s' % rcpZipFile arch = 'ia32' system = None local_name = None if '-linux.gtk.x86.zip' in rcpZipFile: local_name = 'dartium-lucid32' system = 'linux' if '-linux.gtk.x86_64.zip' in rcpZipFile: local_name = 'dartium-lucid64' arch = 'x64' system = 'linux' if 'macosx' in rcpZipFile: local_name = 'dartium-mac' system = 'macos' if 'win32' in rcpZipFile: local_name = 'dartium-win' system = 'windows' namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW) dartiumFile = namer.dartium_variant_zipfilepath( revision, 'dartium', system, arch, 'release') # Download and unzip dartium unzip_dir = os.path.join( tmp_dir, os.path.splitext(os.path.basename(dartiumFile))[0]) if not os.path.exists(unzip_dir): os.makedirs(unzip_dir) # Always download as local_name.zip tmp_zip_file = os.path.join(tmp_dir, "%s.zip" % local_name) if not os.path.exists(tmp_zip_file): if gsu.Copy(dartiumFile, tmp_zip_file, False): raise Exception("gsutil command failed, aborting.") # Dartium is unzipped into unzip_dir/dartium-*/ dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos) dartium_zip.UnZip(unzip_dir) dart_zip_path = join(buildout, rcpZipFile) dart_zip = ziputils.ZipUtil(dart_zip_path, buildos) add_path = glob.glob(join(unzip_dir, 'dartium-*'))[0] if system == 'windows': # TODO(ricow/kustermann): This is hackisch. We should make a generic # black/white-listing mechanism. FileDelete(join(add_path, 'mini_installer.exe')) FileDelete(join(add_path, 'sync_unit_tests.exe')) FileDelete(join(add_path, 'chrome.packed.7z')) # Add dartium to the rcp zip dart_zip.AddDirectoryTree(add_path, 'dart/chromium') shutil.rmtree(tmp_dir, True)
def build_installer(): release_type = bot_utils.ReleaseType.SIGNED if CHANNEL == 'be': # We don't have signed bits on bleeding_edge, so we take the unsigned # editor. release_type = bot_utils.ReleaseType.RAW if SYSTEM == 'win': # The windows installer will use unsigned *.exe files to avoid going # through two rounds of signing. release_type = bot_utils.ReleaseType.RAW def editor_location(arch): namer = bot_utils.GCSNamer(CHANNEL, release_type) editor_path = namer.editor_zipfilepath(REVISION, SYSTEM, arch) return editor_path def create_windows_installer(installer_file, input_dir): # We add a README file to the installation. # The editor uses this to determine that we are a windows # installation. readme_file = os.path.join(input_dir, 'README-WIN') with open(readme_file, 'w') as fd: fd.write("This is the installation directory of the " "Dart Editor and the corresponding dart sdk.\n") msi_builder = os.path.join(DART_PATH, 'tools', 'create_windows_installer.py') wix_bin = os.path.join(DART_PATH, 'third_party', 'wix') version = utils.GetShortVersion() bot_utils.run([ 'python', msi_builder, '--msi_location=%s' % installer_file, '--input_directory=%s' % input_dir, '--version=%s' % version, '--wix_bin=%s' % wix_bin ]) def create_mac_installer(installer_file): dart_folder_icon = os.path.join( DART_PATH, 'editor/tools/plugins/com.google.dart.tools.ui/' + 'icons/dart_about_140_160.png') dmg_builder = os.path.join(DART_PATH, 'tools', 'mac_build_editor_dmg.sh') bot_utils.run([ dmg_builder, installer_file, 'dart', dart_folder_icon, "Dart Installer" ]) if SYSTEM == 'mac' or SYSTEM == 'win': for arch in ['32', '64']: extension = 'dmg' if SYSTEM == 'mac' else 'msi' with utils.TempDir('build_editor_installer') as temp_dir: with utils.ChangedWorkingDirectory(temp_dir): # Fetch the editor zip file from the new location. zip_location = os.path.join(temp_dir, 'editor.zip') if gsu.Copy(editor_location(arch), zip_location, False): raise Exception( "gsutil command failed, aborting.") # Unzip the editor (which contains a directory named 'dart'). editor_zip = ziputils.ZipUtil( zip_location, buildos) editor_zip.UnZip(temp_dir) unzip_dir = os.path.join(temp_dir, 'dart') assert os.path.exists('dart') and os.path.isdir( 'dart') installer_name = 'darteditor-installer.%s' % extension installer_file = os.path.join( temp_dir, installer_name) if SYSTEM == 'mac': create_mac_installer(installer_file) else: create_windows_installer( installer_file, unzip_dir) assert os.path.isfile(installer_file) # Archive to new bucket DartArchiveUploadInstaller( arch, installer_file, extension, release_type=release_type) else: raise Exception("We currently cannot build installers for %s" % SYSTEM)
def _InstallDartium(buildroot, buildout, buildos, gsu): """Install the SDk into the RCP zip files(s). Args: buildroot: the boot of the build output buildout: the location of the ant build output buildos: the OS the build is running under gsu: the gsutil wrapper Raises: Exception: if no dartium files can be found """ print '_InstallDartium({0}, {1}, {2}, gsu)'.format(buildroot, buildout, buildos) file_types = ['linlucful', 'macmacful', 'winwinful'] tmp_zip_name = None try: # dartium-win-full-6091.6091.zip file_name_re = re.compile(r'dartium-([lmw].+)-(.+)-.+\.(\d+)M?.+') tmp_dir = os.path.join(buildroot, 'tmp') unzip_dir = os.path.join(tmp_dir, 'unzip_dartium') elements = gsu.ReadBucket('gs://dartium-archive/latest/*.zip') add_path = None if not elements: raise Exception("can''t find any dartium files") dartum_version = None for element in elements: base_name = os.path.basename(element) print 'processing {0} ({1})'.format(element, base_name) try: dartum_os = ' ' dartum_type = ' ' dartum_version = ' ' file_match = file_name_re.search(base_name) if file_match is not None: dartum_os = file_match.group(1) dartum_type = file_match.group(2) dartum_version = file_match.group(3) except IndexError: _PrintError( 'Regular Expression error processing {0}'.format(element)) key = buildos[:3] + dartum_os[:3] + dartum_type[:3] if key in file_types: tmp_zip_file = tempfile.NamedTemporaryFile(suffix='.zip', prefix='dartium', dir=tmp_dir, delete=False) tmp_zip_name = tmp_zip_file.name tmp_zip_file.close() gsu.Copy(element, tmp_zip_name, False) if not os.path.exists(unzip_dir): os.makedirs(unzip_dir) # Dartium is unzipped into something like unzip_dir/dartium-win-inc-7665.7665 dartium_zip = ziputils.ZipUtil(tmp_zip_name, buildos) dartium_zip.UnZip(unzip_dir) if 'lin' in buildos: paths = glob.glob(os.path.join(unzip_dir, 'dartium-*')) add_path = paths[0] zip_rel_path = 'dart/dart-sdk/chromium' if 'win' in buildos: paths = glob.glob(os.path.join(unzip_dir, 'dartium-*')) add_path = paths[0] zip_rel_path = 'dart/dart-sdk/chromium' # remove extra files os.remove(os.path.join(add_path, 'DumpRenderTree.exe')) os.remove(os.path.join(add_path, 'mini_installer.exe')) os.remove(os.path.join(add_path, 'sync_unit_tests.exe')) if 'mac' in buildos: paths = glob.glob(os.path.join(unzip_dir, 'dartium-*')) add_path = os.path.join(paths[0], 'Chromium.app') zip_rel_path = 'dart/dart-sdk/Chromium.app' if tmp_zip_name is not None and add_path is not None: files = _FindRcpZipFiles(buildout) for f in files: dart_zip_path = os.path.join(buildout, f) print('_installDartium: before ' '{0} is {1}'.format(dart_zip_path, os.path.getsize(dart_zip_path))) dart_zip = ziputils.ZipUtil(dart_zip_path, buildos) dart_zip.AddDirectoryTree(add_path, zip_rel_path) revision_properties = None try: revision_properties_path = os.path.join( tmp_dir, 'chromium.properties') revision_properties = open(revision_properties_path, 'w') revision_properties.write( 'chromium.version = {0}{1}'.format( dartum_version, os.linesep)) finally: revision_properties.close() dart_zip.AddFile(revision_properties_path, 'dart/dart-sdk/chromium.properties') print('_installDartium: after ' '{0} is {1}'.format(dart_zip_path, os.path.getsize(dart_zip_path))) else: msg = 'no Dartium files found' _PrintError(msg) raise Exception(msg) finally: if tmp_zip_name is not None: os.remove(tmp_zip_name)
def main(): """Main entry point for the build program.""" if not sys.argv: print 'Script pathname not known, giving up.' return 1 scriptdir = os.path.abspath(os.path.dirname(sys.argv[0])) global aclfile aclfile = os.path.join(scriptdir, 'acl.xml') editorpath = os.path.abspath(os.path.join(scriptdir, '..')) thirdpartypath = os.path.abspath( os.path.join(scriptdir, '..', '..', 'third_party')) toolspath = os.path.abspath(os.path.join(scriptdir, '..', '..', 'tools')) dartpath = os.path.abspath(os.path.join(scriptdir, '..', '..')) antpath = os.path.join(thirdpartypath, 'apache_ant', 'v1_7_1') bzip2libpath = os.path.join(thirdpartypath, 'bzip2') buildpath = os.path.join(editorpath, 'tools', 'features', 'com.google.dart.tools.deploy.feature_releng') utils = GetUtils(toolspath) buildos = utils.GuessOS() # TODO(devoncarew): remove this hardcoded e:\ path buildroot_parent = { 'linux': dartpath, 'macos': dartpath, 'win32': r'e:\tmp' } buildroot = os.path.join(buildroot_parent[buildos], 'build_root') os.chdir(buildpath) ant_property_file = None sdk_zip = None # gsutil tests # if 'lin' in buildos and not os.environ.get('DONT_RUN_GSUTIL_TESTS'): # gsutil_test = os.path.join(editorpath, 'build', './gsutilTest.py') # cmds = [sys.executable, gsutil_test] # print 'running gsutil tests' # sys.stdout.flush() # p = subprocess.Popen(cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # (out_stream, err_strteam) = p.communicate() # if p.returncode: # print 'gsutil tests:' # print 'stdout:' # print str(out_stream) # print '*' * 40 # print str(err_strteam) # print '*' * 40 try: ant_property_file = tempfile.NamedTemporaryFile(suffix='.property', prefix='AntProperties', delete=False) ant_property_file.close() extra_artifacts = tempfile.mkdtemp(prefix='ExtraArtifacts') ant = AntWrapper(ant_property_file.name, os.path.join(antpath, 'bin'), bzip2libpath) ant.RunAnt(os.getcwd(), '', '', '', '', '', '', buildos, ['-diagnostics']) parser = _BuildOptions() (options, args) = parser.parse_args() # Determine which targets to build. By default we build the "all" target. if args: print 'only options should be passed to this script' parser.print_help() return 2 if str(options.revision) == 'None': print 'missing revision option' parser.print_help() return 3 if str(options.name) == 'None': print 'missing builder name' parser.print_help() return 4 if str(options.out) == 'None': print 'missing output directory' parser.print_help() return 5 print 'buildos = {0}'.format(buildos) print 'scriptdir = {0}'.format(scriptdir) print 'editorpath = {0}'.format(editorpath) print 'thirdpartypath = {0}'.format(thirdpartypath) print 'toolspath = {0}'.format(toolspath) print 'antpath = {0}'.format(antpath) print 'bzip2libpath = {0}'.format(bzip2libpath) print 'buildpath = {0}'.format(buildpath) print 'buildroot = {0}'.format(buildroot) print 'dartpath = {0}'.format(dartpath) print 'revision(in) = |{0}|'.format(options.revision) #this code handles getting the revision on the developer machine #where it can be 123, 123M 123:125M print 'revision(in) = {0}|'.format(options.revision) revision = options.revision.rstrip() lastc = revision[-1] if lastc.isalpha(): revision = revision[0:-1] index = revision.find(':') if index > -1: revision = revision[0:index] print 'revision = |{0}|'.format(revision) buildout = os.path.abspath(options.out) print 'buildout = {0}'.format(buildout) sys.stdout.flush() #get user name if it does not start with chrome then deploy # to the test bucket otherwise deploy to the continuous bucket #I could not find any non-OS specific way to get the user under Python # so the environemnt variables 'USER' Linux and Mac and # 'USERNAME' Windows were used. username = os.environ.get('USER') if username is None: username = os.environ.get('USERNAME') if username is None: _PrintError('could not find the user name' ' tried environment variables' ' USER and USERNAME') return 6 build_skip_tests = os.environ.get('DART_SKIP_RUNNING_TESTS') sdk_environment = os.environ build_util = buildutil.BuildUtil(buildos, buildout, dartpath) if username.startswith('chrome'): to_bucket = 'gs://dart-editor-archive-continuous' staging_bucket = 'gs://dart-editor-build' run_sdk_build = True running_on_buildbot = True else: to_bucket = 'gs://dart-editor-archive-testing' staging_bucket = 'gs://dart-editor-archive-testing-staging' run_sdk_build = True running_on_buildbot = False sdk_environment['DART_LOCAL_BUILD'] = 'dart-editor-archive-testing' homegsutil = os.path.join(os.path.expanduser('~'), 'gsutil', 'gsutil') gsu = gsutil.GsUtil(False, homegsutil, running_on_buildbot=running_on_buildbot) print '@@@BUILD_STEP dart-ide dart clients: %s@@@' % options.name if sdk_environment.has_key('JAVA_HOME'): print 'JAVA_HOME = {0}'.format(str(sdk_environment['JAVA_HOME'])) builder_name = str(options.name) if (run_sdk_build and builder_name != 'dart-editor'): _PrintSeparator('running the build of the Dart SDK') sdkpath = build_util.SdkZipLocation() sdk_zip = os.path.join(sdkpath, 'dart-sdk.zip') sdk_dir = os.path.join(sdkpath, 'dart-sdk') #clean out the old zip file if (os.path.exists(sdk_zip)): os.remove(sdk_zip) dartbuildscript = os.path.join(toolspath, 'build.py') cmds = [ sys.executable, dartbuildscript, '--mode=release', 'create_sdk' ] cwd = os.getcwd() try: os.chdir(dartpath) print ' '.join(cmds) status = subprocess.call(cmds, env=sdk_environment) print 'sdk build returned ' + str(status) if status: _PrintError('the build of the SDK failed') return status finally: os.chdir(cwd) #create zip if it does not exist if (not os.path.exists(sdk_zip)): if (not os.path.exists(sdk_dir)): raise Exception( 'could not find dart-sdk directory ({0})'.format( sdk_dir)) localzip = ziputils.ZipUtil(sdk_zip, buildos, create_new=True) localzip.AddDirectoryTree(sdk_dir, write_path='dart-sdk', mode_in='w') sdk_zip = _CopySdk(buildos, revision, to_bucket, sdkpath, buildroot, gsu) if builder_name == 'dart-editor': buildos = None # else: # _PrintSeparator('new builder running on {0} is' # ' a place holder until the os specific builds' # ' are in place. This is a ' # 'normal termination'.format(builder_name)) # return 0 _PrintSeparator('running the build to produce the Zipped RCP' 's') #tell the ant script where to write the sdk zip file so it can #be expanded later status = ant.RunAnt('.', 'build_rcp.xml', revision, options.name, buildroot, buildout, editorpath, buildos, sdk_zip=sdk_zip, running_on_bot=running_on_buildbot, extra_artifacts=extra_artifacts) #the ant script writes a property file in a known location so #we can read it. properties = _ReadPropertyFile(buildos, ant_property_file.name) if not properties: raise Exception('no data was found in file {0}'.format( ant_property_file.name)) if status: if properties['build.runtime']: _PrintErrorLog(properties['build.runtime']) return status #For the dart-editor build, return at this point. #We don't need to install the sdk+dartium, run tests, or copy to google #storage. if not buildos: print 'skipping sdk and dartium steps for dart-editor build' return 0 sys.stdout.flush() #This is an override to for local testing force_run_install = os.environ.get('FORCE_RUN_INSTALL') if (force_run_install or (run_sdk_build and builder_name != 'dart-editor')): _InstallSdk(buildroot, buildout, buildos, sdk_zip) _InstallDartium(buildroot, buildout, buildos, gsu) if status: return status #process the os specific builds if buildos: found_zips = _FindRcpZipFiles(buildout) if not found_zips: _PrintError('could not find any zipped up RCP files.' ' The Ant build must have failed') return 7 _WriteTagFile(buildos, staging_bucket, revision, gsu) sys.stdout.flush() if not build_skip_tests: _PrintSeparator('Running the tests') junit_status = ant.RunAnt( '../com.google.dart.tools.tests.feature_releng', 'buildTests.xml', revision, options.name, buildroot, buildout, editorpath, buildos, extra_artifacts=extra_artifacts) properties = _ReadPropertyFile(buildos, ant_property_file.name) if buildos: _UploadTestHtml(buildout, to_bucket, revision, buildos, gsu) if junit_status: if properties['build.runtime']: #if there is a build.runtime and the status is not #zero see if there are any *.log entries _PrintErrorLog(properties['build.runtime']) else: junit_status = 0 if buildos: found_zips = _FindRcpZipFiles(buildout) _InstallArtifacts(buildout, buildos, extra_artifacts) (status, gs_objects) = _DeployToContinuous(buildos, to_bucket, found_zips, revision, gsu) if _ShouldMoveToLatest(staging_bucket, revision, gsu): _MoveContinuousToLatest(staging_bucket, to_bucket, revision, gsu) _CleanupStaging(staging_bucket, revision, gsu) return junit_status finally: if ant_property_file is not None: print 'cleaning up temp file {0}'.format(ant_property_file.name) os.remove(ant_property_file.name) if extra_artifacts: print 'cleaning up temp dir {0}'.format(extra_artifacts) shutil.rmtree(extra_artifacts) print 'cleaning up {0}'.format(buildroot) shutil.rmtree(buildroot, True) print 'Build Done'
def InstallDartium(buildroot, buildout, buildos, gsu): """Install Dartium into the RCP zip files. Args: buildroot: the boot of the build output buildout: the location of the ant build output buildos: the OS the build is running under gsu: the gsutil wrapper Raises: Exception: if no dartium files can be found """ print 'InstallDartium(%s, %s, %s)' % (buildroot, buildout, buildos) tmp_dir = os.path.join(buildroot, 'tmp') rcpZipFiles = _FindRcpZipFiles(buildout) for rcpZipFile in rcpZipFiles: print ' found rcp: %s' % rcpZipFile if TRUNK_BUILD: # we look for Dartium artifacts differently for trunk vs continuous builds gsPrefix = "gs://dartium-archive" dartiumFiles = [] dartiumFiles.append( "%s/dartium-mac-full-trunk/dartium-mac-full-trunk-%s.0.zip" % (gsPrefix, REVISION)) dartiumFiles.append( "%s/dartium-lucid32-full-trunk/dartium-lucid32-full-trunk-%s.0.zip" % (gsPrefix, REVISION)) dartiumFiles.append( "%s/dartium-lucid64-full-trunk/dartium-lucid64-full-trunk-%s.0.zip" % (gsPrefix, REVISION)) dartiumFiles.append( "%s/dartium-win-full-trunk/dartium-win-full-trunk-%s.0.zip" % (gsPrefix, REVISION)) else: # dartium-lucid32-full-9420.9420.zip # dartium-lucid64-full-9420.9420.zip # dartium-mac-full-9420.9420.zip # dartium-win-full-9420.9420.zip # exclude dartium-lucid64-full-trunk-9571.9571.zip dartiumFiles = gsu.ReadBucket( 'gs://dartium-archive/latest/dartium-*-full-[0-9]*.zip') if not dartiumFiles: raise Exception("could not find any dartium files") tempList = [] for dartiumFile in dartiumFiles: print ' found dartium: %s' % dartiumFile tempList.append(RemapDartiumUrl(dartiumFile)) dartiumFiles = tempList for rcpZipFile in rcpZipFiles: searchString = None # dart-editor-linux.gtk.x86.zip # dart-editor-linux.gtk.x86_64.zip # dart-editor-macosx.cocoa.x86.zip # dart-editor-macosx.cocoa.x86_64.zip # dart-editor-win32.win32.x86.zip # dart-editor-win32.win32.x86_64.zip if '-linux.gtk.x86.zip' in rcpZipFile: searchString = 'dartium-lucid32-full-' if '-linux.gtk.x86_64.zip' in rcpZipFile: searchString = 'dartium-lucid64-full-' if 'macosx' in rcpZipFile: searchString = 'dartium-mac-full-' if 'win32' in rcpZipFile: searchString = 'dartium-win-full-' for dartiumFile in dartiumFiles: if searchString in dartiumFile: #download and unzip dartium unzip_dir = os.path.join( tmp_dir, os.path.splitext(os.path.basename(dartiumFile))[0]) if not os.path.exists(unzip_dir): os.makedirs(unzip_dir) tmp_zip_file = os.path.join(tmp_dir, os.path.basename(dartiumFile)) if not os.path.exists(tmp_zip_file): gsu.Copy(dartiumFile, tmp_zip_file, False) # Dartium is unzipped into ~ unzip_dir/dartium-win-full-7665.7665 dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos) dartium_zip.UnZip(unzip_dir) else: dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos) add_path = None if 'lin' in buildos: paths = glob.glob(os.path.join(unzip_dir, 'dartium-*')) add_path = paths[0] zip_rel_path = 'dart/chromium' if 'win' in buildos: paths = glob.glob(os.path.join(unzip_dir, 'dartium-*')) add_path = paths[0] zip_rel_path = 'dart/chromium' # remove extra files FileDelete(os.path.join(add_path, 'DumpRenderTree.exe')) FileDelete(os.path.join(add_path, 'mini_installer.exe')) FileDelete(os.path.join(add_path, 'sync_unit_tests.exe')) if 'mac' in buildos: paths = glob.glob(os.path.join(unzip_dir, 'dartium-*')) add_path = os.path.join(paths[0], 'Chromium.app') zip_rel_path = 'dart/Chromium.app' #add to the rcp zip dart_zip_path = os.path.join(buildout, rcpZipFile) dart_zip = ziputils.ZipUtil(dart_zip_path, buildos) dart_zip.AddDirectoryTree(add_path, zip_rel_path) shutil.rmtree(tmp_dir, True)
def InstallDartium(buildroot, buildout, buildos, gsu): """Install Dartium into the RCP zip files and upload a version of Dartium Args: buildroot: the boot of the build output buildout: the location of the ant build output buildos: the OS the build is running under gsu: the gsutil wrapper Raises: Exception: if no dartium files can be found """ print 'InstallDartium(%s, %s, %s)' % (buildroot, buildout, buildos) tmp_dir = os.path.join(buildroot, 'tmp') rcpZipFiles = _FindRcpZipFiles(buildout) for rcpZipFile in rcpZipFiles: print ' found rcp: %s' % rcpZipFile dartiumFiles = [] if TRUNK_BUILD: dartiumFiles.append("gs://dartium-archive/dartium-mac-full-trunk/" + "dartium-mac-full-trunk-%s.0.zip" % REVISION) dartiumFiles.append("gs://dartium-archive/dartium-win-full-trunk/" + "dartium-win-full-trunk-%s.0.zip" % REVISION) dartiumFiles.append( "gs://dartium-archive/dartium-lucid32-full-trunk/" + "dartium-lucid32-full-trunk-%s.0.zip" % REVISION) dartiumFiles.append( "gs://dartium-archive/dartium-lucid64-full-trunk/" + "dartium-lucid64-full-trunk-%s.0.zip" % REVISION) elif MILESTONE_BUILD: dartiumFiles.append( "gs://dartium-archive/dartium-mac-full-milestone/" + "dartium-mac-full-milestone-%s.0.zip" % REVISION) dartiumFiles.append( "gs://dartium-archive/dartium-win-full-milestone/" + "dartium-win-full-milestone-%s.0.zip" % REVISION) dartiumFiles.append( "gs://dartium-archive/dartium-lucid32-full-milestone/" + "dartium-lucid32-full-milestone-%s.0.zip" % REVISION) dartiumFiles.append( "gs://dartium-archive/dartium-lucid64-full-milestone/" + "dartium-lucid64-full-milestone-%s.0.zip" % REVISION) else: dartiumFiles.append("gs://dartium-archive/continuous/dartium-mac.zip") dartiumFiles.append("gs://dartium-archive/continuous/dartium-win.zip") #There is not a dartium-lucid32-inc buildbot. #dartiumFiles.append("gs://dartium-archive/continuous/dartium-lucid32.zip") dartiumFiles.append( "gs://dartium-archive/continuous/dartium-lucid64.zip") for rcpZipFile in rcpZipFiles: searchString = None # dart-editor-linux.gtk.x86.zip, ... if '-linux.gtk.x86.zip' in rcpZipFile: searchString = 'dartium-lucid32' if '-linux.gtk.x86_64.zip' in rcpZipFile: searchString = 'dartium-lucid64' if 'macosx' in rcpZipFile: searchString = 'dartium-mac' if 'win32' in rcpZipFile: searchString = 'dartium-win' for dartiumFile in dartiumFiles: if searchString in dartiumFile: #download and unzip dartium unzip_dir = os.path.join( tmp_dir, os.path.splitext(os.path.basename(dartiumFile))[0]) if not os.path.exists(unzip_dir): os.makedirs(unzip_dir) # Always download as searchString.zip basename = "%s.zip" % searchString tmp_zip_file = os.path.join(tmp_dir, basename) if not os.path.exists(tmp_zip_file): gsu.Copy(dartiumFile, tmp_zip_file, False) # Upload dartium zip to make sure we have consistent dartium downloads UploadFile(tmp_zip_file) # Dartium is unzipped into ~ unzip_dir/dartium-win-full-7665.7665 dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos) dartium_zip.UnZip(unzip_dir) else: dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos) dart_zip_path = join(buildout, rcpZipFile) dart_zip = ziputils.ZipUtil(dart_zip_path, buildos) if 'lin' in buildos: paths = glob.glob(join(unzip_dir, 'dartium-*')) add_path = paths[0] zip_rel_path = 'dart/chromium' # add to the rcp zip dart_zip.AddDirectoryTree(add_path, zip_rel_path) if 'win' in buildos: paths = glob.glob(join(unzip_dir, 'dartium-*')) add_path = paths[0] zip_rel_path = 'dart/chromium' FileDelete(join(add_path, 'mini_installer.exe')) FileDelete(join(add_path, 'sync_unit_tests.exe')) FileDelete(join(add_path, 'chrome.packed.7z')) # add to the rcp zip dart_zip.AddDirectoryTree(add_path, zip_rel_path) if 'mac' in buildos: paths = glob.glob(join(unzip_dir, 'dartium-*')) add_path = paths[0] zip_rel_path = 'dart/chromium' # add to the rcp zip dart_zip.AddDirectoryTree(add_path, zip_rel_path) shutil.rmtree(tmp_dir, True)