def _InstallSdk(buildroot, buildout, buildos, sdk):
"""Install the SDk into the RCP zip files(s).
Args:
buildroot: the boot of the build output
buildout: the location of the ant build output
buildos: the OS the build is running under
sdk: the name of the zipped up sdk
"""
print '_InstallSdk({0}, {1}, {2}, {3})'.format(buildroot, buildout,
buildos, sdk)
tmp_dir = os.path.join(buildroot, 'tmp')
unzip_dir = os.path.join(tmp_dir, 'unzip_sdk')
if not os.path.exists(unzip_dir):
os.makedirs(unzip_dir)
sdk_zip = ziputils.ZipUtil(sdk, buildos)
sdk_zip.UnZip(unzip_dir)
files = _FindRcpZipFiles(buildout)
for f in files:
dart_zip_path = os.path.join(buildout, f)
print('_installSdk: before '
'{0} is {1}'.format(dart_zip_path,
os.path.getsize(dart_zip_path)))
dart_zip = ziputils.ZipUtil(dart_zip_path, buildos)
dart_zip.AddDirectoryTree(unzip_dir, 'dart')
print('_installSdk: after '
'{0} is {1}'.format(dart_zip_path,
os.path.getsize(dart_zip_path)))
def RunEditorTests(buildout, buildos):
StartBuildStep('run_tests')
for editorArchive in _GetTestableRcpArchives(buildout):
with utils.TempDir('editor_') as tempDir:
print 'Running tests for %s...' % editorArchive
zipper = ziputils.ZipUtil(join(buildout, editorArchive), buildos)
zipper.UnZip(tempDir)
# before we run the editor, suppress any 'restore windows' dialogs
if sys.platform == 'darwin':
args = [
'defaults', 'write', 'org.eclipse.eclipse.savedState',
'NSQuitAlwaysKeepsWindows', '-bool', 'false'
]
subprocess.call(args, shell=IsWindows())
editorExecutable = GetEditorExecutable(join(tempDir, 'dart'))
args = [
editorExecutable, '--test', '--auto-exit', '-data',
join(tempDir, 'workspace')
]
# Issue 12638. Enable this as soon as we can run editor tests in xvfb
# again.
##if sys.platform == 'linux2':
## args = ['xvfb-run', '-a'] + args
# this can hang if a 32 bit jvm is not available on windows...
if subprocess.call(args, shell=IsWindows()):
BuildStepFailure()
def add_download_scripts(zipFile, arch):
shell_ending = {
'win': '.bat',
'linux': '.sh',
'mac': '.sh',
}[SYSTEM]
# We don't have 64-bit versions of dartium/content_shell for
# macos/windows.
if SYSTEM in ['mac', 'win']:
arch = '32'
namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RELEASE)
# We're adding download scripts to the chromium directory.
# The directory tree will look like this after that:
# dart/dart-sdk/bin/dart{,.exe}
# /chromium/download_contentshell.{sh,bat}
# /chromium/download_dartium_debug.{sh,bat}
# /chromium/download_file.dart
# Add download_file.dart helper utility to the zip file.
f = ziputils.ZipUtil(zipFile, buildos)
f.AddFile(join(DART_DIR, 'tools', 'dartium', 'download_file.dart'),
'dart/chromium/download_file.dart')
# Add content shell download script
contentshell_name = namer.dartium_variant_zipfilename(
'content_shell', SYSTEM, arch, 'release')
contentshell_download_script = join(scratch_dir,
'download_contentshell')
instantiate_download_script_template(
contentshell_download_script, {
'VAR_DESTINATION':
contentshell_name,
'VAR_DOWNLOAD_URL':
("http://dartlang.org/editor/update/channels/%s/%s/dartium/%s"
% (CHANNEL, REVISION, contentshell_name)),
})
f.AddFile(contentshell_download_script,
'dart/chromium/download_contentshell%s' % shell_ending)
# Add dartium debug download script
dartium_debug_name = namer.dartium_variant_zipfilename(
'dartium', SYSTEM, arch, 'debug')
dartium_download_script = join(scratch_dir,
'download_dartium_debug')
instantiate_download_script_template(
dartium_download_script, {
'VAR_DESTINATION':
dartium_debug_name,
'VAR_DOWNLOAD_URL':
("http://dartlang.org/editor/update/channels/%s/%s/dartium/%s"
% (CHANNEL, REVISION, dartium_debug_name)),
})
f.AddFile(dartium_download_script,
'dart/chromium/download_dartium_debug%s' % shell_ending)
def InstallSdk(buildroot, buildout, buildos, sdk_dir):
"""Install the SDK into the RCP zip files.
Args:
buildroot: the boot of the build output
buildout: the location of the ant build output
buildos: the OS the build is running under
sdk_dir: the directory containing the built SDKs
"""
print 'InstallSdk(%s, %s, %s, %s)' % (buildroot, buildout, buildos,
sdk_dir)
tmp_dir = os.path.join(buildroot, 'tmp')
unzip_dir_32 = os.path.join(tmp_dir, 'unzip_sdk_32')
if not os.path.exists(unzip_dir_32):
os.makedirs(unzip_dir_32)
unzip_dir_64 = os.path.join(tmp_dir, 'unzip_sdk_64')
if not os.path.exists(unzip_dir_64):
os.makedirs(unzip_dir_64)
sdk_zip = ziputils.ZipUtil(join(sdk_dir, "dartsdk-%s-32.zip" % buildos),
buildos)
sdk_zip.UnZip(unzip_dir_32)
sdk_zip = ziputils.ZipUtil(join(sdk_dir, "dartsdk-%s-64.zip" % buildos),
buildos)
sdk_zip.UnZip(unzip_dir_64)
files = _FindRcpZipFiles(buildout)
for f in files:
dart_zip_path = os.path.join(buildout, f)
dart_zip = ziputils.ZipUtil(dart_zip_path, buildos)
# dart-editor-macosx.cocoa.x86_64.zip
if '_64.zip' in f:
dart_zip.AddDirectoryTree(unzip_dir_64, 'dart')
else:
dart_zip.AddDirectoryTree(unzip_dir_32, 'dart')
def _InstallArtifacts(buildout, buildos, extra_artifacts):
"""Install extra build artifacts into the RCP zip files.
Args:
buildout: the location of the ant build output
buildos: the OS the build is running under
extra_artifacts: the directory containing the extra artifacts
"""
print '_InstallArtifacts({%s}, {%s}, {%s})' % (buildout, buildos,
extra_artifacts)
files = _FindRcpZipFiles(buildout)
for f in files:
dart_zip_path = os.path.join(buildout, f)
dart_zip = ziputils.ZipUtil(dart_zip_path, buildos)
dart_zip.AddDirectoryTree(extra_artifacts, 'dart')
def _InstallArtifacts(buildout, buildos, extra_artifacts):
"""Install the SDk into the RCP zip files(s).
Args:
buildout: the location of the ant build output
buildos: the OS the build is running under
extra_artifacts: the directoryt he extra artifacts are in
"""
print '_InstallArtifacts({0}, {1}, {2})'.format(buildout, buildos,
extra_artifacts)
files = _FindRcpZipFiles(buildout)
for f in files:
dart_zip_path = os.path.join(buildout, f)
dart_zip = ziputils.ZipUtil(dart_zip_path, buildos)
dart_zip.AddDirectoryTree(extra_artifacts, 'dart')
def PostProcessEditorBuilds(out_dir, buildos):
"""Post-process the created RCP builds"""
with utils.TempDir('editor_scratch') as scratch_dir:
def instantiate_download_script_template(destination, replacements):
"""Helper function for replacing variables in the
tools/dartium/download_shellscript_templates.{sh,bat} scripts. It will
write the final download script to [destination] after doing all
replacements given in the [replacements] dictionary."""
template_location = {
'win':
join(DART_DIR, 'tools', 'dartium',
'download_shellscript_template.bat'),
'linux':
join(DART_DIR, 'tools', 'dartium',
'download_shellscript_template.sh'),
'mac':
join(DART_DIR, 'tools', 'dartium',
'download_shellscript_template.sh'),
}[SYSTEM]
with open(template_location) as fd:
content = fd.read()
for key in replacements:
content = content.replace(key, replacements[key])
with open(destination, 'w') as fd:
fd.write(content)
# Make it executable if we are not on windows
if SYSTEM != 'win':
os.chmod(destination,
os.stat(destination).st_mode | stat.S_IEXEC)
def add_download_scripts(zipFile, arch):
shell_ending = {
'win': '.bat',
'linux': '.sh',
'mac': '.sh',
}[SYSTEM]
# We don't have 64-bit versions of dartium/content_shell for
# macos/windows.
if SYSTEM in ['mac', 'win']:
arch = '32'
namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RELEASE)
# We're adding download scripts to the chromium directory.
# The directory tree will look like this after that:
# dart/dart-sdk/bin/dart{,.exe}
# /chromium/download_contentshell.{sh,bat}
# /chromium/download_dartium_debug.{sh,bat}
# /chromium/download_file.dart
# Add download_file.dart helper utility to the zip file.
f = ziputils.ZipUtil(zipFile, buildos)
f.AddFile(join(DART_DIR, 'tools', 'dartium', 'download_file.dart'),
'dart/chromium/download_file.dart')
# Add content shell download script
contentshell_name = namer.dartium_variant_zipfilename(
'content_shell', SYSTEM, arch, 'release')
contentshell_download_script = join(scratch_dir,
'download_contentshell')
instantiate_download_script_template(
contentshell_download_script, {
'VAR_DESTINATION':
contentshell_name,
'VAR_DOWNLOAD_URL':
("http://dartlang.org/editor/update/channels/%s/%s/dartium/%s"
% (CHANNEL, REVISION, contentshell_name)),
})
f.AddFile(contentshell_download_script,
'dart/chromium/download_contentshell%s' % shell_ending)
# Add dartium debug download script
dartium_debug_name = namer.dartium_variant_zipfilename(
'dartium', SYSTEM, arch, 'debug')
dartium_download_script = join(scratch_dir,
'download_dartium_debug')
instantiate_download_script_template(
dartium_download_script, {
'VAR_DESTINATION':
dartium_debug_name,
'VAR_DOWNLOAD_URL':
("http://dartlang.org/editor/update/channels/%s/%s/dartium/%s"
% (CHANNEL, REVISION, dartium_debug_name)),
})
f.AddFile(dartium_download_script,
'dart/chromium/download_dartium_debug%s' % shell_ending)
# Create a editor.properties
editor_properties = os.path.join(scratch_dir, 'editor.properties')
with open(editor_properties, 'w') as fd:
fd.write("com.dart.tools.update.core.url=http://dartlang.org"
"/editor/update/channels/%s/\n" % CHANNEL)
for zipFile in _FindRcpZipFiles(out_dir):
basename = os.path.basename(zipFile)
is_64bit = basename.endswith('-64.zip')
print(' processing %s' % basename)
readme_file = join('dart', 'README')
if (basename.startswith('darteditor-win32-')):
seven_zip = os.path.join(DART_DIR, 'third_party', '7zip',
'7za.exe')
bot_utils.run([seven_zip, 'd', zipFile, readme_file],
env=os.environ)
else:
bot_utils.run(['zip', '-d', zipFile, readme_file],
env=os.environ)
# If we're on -dev/-stable build: add an editor.properties file
# pointing to the correct update location of the editor for the channel
# we're building for.
if CHANNEL != 'be':
f = ziputils.ZipUtil(zipFile, buildos)
f.AddFile(editor_properties, 'dart/editor.properties')
# Add a shell/bat script to download contentshell and dartium debug.
# (including the necessary tools/dartium/download_file.dart helper).
add_download_scripts(zipFile, '64' if is_64bit else '32')
# adjust memory params for 64 bit versions
if is_64bit:
if (basename.startswith('darteditor-macos-')):
inifile = join('dart', 'DartEditor.app', 'Contents',
'MacOS', 'DartEditor.ini')
else:
inifile = join('dart', 'DartEditor.ini')
if (basename.startswith('darteditor-win32-')):
f = zipfile.ZipFile(zipFile)
f.extract(inifile.replace('\\', '/'))
f.close()
else:
bot_utils.run(['unzip', zipFile, inifile], env=os.environ)
Modify64BitDartEditorIni(inifile)
if (basename.startswith('darteditor-win32-')):
seven_zip = os.path.join(DART_DIR, 'third_party', '7zip',
'7za.exe')
bot_utils.run([seven_zip, 'd', zipFile, inifile],
env=os.environ)
bot_utils.run([seven_zip, 'a', zipFile, inifile],
env=os.environ)
else:
bot_utils.run(['zip', '-d', zipFile, inifile],
env=os.environ)
bot_utils.run(['zip', '-q', zipFile, inifile],
env=os.environ)
os.remove(inifile)
# post-process the info.plist file
if (basename.startswith('darteditor-macos-')):
infofile = join('dart', 'DartEditor.app', 'Contents',
'Info.plist')
bot_utils.run(['unzip', zipFile, infofile], env=os.environ)
ReplaceInFiles([infofile], [(
'<dict>',
'<dict>\n\t<key>NSHighResolutionCapable</key>\n\t\t<true/>'
)])
bot_utils.run(['zip', '-q', zipFile, infofile], env=os.environ)
os.remove(infofile)
def InstallDartiumFromDartArchive(buildroot, buildout, buildos, gsu):
"""Install Dartium into the RCP zip files.
Args:
buildroot: the boot of the build output
buildout: the location of the ant build output
buildos: the OS the build is running under
gsu: the gsutil wrapper
Raises:
Exception: if no dartium files can be found
"""
print 'InstallDartium(%s, %s, %s)' % (buildroot, buildout, buildos)
tmp_dir = os.path.join(buildroot, 'tmp')
revision = 'latest' if CHANNEL == 'be' else REVISION
rcpZipFiles = _FindRcpZipFiles(buildout)
for rcpZipFile in rcpZipFiles:
print ' found rcp: %s' % rcpZipFile
arch = 'ia32'
system = None
local_name = None
if '-linux.gtk.x86.zip' in rcpZipFile:
local_name = 'dartium-lucid32'
system = 'linux'
if '-linux.gtk.x86_64.zip' in rcpZipFile:
local_name = 'dartium-lucid64'
arch = 'x64'
system = 'linux'
if 'macosx' in rcpZipFile:
local_name = 'dartium-mac'
system = 'macos'
if 'win32' in rcpZipFile:
local_name = 'dartium-win'
system = 'windows'
namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
dartiumFile = namer.dartium_variant_zipfilepath(
revision, 'dartium', system, arch, 'release')
# Download and unzip dartium
unzip_dir = os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(dartiumFile))[0])
if not os.path.exists(unzip_dir):
os.makedirs(unzip_dir)
# Always download as local_name.zip
tmp_zip_file = os.path.join(tmp_dir, "%s.zip" % local_name)
if not os.path.exists(tmp_zip_file):
if gsu.Copy(dartiumFile, tmp_zip_file, False):
raise Exception("gsutil command failed, aborting.")
# Dartium is unzipped into unzip_dir/dartium-*/
dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos)
dartium_zip.UnZip(unzip_dir)
dart_zip_path = join(buildout, rcpZipFile)
dart_zip = ziputils.ZipUtil(dart_zip_path, buildos)
add_path = glob.glob(join(unzip_dir, 'dartium-*'))[0]
if system == 'windows':
# TODO(ricow/kustermann): This is hackisch. We should make a generic
# black/white-listing mechanism.
FileDelete(join(add_path, 'mini_installer.exe'))
FileDelete(join(add_path, 'sync_unit_tests.exe'))
FileDelete(join(add_path, 'chrome.packed.7z'))
# Add dartium to the rcp zip
dart_zip.AddDirectoryTree(add_path, 'dart/chromium')
shutil.rmtree(tmp_dir, True)
def build_installer():
release_type = bot_utils.ReleaseType.SIGNED
if CHANNEL == 'be':
# We don't have signed bits on bleeding_edge, so we take the unsigned
# editor.
release_type = bot_utils.ReleaseType.RAW
if SYSTEM == 'win':
# The windows installer will use unsigned *.exe files to avoid going
# through two rounds of signing.
release_type = bot_utils.ReleaseType.RAW
def editor_location(arch):
namer = bot_utils.GCSNamer(CHANNEL, release_type)
editor_path = namer.editor_zipfilepath(REVISION, SYSTEM, arch)
return editor_path
def create_windows_installer(installer_file, input_dir):
# We add a README file to the installation.
# The editor uses this to determine that we are a windows
# installation.
readme_file = os.path.join(input_dir, 'README-WIN')
with open(readme_file, 'w') as fd:
fd.write("This is the installation directory of the "
"Dart Editor and the corresponding dart sdk.\n")
msi_builder = os.path.join(DART_PATH, 'tools',
'create_windows_installer.py')
wix_bin = os.path.join(DART_PATH, 'third_party', 'wix')
version = utils.GetShortVersion()
bot_utils.run([
'python', msi_builder,
'--msi_location=%s' % installer_file,
'--input_directory=%s' % input_dir,
'--version=%s' % version,
'--wix_bin=%s' % wix_bin
])
def create_mac_installer(installer_file):
dart_folder_icon = os.path.join(
DART_PATH,
'editor/tools/plugins/com.google.dart.tools.ui/' +
'icons/dart_about_140_160.png')
dmg_builder = os.path.join(DART_PATH, 'tools',
'mac_build_editor_dmg.sh')
bot_utils.run([
dmg_builder, installer_file, 'dart', dart_folder_icon,
"Dart Installer"
])
if SYSTEM == 'mac' or SYSTEM == 'win':
for arch in ['32', '64']:
extension = 'dmg' if SYSTEM == 'mac' else 'msi'
with utils.TempDir('build_editor_installer') as temp_dir:
with utils.ChangedWorkingDirectory(temp_dir):
# Fetch the editor zip file from the new location.
zip_location = os.path.join(temp_dir, 'editor.zip')
if gsu.Copy(editor_location(arch), zip_location,
False):
raise Exception(
"gsutil command failed, aborting.")
# Unzip the editor (which contains a directory named 'dart').
editor_zip = ziputils.ZipUtil(
zip_location, buildos)
editor_zip.UnZip(temp_dir)
unzip_dir = os.path.join(temp_dir, 'dart')
assert os.path.exists('dart') and os.path.isdir(
'dart')
installer_name = 'darteditor-installer.%s' % extension
installer_file = os.path.join(
temp_dir, installer_name)
if SYSTEM == 'mac':
create_mac_installer(installer_file)
else:
create_windows_installer(
installer_file, unzip_dir)
assert os.path.isfile(installer_file)
# Archive to new bucket
DartArchiveUploadInstaller(
arch,
installer_file,
extension,
release_type=release_type)
else:
raise Exception("We currently cannot build installers for %s" %
SYSTEM)
def _InstallDartium(buildroot, buildout, buildos, gsu):
"""Install the SDk into the RCP zip files(s).
Args:
buildroot: the boot of the build output
buildout: the location of the ant build output
buildos: the OS the build is running under
gsu: the gsutil wrapper
Raises:
Exception: if no dartium files can be found
"""
print '_InstallDartium({0}, {1}, {2}, gsu)'.format(buildroot, buildout,
buildos)
file_types = ['linlucful', 'macmacful', 'winwinful']
tmp_zip_name = None
try:
# dartium-win-full-6091.6091.zip
file_name_re = re.compile(r'dartium-([lmw].+)-(.+)-.+\.(\d+)M?.+')
tmp_dir = os.path.join(buildroot, 'tmp')
unzip_dir = os.path.join(tmp_dir, 'unzip_dartium')
elements = gsu.ReadBucket('gs://dartium-archive/latest/*.zip')
add_path = None
if not elements:
raise Exception("can''t find any dartium files")
dartum_version = None
for element in elements:
base_name = os.path.basename(element)
print 'processing {0} ({1})'.format(element, base_name)
try:
dartum_os = ' '
dartum_type = ' '
dartum_version = ' '
file_match = file_name_re.search(base_name)
if file_match is not None:
dartum_os = file_match.group(1)
dartum_type = file_match.group(2)
dartum_version = file_match.group(3)
except IndexError:
_PrintError(
'Regular Expression error processing {0}'.format(element))
key = buildos[:3] + dartum_os[:3] + dartum_type[:3]
if key in file_types:
tmp_zip_file = tempfile.NamedTemporaryFile(suffix='.zip',
prefix='dartium',
dir=tmp_dir,
delete=False)
tmp_zip_name = tmp_zip_file.name
tmp_zip_file.close()
gsu.Copy(element, tmp_zip_name, False)
if not os.path.exists(unzip_dir):
os.makedirs(unzip_dir)
# Dartium is unzipped into something like unzip_dir/dartium-win-inc-7665.7665
dartium_zip = ziputils.ZipUtil(tmp_zip_name, buildos)
dartium_zip.UnZip(unzip_dir)
if 'lin' in buildos:
paths = glob.glob(os.path.join(unzip_dir, 'dartium-*'))
add_path = paths[0]
zip_rel_path = 'dart/dart-sdk/chromium'
if 'win' in buildos:
paths = glob.glob(os.path.join(unzip_dir, 'dartium-*'))
add_path = paths[0]
zip_rel_path = 'dart/dart-sdk/chromium'
# remove extra files
os.remove(os.path.join(add_path, 'DumpRenderTree.exe'))
os.remove(os.path.join(add_path, 'mini_installer.exe'))
os.remove(os.path.join(add_path, 'sync_unit_tests.exe'))
if 'mac' in buildos:
paths = glob.glob(os.path.join(unzip_dir, 'dartium-*'))
add_path = os.path.join(paths[0], 'Chromium.app')
zip_rel_path = 'dart/dart-sdk/Chromium.app'
if tmp_zip_name is not None and add_path is not None:
files = _FindRcpZipFiles(buildout)
for f in files:
dart_zip_path = os.path.join(buildout, f)
print('_installDartium: before '
'{0} is {1}'.format(dart_zip_path,
os.path.getsize(dart_zip_path)))
dart_zip = ziputils.ZipUtil(dart_zip_path, buildos)
dart_zip.AddDirectoryTree(add_path, zip_rel_path)
revision_properties = None
try:
revision_properties_path = os.path.join(
tmp_dir, 'chromium.properties')
revision_properties = open(revision_properties_path, 'w')
revision_properties.write(
'chromium.version = {0}{1}'.format(
dartum_version, os.linesep))
finally:
revision_properties.close()
dart_zip.AddFile(revision_properties_path,
'dart/dart-sdk/chromium.properties')
print('_installDartium: after '
'{0} is {1}'.format(dart_zip_path,
os.path.getsize(dart_zip_path)))
else:
msg = 'no Dartium files found'
_PrintError(msg)
raise Exception(msg)
finally:
if tmp_zip_name is not None:
os.remove(tmp_zip_name)
def main():
"""Main entry point for the build program."""
if not sys.argv:
print 'Script pathname not known, giving up.'
return 1
scriptdir = os.path.abspath(os.path.dirname(sys.argv[0]))
global aclfile
aclfile = os.path.join(scriptdir, 'acl.xml')
editorpath = os.path.abspath(os.path.join(scriptdir, '..'))
thirdpartypath = os.path.abspath(
os.path.join(scriptdir, '..', '..', 'third_party'))
toolspath = os.path.abspath(os.path.join(scriptdir, '..', '..', 'tools'))
dartpath = os.path.abspath(os.path.join(scriptdir, '..', '..'))
antpath = os.path.join(thirdpartypath, 'apache_ant', 'v1_7_1')
bzip2libpath = os.path.join(thirdpartypath, 'bzip2')
buildpath = os.path.join(editorpath, 'tools', 'features',
'com.google.dart.tools.deploy.feature_releng')
utils = GetUtils(toolspath)
buildos = utils.GuessOS()
# TODO(devoncarew): remove this hardcoded e:\ path
buildroot_parent = {
'linux': dartpath,
'macos': dartpath,
'win32': r'e:\tmp'
}
buildroot = os.path.join(buildroot_parent[buildos], 'build_root')
os.chdir(buildpath)
ant_property_file = None
sdk_zip = None
# gsutil tests
# if 'lin' in buildos and not os.environ.get('DONT_RUN_GSUTIL_TESTS'):
# gsutil_test = os.path.join(editorpath, 'build', './gsutilTest.py')
# cmds = [sys.executable, gsutil_test]
# print 'running gsutil tests'
# sys.stdout.flush()
# p = subprocess.Popen(cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# (out_stream, err_strteam) = p.communicate()
# if p.returncode:
# print 'gsutil tests:'
# print 'stdout:'
# print str(out_stream)
# print '*' * 40
# print str(err_strteam)
# print '*' * 40
try:
ant_property_file = tempfile.NamedTemporaryFile(suffix='.property',
prefix='AntProperties',
delete=False)
ant_property_file.close()
extra_artifacts = tempfile.mkdtemp(prefix='ExtraArtifacts')
ant = AntWrapper(ant_property_file.name, os.path.join(antpath, 'bin'),
bzip2libpath)
ant.RunAnt(os.getcwd(), '', '', '', '', '', '', buildos,
['-diagnostics'])
parser = _BuildOptions()
(options, args) = parser.parse_args()
# Determine which targets to build. By default we build the "all" target.
if args:
print 'only options should be passed to this script'
parser.print_help()
return 2
if str(options.revision) == 'None':
print 'missing revision option'
parser.print_help()
return 3
if str(options.name) == 'None':
print 'missing builder name'
parser.print_help()
return 4
if str(options.out) == 'None':
print 'missing output directory'
parser.print_help()
return 5
print 'buildos = {0}'.format(buildos)
print 'scriptdir = {0}'.format(scriptdir)
print 'editorpath = {0}'.format(editorpath)
print 'thirdpartypath = {0}'.format(thirdpartypath)
print 'toolspath = {0}'.format(toolspath)
print 'antpath = {0}'.format(antpath)
print 'bzip2libpath = {0}'.format(bzip2libpath)
print 'buildpath = {0}'.format(buildpath)
print 'buildroot = {0}'.format(buildroot)
print 'dartpath = {0}'.format(dartpath)
print 'revision(in) = |{0}|'.format(options.revision)
#this code handles getting the revision on the developer machine
#where it can be 123, 123M 123:125M
print 'revision(in) = {0}|'.format(options.revision)
revision = options.revision.rstrip()
lastc = revision[-1]
if lastc.isalpha():
revision = revision[0:-1]
index = revision.find(':')
if index > -1:
revision = revision[0:index]
print 'revision = |{0}|'.format(revision)
buildout = os.path.abspath(options.out)
print 'buildout = {0}'.format(buildout)
sys.stdout.flush()
#get user name if it does not start with chrome then deploy
# to the test bucket otherwise deploy to the continuous bucket
#I could not find any non-OS specific way to get the user under Python
# so the environemnt variables 'USER' Linux and Mac and
# 'USERNAME' Windows were used.
username = os.environ.get('USER')
if username is None:
username = os.environ.get('USERNAME')
if username is None:
_PrintError('could not find the user name'
' tried environment variables'
' USER and USERNAME')
return 6
build_skip_tests = os.environ.get('DART_SKIP_RUNNING_TESTS')
sdk_environment = os.environ
build_util = buildutil.BuildUtil(buildos, buildout, dartpath)
if username.startswith('chrome'):
to_bucket = 'gs://dart-editor-archive-continuous'
staging_bucket = 'gs://dart-editor-build'
run_sdk_build = True
running_on_buildbot = True
else:
to_bucket = 'gs://dart-editor-archive-testing'
staging_bucket = 'gs://dart-editor-archive-testing-staging'
run_sdk_build = True
running_on_buildbot = False
sdk_environment['DART_LOCAL_BUILD'] = 'dart-editor-archive-testing'
homegsutil = os.path.join(os.path.expanduser('~'), 'gsutil', 'gsutil')
gsu = gsutil.GsUtil(False,
homegsutil,
running_on_buildbot=running_on_buildbot)
print '@@@BUILD_STEP dart-ide dart clients: %s@@@' % options.name
if sdk_environment.has_key('JAVA_HOME'):
print 'JAVA_HOME = {0}'.format(str(sdk_environment['JAVA_HOME']))
builder_name = str(options.name)
if (run_sdk_build and builder_name != 'dart-editor'):
_PrintSeparator('running the build of the Dart SDK')
sdkpath = build_util.SdkZipLocation()
sdk_zip = os.path.join(sdkpath, 'dart-sdk.zip')
sdk_dir = os.path.join(sdkpath, 'dart-sdk')
#clean out the old zip file
if (os.path.exists(sdk_zip)):
os.remove(sdk_zip)
dartbuildscript = os.path.join(toolspath, 'build.py')
cmds = [
sys.executable, dartbuildscript, '--mode=release', 'create_sdk'
]
cwd = os.getcwd()
try:
os.chdir(dartpath)
print ' '.join(cmds)
status = subprocess.call(cmds, env=sdk_environment)
print 'sdk build returned ' + str(status)
if status:
_PrintError('the build of the SDK failed')
return status
finally:
os.chdir(cwd)
#create zip if it does not exist
if (not os.path.exists(sdk_zip)):
if (not os.path.exists(sdk_dir)):
raise Exception(
'could not find dart-sdk directory ({0})'.format(
sdk_dir))
localzip = ziputils.ZipUtil(sdk_zip, buildos, create_new=True)
localzip.AddDirectoryTree(sdk_dir,
write_path='dart-sdk',
mode_in='w')
sdk_zip = _CopySdk(buildos, revision, to_bucket, sdkpath,
buildroot, gsu)
if builder_name == 'dart-editor':
buildos = None
# else:
# _PrintSeparator('new builder running on {0} is'
# ' a place holder until the os specific builds'
# ' are in place. This is a '
# 'normal termination'.format(builder_name))
# return 0
_PrintSeparator('running the build to produce the Zipped RCP' 's')
#tell the ant script where to write the sdk zip file so it can
#be expanded later
status = ant.RunAnt('.',
'build_rcp.xml',
revision,
options.name,
buildroot,
buildout,
editorpath,
buildos,
sdk_zip=sdk_zip,
running_on_bot=running_on_buildbot,
extra_artifacts=extra_artifacts)
#the ant script writes a property file in a known location so
#we can read it.
properties = _ReadPropertyFile(buildos, ant_property_file.name)
if not properties:
raise Exception('no data was found in file {0}'.format(
ant_property_file.name))
if status:
if properties['build.runtime']:
_PrintErrorLog(properties['build.runtime'])
return status
#For the dart-editor build, return at this point.
#We don't need to install the sdk+dartium, run tests, or copy to google
#storage.
if not buildos:
print 'skipping sdk and dartium steps for dart-editor build'
return 0
sys.stdout.flush()
#This is an override to for local testing
force_run_install = os.environ.get('FORCE_RUN_INSTALL')
if (force_run_install
or (run_sdk_build and builder_name != 'dart-editor')):
_InstallSdk(buildroot, buildout, buildos, sdk_zip)
_InstallDartium(buildroot, buildout, buildos, gsu)
if status:
return status
#process the os specific builds
if buildos:
found_zips = _FindRcpZipFiles(buildout)
if not found_zips:
_PrintError('could not find any zipped up RCP files.'
' The Ant build must have failed')
return 7
_WriteTagFile(buildos, staging_bucket, revision, gsu)
sys.stdout.flush()
if not build_skip_tests:
_PrintSeparator('Running the tests')
junit_status = ant.RunAnt(
'../com.google.dart.tools.tests.feature_releng',
'buildTests.xml',
revision,
options.name,
buildroot,
buildout,
editorpath,
buildos,
extra_artifacts=extra_artifacts)
properties = _ReadPropertyFile(buildos, ant_property_file.name)
if buildos:
_UploadTestHtml(buildout, to_bucket, revision, buildos, gsu)
if junit_status:
if properties['build.runtime']:
#if there is a build.runtime and the status is not
#zero see if there are any *.log entries
_PrintErrorLog(properties['build.runtime'])
else:
junit_status = 0
if buildos:
found_zips = _FindRcpZipFiles(buildout)
_InstallArtifacts(buildout, buildos, extra_artifacts)
(status, gs_objects) = _DeployToContinuous(buildos, to_bucket,
found_zips, revision,
gsu)
if _ShouldMoveToLatest(staging_bucket, revision, gsu):
_MoveContinuousToLatest(staging_bucket, to_bucket, revision,
gsu)
_CleanupStaging(staging_bucket, revision, gsu)
return junit_status
finally:
if ant_property_file is not None:
print 'cleaning up temp file {0}'.format(ant_property_file.name)
os.remove(ant_property_file.name)
if extra_artifacts:
print 'cleaning up temp dir {0}'.format(extra_artifacts)
shutil.rmtree(extra_artifacts)
print 'cleaning up {0}'.format(buildroot)
shutil.rmtree(buildroot, True)
print 'Build Done'
def InstallDartium(buildroot, buildout, buildos, gsu):
"""Install Dartium into the RCP zip files.
Args:
buildroot: the boot of the build output
buildout: the location of the ant build output
buildos: the OS the build is running under
gsu: the gsutil wrapper
Raises:
Exception: if no dartium files can be found
"""
print 'InstallDartium(%s, %s, %s)' % (buildroot, buildout, buildos)
tmp_dir = os.path.join(buildroot, 'tmp')
rcpZipFiles = _FindRcpZipFiles(buildout)
for rcpZipFile in rcpZipFiles:
print ' found rcp: %s' % rcpZipFile
if TRUNK_BUILD:
# we look for Dartium artifacts differently for trunk vs continuous builds
gsPrefix = "gs://dartium-archive"
dartiumFiles = []
dartiumFiles.append(
"%s/dartium-mac-full-trunk/dartium-mac-full-trunk-%s.0.zip" %
(gsPrefix, REVISION))
dartiumFiles.append(
"%s/dartium-lucid32-full-trunk/dartium-lucid32-full-trunk-%s.0.zip"
% (gsPrefix, REVISION))
dartiumFiles.append(
"%s/dartium-lucid64-full-trunk/dartium-lucid64-full-trunk-%s.0.zip"
% (gsPrefix, REVISION))
dartiumFiles.append(
"%s/dartium-win-full-trunk/dartium-win-full-trunk-%s.0.zip" %
(gsPrefix, REVISION))
else:
# dartium-lucid32-full-9420.9420.zip
# dartium-lucid64-full-9420.9420.zip
# dartium-mac-full-9420.9420.zip
# dartium-win-full-9420.9420.zip
# exclude dartium-lucid64-full-trunk-9571.9571.zip
dartiumFiles = gsu.ReadBucket(
'gs://dartium-archive/latest/dartium-*-full-[0-9]*.zip')
if not dartiumFiles:
raise Exception("could not find any dartium files")
tempList = []
for dartiumFile in dartiumFiles:
print ' found dartium: %s' % dartiumFile
tempList.append(RemapDartiumUrl(dartiumFile))
dartiumFiles = tempList
for rcpZipFile in rcpZipFiles:
searchString = None
# dart-editor-linux.gtk.x86.zip
# dart-editor-linux.gtk.x86_64.zip
# dart-editor-macosx.cocoa.x86.zip
# dart-editor-macosx.cocoa.x86_64.zip
# dart-editor-win32.win32.x86.zip
# dart-editor-win32.win32.x86_64.zip
if '-linux.gtk.x86.zip' in rcpZipFile:
searchString = 'dartium-lucid32-full-'
if '-linux.gtk.x86_64.zip' in rcpZipFile:
searchString = 'dartium-lucid64-full-'
if 'macosx' in rcpZipFile:
searchString = 'dartium-mac-full-'
if 'win32' in rcpZipFile:
searchString = 'dartium-win-full-'
for dartiumFile in dartiumFiles:
if searchString in dartiumFile:
#download and unzip dartium
unzip_dir = os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(dartiumFile))[0])
if not os.path.exists(unzip_dir):
os.makedirs(unzip_dir)
tmp_zip_file = os.path.join(tmp_dir,
os.path.basename(dartiumFile))
if not os.path.exists(tmp_zip_file):
gsu.Copy(dartiumFile, tmp_zip_file, False)
# Dartium is unzipped into ~ unzip_dir/dartium-win-full-7665.7665
dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos)
dartium_zip.UnZip(unzip_dir)
else:
dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos)
add_path = None
if 'lin' in buildos:
paths = glob.glob(os.path.join(unzip_dir, 'dartium-*'))
add_path = paths[0]
zip_rel_path = 'dart/chromium'
if 'win' in buildos:
paths = glob.glob(os.path.join(unzip_dir, 'dartium-*'))
add_path = paths[0]
zip_rel_path = 'dart/chromium'
# remove extra files
FileDelete(os.path.join(add_path, 'DumpRenderTree.exe'))
FileDelete(os.path.join(add_path, 'mini_installer.exe'))
FileDelete(os.path.join(add_path, 'sync_unit_tests.exe'))
if 'mac' in buildos:
paths = glob.glob(os.path.join(unzip_dir, 'dartium-*'))
add_path = os.path.join(paths[0], 'Chromium.app')
zip_rel_path = 'dart/Chromium.app'
#add to the rcp zip
dart_zip_path = os.path.join(buildout, rcpZipFile)
dart_zip = ziputils.ZipUtil(dart_zip_path, buildos)
dart_zip.AddDirectoryTree(add_path, zip_rel_path)
shutil.rmtree(tmp_dir, True)
def InstallDartium(buildroot, buildout, buildos, gsu):
"""Install Dartium into the RCP zip files and upload a version of Dartium
Args:
buildroot: the boot of the build output
buildout: the location of the ant build output
buildos: the OS the build is running under
gsu: the gsutil wrapper
Raises:
Exception: if no dartium files can be found
"""
print 'InstallDartium(%s, %s, %s)' % (buildroot, buildout, buildos)
tmp_dir = os.path.join(buildroot, 'tmp')
rcpZipFiles = _FindRcpZipFiles(buildout)
for rcpZipFile in rcpZipFiles:
print ' found rcp: %s' % rcpZipFile
dartiumFiles = []
if TRUNK_BUILD:
dartiumFiles.append("gs://dartium-archive/dartium-mac-full-trunk/" +
"dartium-mac-full-trunk-%s.0.zip" % REVISION)
dartiumFiles.append("gs://dartium-archive/dartium-win-full-trunk/" +
"dartium-win-full-trunk-%s.0.zip" % REVISION)
dartiumFiles.append(
"gs://dartium-archive/dartium-lucid32-full-trunk/" +
"dartium-lucid32-full-trunk-%s.0.zip" % REVISION)
dartiumFiles.append(
"gs://dartium-archive/dartium-lucid64-full-trunk/" +
"dartium-lucid64-full-trunk-%s.0.zip" % REVISION)
elif MILESTONE_BUILD:
dartiumFiles.append(
"gs://dartium-archive/dartium-mac-full-milestone/" +
"dartium-mac-full-milestone-%s.0.zip" % REVISION)
dartiumFiles.append(
"gs://dartium-archive/dartium-win-full-milestone/" +
"dartium-win-full-milestone-%s.0.zip" % REVISION)
dartiumFiles.append(
"gs://dartium-archive/dartium-lucid32-full-milestone/" +
"dartium-lucid32-full-milestone-%s.0.zip" % REVISION)
dartiumFiles.append(
"gs://dartium-archive/dartium-lucid64-full-milestone/" +
"dartium-lucid64-full-milestone-%s.0.zip" % REVISION)
else:
dartiumFiles.append("gs://dartium-archive/continuous/dartium-mac.zip")
dartiumFiles.append("gs://dartium-archive/continuous/dartium-win.zip")
#There is not a dartium-lucid32-inc buildbot.
#dartiumFiles.append("gs://dartium-archive/continuous/dartium-lucid32.zip")
dartiumFiles.append(
"gs://dartium-archive/continuous/dartium-lucid64.zip")
for rcpZipFile in rcpZipFiles:
searchString = None
# dart-editor-linux.gtk.x86.zip, ...
if '-linux.gtk.x86.zip' in rcpZipFile:
searchString = 'dartium-lucid32'
if '-linux.gtk.x86_64.zip' in rcpZipFile:
searchString = 'dartium-lucid64'
if 'macosx' in rcpZipFile:
searchString = 'dartium-mac'
if 'win32' in rcpZipFile:
searchString = 'dartium-win'
for dartiumFile in dartiumFiles:
if searchString in dartiumFile:
#download and unzip dartium
unzip_dir = os.path.join(
tmp_dir,
os.path.splitext(os.path.basename(dartiumFile))[0])
if not os.path.exists(unzip_dir):
os.makedirs(unzip_dir)
# Always download as searchString.zip
basename = "%s.zip" % searchString
tmp_zip_file = os.path.join(tmp_dir, basename)
if not os.path.exists(tmp_zip_file):
gsu.Copy(dartiumFile, tmp_zip_file, False)
# Upload dartium zip to make sure we have consistent dartium downloads
UploadFile(tmp_zip_file)
# Dartium is unzipped into ~ unzip_dir/dartium-win-full-7665.7665
dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos)
dartium_zip.UnZip(unzip_dir)
else:
dartium_zip = ziputils.ZipUtil(tmp_zip_file, buildos)
dart_zip_path = join(buildout, rcpZipFile)
dart_zip = ziputils.ZipUtil(dart_zip_path, buildos)
if 'lin' in buildos:
paths = glob.glob(join(unzip_dir, 'dartium-*'))
add_path = paths[0]
zip_rel_path = 'dart/chromium'
# add to the rcp zip
dart_zip.AddDirectoryTree(add_path, zip_rel_path)
if 'win' in buildos:
paths = glob.glob(join(unzip_dir, 'dartium-*'))
add_path = paths[0]
zip_rel_path = 'dart/chromium'
FileDelete(join(add_path, 'mini_installer.exe'))
FileDelete(join(add_path, 'sync_unit_tests.exe'))
FileDelete(join(add_path, 'chrome.packed.7z'))
# add to the rcp zip
dart_zip.AddDirectoryTree(add_path, zip_rel_path)
if 'mac' in buildos:
paths = glob.glob(join(unzip_dir, 'dartium-*'))
add_path = paths[0]
zip_rel_path = 'dart/chromium'
# add to the rcp zip
dart_zip.AddDirectoryTree(add_path, zip_rel_path)
shutil.rmtree(tmp_dir, True)
