def setUp(self):
"""Setup the test."""
self._iswindows = False
operating_system = platform.system()
if operating_system == 'Windows' or operating_system == 'Microsoft':
# On Windows Vista platform.system() can return "Microsoft" with some
# versions of Python, see http://bugs.python.org/issue1082 for details.
self._iswindows = True
username = os.environ.get('USER')
if username is None:
username = os.environ.get('USERNAME')
if username is None:
self.fail(
'could not find the user name tried environment variables'
' USER and USERNAME')
self.test_folder = 'test_folder-{0}-{1}'.format(
username, operating_system)
if username.startswith('chrome'):
self.running_on_buildbot = True
else:
self.running_on_buildbot = False
self._gsu = gsutil.GsUtil(running_on_buildbot=self.running_on_buildbot)
self._CleanFolder(self.test_folder)
self._SetupFolder(self.test_bucket, self.test_folder, self.build_count)
def main():
"""Main entry point for the Google Storage Cleanup."""
_PrintSeparator('main')
parser = _BuildOptions()
(options, args) = parser.parse_args()
if not args:
print 'At least one command must be specified'
parser.print_help()
sys.exit(1)
if args[0] == 'promote':
command = 'promote'
if options.revision is None:
print 'You must specify a --revision to specify which revision to promote'
parser.print_help()
sys.exit(3)
if options.continuous is None and options.integration is None:
print 'You must specify one of --continuous or --integration'
parser.print_help()
sys.exit(4)
if options.continuous and options.integration:
print 'continuous and integration can not be specified at the same time'
parser.print_help()
sys.exit(5)
elif args[0] == 'cleanup':
command = 'cleanup'
if options.keepcount is None:
print 'You must specify --keepcount'
parser.print_help()
sys.exit(6)
else:
print 'At least one command must be specified'
parser.print_help()
sys.exit(2)
gsu = gsutil.GsUtil(options.dryrun)
if options.continuous:
bucket_from = CONTINUOUS
bucket_to = INTEGRATION
elif options.integration:
bucket_from = INTEGRATION
bucket_to = RELEASE
if command == 'cleanup':
version_dirs = _ReadBucket(gsu, options.gsbucketuri)
_RemoveElements(gsu, version_dirs, options.keepcount)
elif command == 'promote':
_PromoteBuild(gsu, options.revision, bucket_from, bucket_to)
def main():
parser = _ParseOptions()
(options, args) = parser.parse_args()
# Determine which targets to build. By default we build the "all" target.
if args:
print 'only options should be passed to this script'
parser.print_help()
return 1
revision = options.revision
src_rev = options.srcrev
gsu = gsutil.GsUtil(running_on_buildbot=False)
elements = [
'DartBuild-linux.gtk.x86_64.zip', 'DartBuild-linux.gtk.x86.zip',
'DartBuild-macosx.cocoa.x86_64.zip', 'DartBuild-macosx.cocoa.x86.zip',
'DartBuild-win32.win32.x86_64.zip', 'DartBuild-win32.win32.x86.zip',
'dart-editor-linux.gtk.x86_64.zip', 'dart-editor-linux.gtk.x86.zip',
'dart-editor-macosx.cocoa.x86_64.zip',
'dart-editor-macosx.cocoa.x86.zip',
'dart-editor-win32.win32.x86_64.zip', 'dart-editor-win32.win32.x86.zip'
]
os_names = []
re_filename = re.compile(
r'^(dart-editor|DartBuild)-(\w.+)\.(\w.+)\.(\w+)\.zip')
for element in elements:
matcher = re_filename.match(element)
os_name = matcher.group(2)
if os_name in 'macosx':
os_name = os_name[:-1]
if os_name not in os_names:
os_names.append(os_name)
print os_name
gsu.Copy(
'gs://dart-editor-archive-continuous/{0}/{1}'.format(
src_rev, element), 'gs://dart-editor-archive-testing/'
'{0}/{1}'.format(revision, element))
gsu.SetAclFromFile(
'gs://dart-editor-archive-testing/'
'{0}/{1}'.format(revision, element), 'acl.xml')
print 'copied {0}'.format(element)
f = None
try:
f = tempfile.NamedTemporaryFile(suffix='.txt', prefix='tag')
for os_name in os_names:
gsu.Copy(
f.name, 'gs://dart-editor-archive-testing-staging/tags'
'/done-{0}-{1}'.format(revision, os_name))
finally:
if f is not None:
f.close()
def main():
"""Main entry point for the build program."""
if not sys.argv:
print 'Script pathname not known, giving up.'
return 1
buildos = sys.platform
scriptdir = os.path.dirname(sys.argv[0])
editorpath = os.path.abspath(os.path.join(scriptdir, '..'))
thirdpartypath = os.path.abspath(
os.path.join(scriptdir, '..', '..', 'third_party'))
toolspath = os.path.abspath(os.path.join(scriptdir, '..', '..', 'tools'))
dartpath = os.path.abspath(os.path.join(scriptdir, '..', '..'))
antpath = os.path.join(thirdpartypath, 'apache_ant', 'v1_7_1')
bzip2libpath = os.path.join(thirdpartypath, 'bzip2')
buildpath = os.path.join(editorpath, 'tools', 'features',
'com.google.dart.tools.deploy.feature_releng')
buildroot = os.path.join(editorpath, 'build_root')
print 'buildos = {0}'.format(buildos)
print 'scriptdir = {0}'.format(scriptdir)
print 'editorpath = {0}'.format(editorpath)
print 'thirdpartypath = {0}'.format(thirdpartypath)
print 'toolspath = {0}'.format(toolspath)
print 'antpath = {0}'.format(antpath)
print 'bzip2libpath = {0}'.format(bzip2libpath)
print 'buildpath = {0}'.format(buildpath)
print 'buildroot = {0}'.format(buildroot)
print 'dartpath = {0}'.format(dartpath)
os.chdir(buildpath)
homegsutil = os.path.join(os.path.expanduser('~'), 'gsutil', 'gsutil')
gsu = gsutil.GsUtil(False, homegsutil)
parser = _BuildOptions()
(options, args) = parser.parse_args()
# Determine which targets to build. By default we build the "all" target.
if args:
print 'only options should be passed to this script'
parser.print_help()
return 1
if str(options.revision) == 'None':
print 'missing revision option'
parser.print_help()
return 2
if str(options.name) == 'None':
print 'missing builder name'
parser.print_help()
return 2
if str(options.out) == 'None':
print 'missing output directory'
parser.print_help()
return 2
buildout = os.path.join(buildroot, options.out)
#get user name if it does not start with chrome then deploy
# to the test bucket otherwise deploy to the continuous bucket
#I could not find any non-OS specific way to get the user under Python
# so the environemnt variables 'USER' Linux and Mac and
# 'USERNAME' Windows were used.
username = os.environ.get('USER')
if username is None:
username = os.environ.get('USERNAME')
if username is None:
_PrintError('could not find the user name'
' tried environment variables'
' USER and USERNAME')
return 1
if username.startswith('chrome'):
to_bucket = 'gs://dart-editor-archive-continuous'
else:
to_bucket = 'gs://dart-editor-archive-testing'
print '@@@BUILD_STEP dart-ide dart clients: %s@@@' % options.name
builder_name = str(options.name)
_PrintSeparator('running the build of the Dart SDK')
dartbuildscript = os.path.join(toolspath, 'build.py')
cmds = [sys.executable, dartbuildscript, '--mode=release', 'create_sdk']
print ' '.join(cmds)
cwd = os.getcwd()
try:
os.chdir(dartpath)
status = subprocess.call(cmds)
if status:
_PrintError('the build of the SDK failed')
return status
finally:
os.chdir(cwd)
if builder_name == 'dart-editor':
buildos = None
else:
_PrintSeparator('new builder running on {0} is'
' terminating until os specific builds'
' are in place'.format(builder_name))
return 0
_PrintSeparator('running the build to produce the Zipped RCP' 's')
status = _RunAnt('.', 'build_rcp.xml', options.revision, options.name,
buildroot, buildout, editorpath, buildos)
property_file = os.path.join('/var/tmp/' + options.name +
'-build.properties')
#the ant script writes a property file in a known location so
#we can read it. This build script is currently not using any post
#processin
properties = _ReadPropertyFile(property_file)
if status and properties['build.runtime']:
_PrintErrorLog(properties['build.runtime'])
#This build script is currently not using any post processing
#so this line is commented out
# If the preprocessor needs to be run in the
# if not status and properties['build.tmp']:
# postProcessZips(properties['build.tmp'], buildout)
sys.stdout.flush()
if status:
return status
#if the build passed run the deploy artifacts
_PrintSeparator("Deploying the built RCP's to Google Storage")
status = _DeployArtifacts(buildout, to_bucket, properties['build.tmp'],
options.revision, gsu)
if status:
return status
_PrintSeparator("Setting the ACL'sfor the RCP's in Google Storage")
_SetAclOnArtifacts(to_bucket, [options.revision, 'latest'], gsu)
sys.stdout.flush()
_PrintSeparator('Running the tests')
status = _RunAnt('../com.google.dart.tools.tests.feature_releng',
'buildTests.xml', options.revision, options.name,
buildroot, buildout, editorpath, buildos)
properties = _ReadPropertyFile(property_file)
if status and properties['build.runtime']:
#if there is a build.runtime and the status is not
#zero see if there are any *.log entries
_PrintErrorLog(properties['build.runtime'])
return status
def main():
"""Main entry point for the build program."""
global BUILD_OS
global CHANNEL
global DART_PATH
global NO_UPLOAD
global PLUGINS_BUILD
global REVISION
global SYSTEM
global TOOLS_PATH
if not sys.argv:
print 'Script pathname not known, giving up.'
return 1
scriptdir = os.path.abspath(os.path.dirname(sys.argv[0]))
editorpath = os.path.abspath(os.path.join(scriptdir, '..'))
thirdpartypath = os.path.abspath(
os.path.join(scriptdir, '..', '..', 'third_party'))
toolspath = os.path.abspath(os.path.join(scriptdir, '..', '..', 'tools'))
dartpath = os.path.abspath(os.path.join(scriptdir, '..', '..'))
antpath = os.path.join(thirdpartypath, 'apache_ant', '1.8.4')
bzip2libpath = os.path.join(thirdpartypath, 'bzip2')
buildpath = os.path.join(editorpath, 'tools', 'features',
'com.google.dart.tools.deploy.feature_releng')
buildos = utils.GuessOS()
BUILD_OS = utils.GuessOS()
DART_PATH = dartpath
TOOLS_PATH = toolspath
if (os.environ.get('DART_NO_UPLOAD') is not None):
NO_UPLOAD = True
# TODO(devoncarew): remove this hardcoded e:\ path
buildroot_parent = {
'linux': dartpath,
'macos': dartpath,
'win32': r'e:\tmp'
}
buildroot = os.path.join(buildroot_parent[buildos], 'build_root')
os.chdir(buildpath)
ant_property_file = None
sdk_zip = None
try:
parser = BuildOptions()
(options, args) = parser.parse_args()
# Determine which targets to build. By default we build the "all" target.
if args:
print 'only options should be passed to this script'
parser.print_help()
return 2
if str(options.revision) == 'None':
print 'missing revision option'
parser.print_help()
return 3
if str(options.name) == 'None':
print 'missing builder name'
parser.print_help()
return 4
if str(options.out) == 'None':
print 'missing output directory'
parser.print_help()
return 5
print 'buildos = {0}'.format(buildos)
print 'scriptdir = {0}'.format(scriptdir)
print 'editorpath = {0}'.format(editorpath)
print 'thirdpartypath = {0}'.format(thirdpartypath)
print 'toolspath = {0}'.format(toolspath)
print 'antpath = {0}'.format(antpath)
print 'bzip2libpath = {0}'.format(bzip2libpath)
print 'buildpath = {0}'.format(buildpath)
print 'buildroot = {0}'.format(buildroot)
print 'dartpath = {0}'.format(dartpath)
print 'revision(in) = |{0}|'.format(options.revision)
#this code handles getting the revision on the developer machine
#where it can be 123, 123M 123:125M
print 'revision(in) = |{0}|'.format(options.revision)
revision = options.revision.rstrip()
lastc = revision[-1]
if lastc.isalpha():
revision = revision[0:-1]
index = revision.find(':')
if index > -1:
revision = revision[0:index]
print 'revision = |{0}|'.format(revision)
buildout = os.path.abspath(options.out)
print 'buildout = {0}'.format(buildout)
username = utils.GetUserName()
if not username:
print 'Could not find username'
return 6
build_skip_tests = os.environ.get('DART_SKIP_RUNNING_TESTS')
sdk_environment = os.environ
if sdk_environment.has_key('JAVA_HOME'):
print 'JAVA_HOME = {0}'.format(str(sdk_environment['JAVA_HOME']))
EDITOR_REGEXP = (r'^dart-editor(-(?P<installer>(installer)))?'
'(-(?P<system>(win|mac|linux)))?' +
'(-(?P<channel>(be|dev|stable)))?$')
builder_name = str(options.name)
match = re.match(EDITOR_REGEXP, builder_name)
if not match:
raise Exception("Buildername '%s' does not match pattern '%s'." %
(builder_name, EDITOR_REGEXP))
CHANNEL = match.groupdict()['channel'] or 'be'
SYSTEM = match.groupdict()['system']
BUILD_INSTALLER = bool(match.groupdict()['installer'])
PLUGINS_BUILD = SYSTEM is None
REVISION = revision
# Make sure the buildername and the options agree
assert BUILD_INSTALLER == options.build_installer
running_on_buildbot = username.startswith('chrome')
homegsutil = join(DART_PATH, 'third_party', 'gsutil', 'gsutil')
gsu = gsutil.GsUtil(False,
homegsutil,
running_on_buildbot=running_on_buildbot)
def build_installer():
release_type = bot_utils.ReleaseType.SIGNED
if CHANNEL == 'be':
# We don't have signed bits on bleeding_edge, so we take the unsigned
# editor.
release_type = bot_utils.ReleaseType.RAW
if SYSTEM == 'win':
# The windows installer will use unsigned *.exe files to avoid going
# through two rounds of signing.
release_type = bot_utils.ReleaseType.RAW
def editor_location(arch):
namer = bot_utils.GCSNamer(CHANNEL, release_type)
editor_path = namer.editor_zipfilepath(REVISION, SYSTEM, arch)
return editor_path
def create_windows_installer(installer_file, input_dir):
# We add a README file to the installation.
# The editor uses this to determine that we are a windows
# installation.
readme_file = os.path.join(input_dir, 'README-WIN')
with open(readme_file, 'w') as fd:
fd.write("This is the installation directory of the "
"Dart Editor and the corresponding dart sdk.\n")
msi_builder = os.path.join(DART_PATH, 'tools',
'create_windows_installer.py')
wix_bin = os.path.join(DART_PATH, 'third_party', 'wix')
version = utils.GetShortVersion()
bot_utils.run([
'python', msi_builder,
'--msi_location=%s' % installer_file,
'--input_directory=%s' % input_dir,
'--version=%s' % version,
'--wix_bin=%s' % wix_bin
])
def create_mac_installer(installer_file):
dart_folder_icon = os.path.join(
DART_PATH,
'editor/tools/plugins/com.google.dart.tools.ui/' +
'icons/dart_about_140_160.png')
dmg_builder = os.path.join(DART_PATH, 'tools',
'mac_build_editor_dmg.sh')
bot_utils.run([
dmg_builder, installer_file, 'dart', dart_folder_icon,
"Dart Installer"
])
if SYSTEM == 'mac' or SYSTEM == 'win':
for arch in ['32', '64']:
extension = 'dmg' if SYSTEM == 'mac' else 'msi'
with utils.TempDir('build_editor_installer') as temp_dir:
with utils.ChangedWorkingDirectory(temp_dir):
# Fetch the editor zip file from the new location.
zip_location = os.path.join(temp_dir, 'editor.zip')
if gsu.Copy(editor_location(arch), zip_location,
False):
raise Exception(
"gsutil command failed, aborting.")
# Unzip the editor (which contains a directory named 'dart').
editor_zip = ziputils.ZipUtil(
zip_location, buildos)
editor_zip.UnZip(temp_dir)
unzip_dir = os.path.join(temp_dir, 'dart')
assert os.path.exists('dart') and os.path.isdir(
'dart')
installer_name = 'darteditor-installer.%s' % extension
installer_file = os.path.join(
temp_dir, installer_name)
if SYSTEM == 'mac':
create_mac_installer(installer_file)
else:
create_windows_installer(
installer_file, unzip_dir)
assert os.path.isfile(installer_file)
# Archive to new bucket
DartArchiveUploadInstaller(
arch,
installer_file,
extension,
release_type=release_type)
else:
raise Exception("We currently cannot build installers for %s" %
SYSTEM)
def build_editor(sdk_zip):
ant_property_file = tempfile.NamedTemporaryFile(
suffix='.property', prefix='AntProperties', delete=False)
ant_property_file.close()
ant = AntWrapper(ant_property_file.name,
os.path.join(antpath, 'bin'), bzip2libpath)
ant.RunAnt(os.getcwd(), '', '', '', '', '', '', buildos,
['-diagnostics'])
if not os.path.exists(buildout):
os.makedirs(buildout)
# clean out old build artifacts
for f in os.listdir(buildout):
if ('dartsdk-' in f) or ('darteditor-' in f) or ('dart-editor-'
in f):
os.remove(join(buildout, f))
InstallDartium(buildroot, buildout, buildos, gsu)
if not PLUGINS_BUILD:
StartBuildStep('create_sdk')
EnsureDirectoryExists(buildout)
try:
sdk_zip = CreateSDK(buildout)
except:
BuildStepFailure()
if builder_name.startswith('dart-editor-linux'):
StartBuildStep('api_docs')
try:
CreateApiDocs(buildout)
except:
BuildStepFailure()
StartBuildStep(builder_name)
if PLUGINS_BUILD:
status = BuildUpdateSite(ant, revision, builder_name,
buildroot, buildout, editorpath,
buildos)
return status
with utils.TempDir('ExtraArtifacts') as extra_artifacts:
# Tell the ant script where to write the sdk zip file so it can
# be expanded later
status = ant.RunAnt('.',
'build_rcp.xml',
revision,
builder_name,
buildroot,
buildout,
editorpath,
buildos,
sdk_zip=sdk_zip,
running_on_bot=running_on_buildbot,
extra_artifacts=extra_artifacts)
# The ant script writes a property file in a known location so
# we can read it.
properties = ReadPropertyFile(buildos, ant_property_file.name)
if not properties:
raise Exception('no data was found in file {%s}' %
ant_property_file.name)
if status:
if properties['build.runtime']:
PrintErrorLog(properties['build.runtime'])
return status
# For the dart-editor build, return at this point.
# We don't need to install the sdk+dartium, run tests, or copy to google
# storage.
if not buildos:
print 'skipping sdk and dartium steps for dart-editor build'
return 0
# This is an override for local testing
force_run_install = os.environ.get('FORCE_RUN_INSTALL')
if force_run_install or (not PLUGINS_BUILD):
InstallSdk(buildroot, buildout, buildos, buildout)
InstallDartium(buildroot, buildout, buildos, gsu)
if status:
return status
if not build_skip_tests:
RunEditorTests(buildout, buildos)
if buildos:
StartBuildStep('upload_artifacts')
_InstallArtifacts(buildout, buildos, extra_artifacts)
# dart-editor-linux.gtk.x86.zip --> darteditor-linux-32.zip
RenameRcpZipFiles(buildout)
PostProcessEditorBuilds(buildout, buildos)
if running_on_buildbot:
version_file = _FindVersionFile(buildout)
if version_file:
DartArchiveUploadVersionFile(version_file)
found_zips = _FindRcpZipFiles(buildout)
for zipfile in found_zips:
DartArchiveUploadEditorZipFile(zipfile)
return 0
if BUILD_INSTALLER:
build_installer()
else:
build_editor(sdk_zip)
finally:
if ant_property_file is not None:
print 'cleaning up temp file {0}'.format(ant_property_file.name)
os.remove(ant_property_file.name)
print 'cleaning up {0}'.format(buildroot)
shutil.rmtree(buildroot, True)
print 'Build Done'
def main():
"""Main entry point for Google Storage Tools."""
parser = _BuildOptions()
(options, args) = parser.parse_args()
if not args:
print 'At least one command must be specified'
parser.print_help()
sys.exit(1)
if args[0] == 'promote':
command = 'promote'
if options.revision is None:
print 'You must specify a --revision to specify which revision to promote'
parser.print_help()
sys.exit(3)
if not (options.continuous or options.integration or options.testing
or options.trunk or options.internal):
print 'Specify --continuous, --integration, --testing, or --trunk'
parser.print_help()
sys.exit(4)
if options.continuous and options.integration:
print 'continuous and integration can not be specified at the same time'
parser.print_help()
sys.exit(5)
if (options.continuous or options.integration) and options.testing:
print """Warning --continuous or --integration and --testing are
specified. The --testing flag will take precedence and all data will
go to the testing bucket {0}""".format(TESTING)
elif args[0] == 'cleanup':
command = 'cleanup'
if options.keepcount is None:
print 'You must specify --keepcount'
parser.print_help()
sys.exit(6)
else:
print 'Invalid command specified: {0}. See help below'.format(args[0])
parser.print_help()
sys.exit(2)
gsu = gsutil.GsUtil(options.dryrun, options.gsutilloc)
if options.testing:
bucket_from = CONTINUOUS
bucket_to = TESTING
print """The --testing attribute is specified. All data will go to the
testing bucket {0}. Press enter to continue""".format(TESTING)
raw_input('Press Enter to continue')
elif options.continuous:
bucket_from = CONTINUOUS
bucket_to = INTEGRATION
elif options.trunk:
bucket_from = TRUNK
bucket_to = INTEGRATION
elif options.integration:
bucket_from = INTEGRATION
bucket_to = RELEASE
elif options.internal:
bucket_from = TRUNK
bucket_to = INTERNAL
if command == 'cleanup':
#if the testing flag is set remove the date from the testing bucket
if options.testing:
bucket = TESTING
#otherwise use the value passed as --gsbucketuri
else:
bucket = options.gsbucketuri
version_dirs = _ReadBucket(gsu, '{0}/{1}'.format(bucket, '[0-9]*'))
_RemoveElements(gsu, bucket, version_dirs, options.keepcount)
elif command == 'promote':
_PromoteBuild(options.revision, bucket_from, bucket_to)
def main():
"""Main entry point for the build program."""
if not sys.argv:
print 'Script pathname not known, giving up.'
return 1
scriptdir = os.path.abspath(os.path.dirname(sys.argv[0]))
global aclfile
aclfile = os.path.join(scriptdir, 'acl.xml')
editorpath = os.path.abspath(os.path.join(scriptdir, '..'))
thirdpartypath = os.path.abspath(
os.path.join(scriptdir, '..', '..', 'third_party'))
toolspath = os.path.abspath(os.path.join(scriptdir, '..', '..', 'tools'))
dartpath = os.path.abspath(os.path.join(scriptdir, '..', '..'))
antpath = os.path.join(thirdpartypath, 'apache_ant', 'v1_7_1')
bzip2libpath = os.path.join(thirdpartypath, 'bzip2')
buildpath = os.path.join(editorpath, 'tools', 'features',
'com.google.dart.tools.deploy.feature_releng')
utils = GetUtils(toolspath)
buildos = utils.GuessOS()
# TODO(devoncarew): remove this hardcoded e:\ path
buildroot_parent = {
'linux': dartpath,
'macos': dartpath,
'win32': r'e:\tmp'
}
buildroot = os.path.join(buildroot_parent[buildos], 'build_root')
os.chdir(buildpath)
ant_property_file = None
sdk_zip = None
# gsutil tests
# if 'lin' in buildos and not os.environ.get('DONT_RUN_GSUTIL_TESTS'):
# gsutil_test = os.path.join(editorpath, 'build', './gsutilTest.py')
# cmds = [sys.executable, gsutil_test]
# print 'running gsutil tests'
# sys.stdout.flush()
# p = subprocess.Popen(cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# (out_stream, err_strteam) = p.communicate()
# if p.returncode:
# print 'gsutil tests:'
# print 'stdout:'
# print str(out_stream)
# print '*' * 40
# print str(err_strteam)
# print '*' * 40
try:
ant_property_file = tempfile.NamedTemporaryFile(suffix='.property',
prefix='AntProperties',
delete=False)
ant_property_file.close()
extra_artifacts = tempfile.mkdtemp(prefix='ExtraArtifacts')
ant = AntWrapper(ant_property_file.name, os.path.join(antpath, 'bin'),
bzip2libpath)
ant.RunAnt(os.getcwd(), '', '', '', '', '', '', buildos,
['-diagnostics'])
parser = _BuildOptions()
(options, args) = parser.parse_args()
# Determine which targets to build. By default we build the "all" target.
if args:
print 'only options should be passed to this script'
parser.print_help()
return 2
if str(options.revision) == 'None':
print 'missing revision option'
parser.print_help()
return 3
if str(options.name) == 'None':
print 'missing builder name'
parser.print_help()
return 4
if str(options.out) == 'None':
print 'missing output directory'
parser.print_help()
return 5
print 'buildos = {0}'.format(buildos)
print 'scriptdir = {0}'.format(scriptdir)
print 'editorpath = {0}'.format(editorpath)
print 'thirdpartypath = {0}'.format(thirdpartypath)
print 'toolspath = {0}'.format(toolspath)
print 'antpath = {0}'.format(antpath)
print 'bzip2libpath = {0}'.format(bzip2libpath)
print 'buildpath = {0}'.format(buildpath)
print 'buildroot = {0}'.format(buildroot)
print 'dartpath = {0}'.format(dartpath)
print 'revision(in) = |{0}|'.format(options.revision)
#this code handles getting the revision on the developer machine
#where it can be 123, 123M 123:125M
print 'revision(in) = {0}|'.format(options.revision)
revision = options.revision.rstrip()
lastc = revision[-1]
if lastc.isalpha():
revision = revision[0:-1]
index = revision.find(':')
if index > -1:
revision = revision[0:index]
print 'revision = |{0}|'.format(revision)
buildout = os.path.abspath(options.out)
print 'buildout = {0}'.format(buildout)
sys.stdout.flush()
#get user name if it does not start with chrome then deploy
# to the test bucket otherwise deploy to the continuous bucket
#I could not find any non-OS specific way to get the user under Python
# so the environemnt variables 'USER' Linux and Mac and
# 'USERNAME' Windows were used.
username = os.environ.get('USER')
if username is None:
username = os.environ.get('USERNAME')
if username is None:
_PrintError('could not find the user name'
' tried environment variables'
' USER and USERNAME')
return 6
build_skip_tests = os.environ.get('DART_SKIP_RUNNING_TESTS')
sdk_environment = os.environ
build_util = buildutil.BuildUtil(buildos, buildout, dartpath)
if username.startswith('chrome'):
to_bucket = 'gs://dart-editor-archive-continuous'
staging_bucket = 'gs://dart-editor-build'
run_sdk_build = True
running_on_buildbot = True
else:
to_bucket = 'gs://dart-editor-archive-testing'
staging_bucket = 'gs://dart-editor-archive-testing-staging'
run_sdk_build = True
running_on_buildbot = False
sdk_environment['DART_LOCAL_BUILD'] = 'dart-editor-archive-testing'
homegsutil = os.path.join(os.path.expanduser('~'), 'gsutil', 'gsutil')
gsu = gsutil.GsUtil(False,
homegsutil,
running_on_buildbot=running_on_buildbot)
print '@@@BUILD_STEP dart-ide dart clients: %s@@@' % options.name
if sdk_environment.has_key('JAVA_HOME'):
print 'JAVA_HOME = {0}'.format(str(sdk_environment['JAVA_HOME']))
builder_name = str(options.name)
if (run_sdk_build and builder_name != 'dart-editor'):
_PrintSeparator('running the build of the Dart SDK')
sdkpath = build_util.SdkZipLocation()
sdk_zip = os.path.join(sdkpath, 'dart-sdk.zip')
sdk_dir = os.path.join(sdkpath, 'dart-sdk')
#clean out the old zip file
if (os.path.exists(sdk_zip)):
os.remove(sdk_zip)
dartbuildscript = os.path.join(toolspath, 'build.py')
cmds = [
sys.executable, dartbuildscript, '--mode=release', 'create_sdk'
]
cwd = os.getcwd()
try:
os.chdir(dartpath)
print ' '.join(cmds)
status = subprocess.call(cmds, env=sdk_environment)
print 'sdk build returned ' + str(status)
if status:
_PrintError('the build of the SDK failed')
return status
finally:
os.chdir(cwd)
#create zip if it does not exist
if (not os.path.exists(sdk_zip)):
if (not os.path.exists(sdk_dir)):
raise Exception(
'could not find dart-sdk directory ({0})'.format(
sdk_dir))
localzip = ziputils.ZipUtil(sdk_zip, buildos, create_new=True)
localzip.AddDirectoryTree(sdk_dir,
write_path='dart-sdk',
mode_in='w')
sdk_zip = _CopySdk(buildos, revision, to_bucket, sdkpath,
buildroot, gsu)
if builder_name == 'dart-editor':
buildos = None
# else:
# _PrintSeparator('new builder running on {0} is'
# ' a place holder until the os specific builds'
# ' are in place. This is a '
# 'normal termination'.format(builder_name))
# return 0
_PrintSeparator('running the build to produce the Zipped RCP' 's')
#tell the ant script where to write the sdk zip file so it can
#be expanded later
status = ant.RunAnt('.',
'build_rcp.xml',
revision,
options.name,
buildroot,
buildout,
editorpath,
buildos,
sdk_zip=sdk_zip,
running_on_bot=running_on_buildbot,
extra_artifacts=extra_artifacts)
#the ant script writes a property file in a known location so
#we can read it.
properties = _ReadPropertyFile(buildos, ant_property_file.name)
if not properties:
raise Exception('no data was found in file {0}'.format(
ant_property_file.name))
if status:
if properties['build.runtime']:
_PrintErrorLog(properties['build.runtime'])
return status
#For the dart-editor build, return at this point.
#We don't need to install the sdk+dartium, run tests, or copy to google
#storage.
if not buildos:
print 'skipping sdk and dartium steps for dart-editor build'
return 0
sys.stdout.flush()
#This is an override to for local testing
force_run_install = os.environ.get('FORCE_RUN_INSTALL')
if (force_run_install
or (run_sdk_build and builder_name != 'dart-editor')):
_InstallSdk(buildroot, buildout, buildos, sdk_zip)
_InstallDartium(buildroot, buildout, buildos, gsu)
if status:
return status
#process the os specific builds
if buildos:
found_zips = _FindRcpZipFiles(buildout)
if not found_zips:
_PrintError('could not find any zipped up RCP files.'
' The Ant build must have failed')
return 7
_WriteTagFile(buildos, staging_bucket, revision, gsu)
sys.stdout.flush()
if not build_skip_tests:
_PrintSeparator('Running the tests')
junit_status = ant.RunAnt(
'../com.google.dart.tools.tests.feature_releng',
'buildTests.xml',
revision,
options.name,
buildroot,
buildout,
editorpath,
buildos,
extra_artifacts=extra_artifacts)
properties = _ReadPropertyFile(buildos, ant_property_file.name)
if buildos:
_UploadTestHtml(buildout, to_bucket, revision, buildos, gsu)
if junit_status:
if properties['build.runtime']:
#if there is a build.runtime and the status is not
#zero see if there are any *.log entries
_PrintErrorLog(properties['build.runtime'])
else:
junit_status = 0
if buildos:
found_zips = _FindRcpZipFiles(buildout)
_InstallArtifacts(buildout, buildos, extra_artifacts)
(status, gs_objects) = _DeployToContinuous(buildos, to_bucket,
found_zips, revision,
gsu)
if _ShouldMoveToLatest(staging_bucket, revision, gsu):
_MoveContinuousToLatest(staging_bucket, to_bucket, revision,
gsu)
_CleanupStaging(staging_bucket, revision, gsu)
return junit_status
finally:
if ant_property_file is not None:
print 'cleaning up temp file {0}'.format(ant_property_file.name)
os.remove(ant_property_file.name)
if extra_artifacts:
print 'cleaning up temp dir {0}'.format(extra_artifacts)
shutil.rmtree(extra_artifacts)
print 'cleaning up {0}'.format(buildroot)
shutil.rmtree(buildroot, True)
print 'Build Done'
def main():
"""Main entry point for the build program."""
global BUILD_OS
global DART_PATH
global TOOLS_PATH
global GSU_PATH_REV
global GSU_API_DOCS_PATH
global GSU_PATH_LATEST
global REVISION
global TRUNK_BUILD
global PLUGINS_BUILD
global utils
if not sys.argv:
print 'Script pathname not known, giving up.'
return 1
scriptdir = os.path.abspath(os.path.dirname(sys.argv[0]))
editorpath = os.path.abspath(os.path.join(scriptdir, '..'))
thirdpartypath = os.path.abspath(
os.path.join(scriptdir, '..', '..', 'third_party'))
toolspath = os.path.abspath(os.path.join(scriptdir, '..', '..', 'tools'))
dartpath = os.path.abspath(os.path.join(scriptdir, '..', '..'))
antpath = os.path.join(thirdpartypath, 'apache_ant', 'v1_7_1')
bzip2libpath = os.path.join(thirdpartypath, 'bzip2')
buildpath = os.path.join(editorpath, 'tools', 'features',
'com.google.dart.tools.deploy.feature_releng')
utils = GetUtils(toolspath)
buildos = utils.GuessOS()
BUILD_OS = utils.GuessOS()
DART_PATH = dartpath
TOOLS_PATH = toolspath
# TODO(devoncarew): remove this hardcoded e:\ path
buildroot_parent = {
'linux': dartpath,
'macos': dartpath,
'win32': r'e:\tmp'
}
buildroot = os.path.join(buildroot_parent[buildos], 'build_root')
os.chdir(buildpath)
ant_property_file = None
sdk_zip = None
try:
ant_property_file = tempfile.NamedTemporaryFile(suffix='.property',
prefix='AntProperties',
delete=False)
ant_property_file.close()
extra_artifacts = tempfile.mkdtemp(prefix='ExtraArtifacts')
ant = AntWrapper(ant_property_file.name, os.path.join(antpath, 'bin'),
bzip2libpath)
ant.RunAnt(os.getcwd(), '', '', '', '', '', '', buildos,
['-diagnostics'])
parser = BuildOptions()
(options, args) = parser.parse_args()
# Determine which targets to build. By default we build the "all" target.
if args:
print 'only options should be passed to this script'
parser.print_help()
return 2
if str(options.revision) == 'None':
print 'missing revision option'
parser.print_help()
return 3
if str(options.name) == 'None':
print 'missing builder name'
parser.print_help()
return 4
if str(options.out) == 'None':
print 'missing output directory'
parser.print_help()
return 5
print 'buildos = {0}'.format(buildos)
print 'scriptdir = {0}'.format(scriptdir)
print 'editorpath = {0}'.format(editorpath)
print 'thirdpartypath = {0}'.format(thirdpartypath)
print 'toolspath = {0}'.format(toolspath)
print 'antpath = {0}'.format(antpath)
print 'bzip2libpath = {0}'.format(bzip2libpath)
print 'buildpath = {0}'.format(buildpath)
print 'buildroot = {0}'.format(buildroot)
print 'dartpath = {0}'.format(dartpath)
print 'revision(in) = |{0}|'.format(options.revision)
#this code handles getting the revision on the developer machine
#where it can be 123, 123M 123:125M
print 'revision(in) = |{0}|'.format(options.revision)
revision = options.revision.rstrip()
lastc = revision[-1]
if lastc.isalpha():
revision = revision[0:-1]
index = revision.find(':')
if index > -1:
revision = revision[0:index]
print 'revision = |{0}|'.format(revision)
buildout = os.path.abspath(options.out)
print 'buildout = {0}'.format(buildout)
if not os.path.exists(buildout):
os.makedirs(buildout)
# clean out old build artifacts
for f in os.listdir(buildout):
if ('dartsdk-' in f) or ('darteditor-' in f) or ('dart-editor-'
in f):
os.remove(join(buildout, f))
#get user name if it does not start with chrome then deploy
# to the test bucket otherwise deploy to the continuous bucket
#I could not find any non-OS specific way to get the user under Python
# so the environemnt variables 'USER' Linux and Mac and
# 'USERNAME' Windows were used.
username = os.environ.get('USER')
if username is None:
username = os.environ.get('USERNAME')
if username is None:
PrintError('Could not find username')
return 6
# dart-editor[-trunk], dart-editor-(win/mac/linux)[-trunk]
builder_name = str(options.name)
TRUNK_BUILD = builder_name.endswith("-trunk")
PLUGINS_BUILD = (builder_name == 'dart-editor'
or builder_name == 'dart-editor-trunk')
build_skip_tests = os.environ.get('DART_SKIP_RUNNING_TESTS')
sdk_environment = os.environ
if username.startswith('chrome'):
if TRUNK_BUILD:
to_bucket = 'gs://dart-editor-archive-trunk'
else:
to_bucket = 'gs://dart-editor-archive-continuous'
running_on_buildbot = True
else:
to_bucket = 'gs://dart-editor-archive-testing'
running_on_buildbot = False
sdk_environment['DART_LOCAL_BUILD'] = 'dart-editor-archive-testing'
REVISION = revision
GSU_PATH_REV = '%s/%s' % (to_bucket, REVISION)
GSU_PATH_LATEST = '%s/%s' % (to_bucket, 'latest')
GSU_API_DOCS_PATH = '%s/%s' % (GSU_API_DOCS_BUCKET, REVISION)
homegsutil = join(DART_PATH, 'third_party', 'gsutil', 'gsutil')
gsu = gsutil.GsUtil(False,
homegsutil,
running_on_buildbot=running_on_buildbot)
print '@@@BUILD_STEP dart-ide dart clients: %s@@@' % options.name
if sdk_environment.has_key('JAVA_HOME'):
print 'JAVA_HOME = {0}'.format(str(sdk_environment['JAVA_HOME']))
if not PLUGINS_BUILD:
PrintSeparator('running the build of the Dart SDK')
EnsureDirectoryExists(buildout)
sdk_zip = CreateSDK(buildout)
if builder_name.startswith('dart-editor-linux'):
CreateApiDocs(buildout)
if PLUGINS_BUILD:
BuildUpdateSite(ant, revision, options.name, buildroot, buildout,
editorpath, buildos)
return 0
PrintSeparator('running the build to produce the Zipped RCP' 's')
#tell the ant script where to write the sdk zip file so it can
#be expanded later
status = ant.RunAnt('.',
'build_rcp.xml',
revision,
options.name,
buildroot,
buildout,
editorpath,
buildos,
sdk_zip=sdk_zip,
running_on_bot=running_on_buildbot,
extra_artifacts=extra_artifacts)
#the ant script writes a property file in a known location so
#we can read it.
properties = ReadPropertyFile(buildos, ant_property_file.name)
if not properties:
raise Exception('no data was found in file {%s}' %
ant_property_file.name)
if status:
if properties['build.runtime']:
PrintErrorLog(properties['build.runtime'])
return status
#For the dart-editor build, return at this point.
#We don't need to install the sdk+dartium, run tests, or copy to google
#storage.
if not buildos:
print 'skipping sdk and dartium steps for dart-editor build'
return 0
#This is an override for local testing
force_run_install = os.environ.get('FORCE_RUN_INSTALL')
if force_run_install or (not PLUGINS_BUILD):
InstallSdk(buildroot, buildout, buildos, buildout)
InstallDartium(buildroot, buildout, buildos, gsu)
if status:
return status
if not build_skip_tests:
PrintSeparator('Running the tests')
junit_status = ant.RunAnt(
'../com.google.dart.tools.tests.feature_releng',
'buildTests.xml',
revision,
options.name,
buildroot,
buildout,
editorpath,
buildos,
extra_artifacts=extra_artifacts)
properties = ReadPropertyFile(buildos, ant_property_file.name)
if buildos:
UploadTestHtml(buildout, to_bucket, revision, buildos, gsu)
if junit_status:
if properties['build.runtime']:
#if there is a build.runtime and the status is not
#zero see if there are any *.log entries
PrintErrorLog(properties['build.runtime'])
else:
junit_status = 0
if buildos:
_InstallArtifacts(buildout, buildos, extra_artifacts)
# dart-editor-linux.gtk.x86.zip --> darteditor-linux-32.zip
RenameRcpZipFiles(buildout)
PostProcessEditorBuilds(buildout)
version_file = _FindVersionFile(buildout)
if version_file:
UploadFile(version_file, False)
found_zips = _FindRcpZipFiles(buildout)
for zipfile in found_zips:
UploadFile(zipfile)
return junit_status
finally:
if ant_property_file is not None:
print 'cleaning up temp file {0}'.format(ant_property_file.name)
os.remove(ant_property_file.name)
if extra_artifacts:
print 'cleaning up temp dir {0}'.format(extra_artifacts)
shutil.rmtree(extra_artifacts)
print 'cleaning up {0}'.format(buildroot)
shutil.rmtree(buildroot, True)
print 'Build Done'
