def _archive(self, fletch_vm, coredumps):
assert coredumps
files = [fletch_vm] + coredumps
for filename in files:
assert os.path.exists(filename)
gsutil = bot_utils.GSUtil()
storage_path = '%s/%s/' % (self._bucket, uuid.uuid4())
gs_prefix = 'gs://%s' % storage_path
http_prefix = 'https://storage.cloud.google.com/%s' % storage_path
for filename in files:
gs_url = '%s%s' % (gs_prefix, filename)
http_url = '%s%s' % (http_prefix, filename)
try:
gsutil.upload(filename, gs_url)
print '@@@STEP_LOG_LINE@coredumps@%s (%s)@@@' % (gs_url, http_url)
except Exception as error:
message = "Failed to upload coredump %s, error: %s" % (filename, error)
print '@@@STEP_LOG_LINE@coredumps@%s@@@' % message
for filename in coredumps:
os.remove(filename)
print '@@@STEP_LOG_END@coredumps@@@'
MarkCurrentStep(fatal=False)
def StepsCreateArchiveRaspbianImge():
EnsureRaspbianBase()
with bot.BuildStep('Modifying raspbian image'):
namer = GetNamer(temporary=IsBleedingEdge())
raspbian_src = os.path.join('third_party', 'raspbian', 'image',
'jessie.img')
raspbian_dst = os.path.join('out', namer.raspbian_filename())
print 'Copying %s to %s' % (raspbian_src, raspbian_dst)
shutil.copyfile(raspbian_src, raspbian_dst)
version = utils.GetSemanticSDKVersion()
deb_file = os.path.join('out', namer.arm_agent_filename(version))
src_file = os.path.join('out', namer.src_tar_name(version))
Run([
'tools/raspberry-pi2/raspbian_prepare.py',
'--image=%s' % raspbian_dst,
'--agent=%s' % deb_file,
'--src=%s' % src_file
])
zip_file = os.path.join('out', namer.raspbian_zipfilename())
if os.path.exists(zip_file):
os.remove(zip_file)
CreateZip(raspbian_dst, namer.raspbian_zipfilename())
gsutil = bot_utils.GSUtil()
gs_path = namer.raspbian_zipfilepath(version)
http_path = GetDownloadLink(gs_path)
gsutil.upload(zip_file, gs_path, public=True)
print '@@@STEP_LINK@download@%s@@@' % http_path
def Archive(gcs_name, vm_path, link_name): download_link = 'https://storage.googleapis.com/%s' % gcs_name gcs_path = 'gs://%s' % gcs_name gsutil = bot_utils.GSUtil() gsutil.upload(vm_path, gcs_path) print '@@@STEP_LINK@download %s@%s@@@' % (link_name, download_link) sys.stdout.flush()
def UploadApiDocs(dir_name):
apidocs_namer = bot_utils.GCSNamerApiDocs(CHANNEL)
revision = utils.GetSVNRevision()
apidocs_destination_gcsdir = apidocs_namer.docs_dirpath(revision)
apidocs_destination_latestfile = apidocs_namer.docs_latestpath(revision)
# Return early if the documents have already been uploaded.
# (This can happen if a build was forced, or a commit had no changes in the
# dart repository (e.g. DEPS file update).)
if GsutilExists(apidocs_destination_gcsdir):
print("Not uploading api docs, since %s is already present." %
apidocs_destination_gcsdir)
return
# Upload everything inside the built apidocs directory.
gsutil = bot_utils.GSUtil()
gsutil.upload(dir_name,
apidocs_destination_gcsdir,
recursive=True,
public=True)
# Update latest.txt to contain the newest revision.
with utils.TempDir('latest_file') as temp_dir:
latest_file = os.path.join(temp_dir, 'latest.txt')
with open(latest_file, 'w') as f:
f.write('%s' % revision)
DartArchiveFile(latest_file, apidocs_destination_latestfile)
def _archive(self, driver, dartino_vm, coredumps):
assert coredumps
files = [driver, dartino_vm] + coredumps
for filename in files:
assert os.path.exists(filename)
gsutil = bot_utils.GSUtil()
storage_path = '%s/%s/' % (self._bucket, uuid.uuid4())
gs_prefix = 'gs://%s' % storage_path
http_prefix = 'https://storage.cloud.google.com/%s' % storage_path
for filename in files:
# Remove / from absolute path to not have // in gs path.
gs_url = '%s%s' % (gs_prefix, filename.lstrip('/'))
http_url = '%s%s' % (http_prefix, filename.lstrip('/'))
try:
gsutil.upload(filename, gs_url)
print '@@@STEP_LOG_LINE@coredumps@%s (%s)@@@' % (gs_url,
http_url)
except Exception as error:
message = "Failed to upload coredump %s, error: %s" % (
filename, error)
print '@@@STEP_LOG_LINE@coredumps@%s@@@' % message
print '@@@STEP_LOG_END@coredumps@@@'
MarkCurrentStep(fatal=False)
def UploadAPKs(options):
with bot.BuildStep('Upload apk'):
revision = utils.GetSVNRevision()
namer = bot_utils.GCSNamer(internal=True)
# The version of gsutil we have on the bots is not new enough to support
# the acl set commands.
bot_utils.GSUtil.USE_DART_REPO_VERSION = True
gsutil = bot_utils.GSUtil()
# Archive dartuim
local = os.path.join(options.build_products_dir, APK_LOCATION)
# TODO(whesse): pass in arch and mode from reciepe
remote = namer.dartium_android_apk_filepath(revision,
'dartium-android', 'arm',
'release')
web_link_prefix = 'https://storage.cloud.google.com/'
dartium_link = string.replace(remote, 'gs://', web_link_prefix)
UploadSetACL(gsutil, local, remote)
print "Uploaded dartium, available from: %s" % dartium_link
# Archive content shell
local = os.path.join(options.build_products_dir, CS_LOCATION)
# TODO(whesse): pass in arch and mode from reciepe
remote = namer.dartium_android_apk_filepath(revision,
'content_shell-android',
'arm', 'release')
content_shell_link = string.replace(remote, 'gs://', web_link_prefix)
UploadSetACL(gsutil, local, remote)
print "Uploaded content shell, available from: %s" % content_shell_link
def FTSlave(config):
# Run SWTBot tests
if len(sys.argv) > 0:
scriptdir = os.path.dirname(sys.argv[0])
builddir = os.path.join(scriptdir, '..', '..', 'editor', 'build')
testScript = os.path.join(builddir, 'testswteditor.py')
cmd = [sys.executable, testScript]
try:
subprocess.call(cmd, shell=IsWindows())
except:
pass
# Prepare to run EggPlant tests
with bot.BuildStep('Fetching editor'):
revision = int(os.environ['BUILDBOT_GOT_REVISION'])
bot_name, _ = bot.GetBotName()
print bot_name
channel = bot_utils.GetChannelFromName(bot_name)
namer = bot_utils.GCSNamer(channel=channel)
system = config.system
if system == 'mac':
system = 'macos'
editor_path = namer.editor_zipfilepath(revision, system, 'x64')
gsutils = bot_utils.GSUtil()
editor_location = '/home/chrome-bot/Desktop'
if system == 'macos':
editor_location = '/Users/chrome-bot/Desktop'
local_path = os.path.join(editor_location, 'editor.zip')
if os.path.exists(local_path):
os.remove(local_path)
local_extracted = os.path.join(editor_location, 'dart')
shutil.rmtree(local_extracted, ignore_errors=True)
gsutils.execute(['cp', editor_path, local_path])
Run(['unzip', local_path, '-d', editor_location])
def GetLatestVersionFromGCS(channel):
namer = bot_utils.GCSNamer(channel=channel)
gsutil = bot_utils.GSUtil()
gcs_version_path = namer.version_filepath('latest')
print 'Getting latest version from: %s' % gcs_version_path
version_json = gsutil.cat(gcs_version_path)
version_map = json.loads(version_json)
return version_map['version']
def StepsGetDocs():
with bot.BuildStep('Get docs'):
version = utils.GetSemanticSDKVersion()
gsutil = bot_utils.GSUtil()
namer = GetNamer()
docs_out = os.path.join('out')
gs_path = namer.docs_filepath(version)
gsutil.execute(['-m', 'cp', '-r', gs_path, docs_out])
def StepsArchiveDebianPackage():
with bot.BuildStep('Archive arm agent deb'):
version = utils.GetSemanticSDKVersion()
namer = GetNamer()
gsutil = bot_utils.GSUtil()
deb_file = os.path.join('out', namer.arm_agent_filename(version))
gs_path = namer.arm_agent_filepath(version)
http_path = GetDownloadLink(gs_path)
gsutil.upload(deb_file, gs_path, public=True)
print '@@@STEP_LINK@download@%s@@@' % http_path
def StepsGetArmDeb():
with bot.BuildStep('Get agent deb'):
version = utils.GetSemanticSDKVersion()
gsutil = bot_utils.GSUtil()
namer = GetNamer()
deb_file = os.path.join('out', namer.arm_agent_filename(version))
gs_path = namer.arm_agent_filepath(version)
if os.path.exists(deb_file):
os.remove(deb_file)
gsutil.execute(['cp', gs_path, deb_file])
def StepsArchiveCrossCompileBundle(cross_mode, cross_arch):
with bot.BuildStep('Archive arm binaries %s' % cross_mode):
build_conf = GetConfigurationName(cross_mode, cross_arch, '', False)
version = utils.GetSemanticSDKVersion()
namer = GetNamer()
gsutil = bot_utils.GSUtil()
zip_file = namer.arm_binaries_zipfilename(cross_mode)
CreateZip(os.path.join('out', build_conf), zip_file)
gs_path = namer.arm_binaries_zipfilepath(version, cross_mode)
http_path = GetDownloadLink(gs_path)
gsutil.upload(os.path.join('out', zip_file), gs_path, public=True)
print '@@@STEP_LINK@download@%s@@@' % http_path
def ArchiveThirdPartyTool(name, zip_name, system, gs_path):
zip_file = os.path.join('out', zip_name)
if os.path.exists(zip_file):
os.remove(zip_file)
copy = os.path.join('out', name)
if os.path.exists(copy):
shutil.rmtree(copy)
src = os.path.join('third_party', name, system, name)
shutil.copytree(src, copy)
CreateZip(copy, zip_name)
gsutil = bot_utils.GSUtil()
gsutil.upload(zip_file, gs_path, public=True)
http_path = GetDownloadLink(gs_path)
print '@@@STEP_LINK@download@%s@@@' % http_path
def BuildSteps(build_info):
with bot.BuildStep('Upload VM to GCS'):
sdk_bin_path = utils.GetBuildSdkBin(build_info.system, build_info.mode,
build_info.arch)
revision = utils.GetGitRevision()
name = 'fletch-archive/patched_dart_sdks/%s/dart-vm-%s-%s' % (
revision, build_info.arch, build_info.system)
download_link = 'https://storage.googleapis.com/%s' % name
gcs_path = 'gs://%s' % name
vm_path = os.path.join(sdk_bin_path, 'dart')
gsutil = bot_utils.GSUtil()
gsutil.upload(vm_path, gcs_path)
print '@@@STEP_LINK@download@%s@@@' % download_link
sys.stdout.flush()
def DartArchiveFile(local_path, remote_path, checksum_files=False):
gsutil = bot_utils.GSUtil()
gsutil.upload(local_path, remote_path, public=True)
if checksum_files:
# 'local_path' may have a different filename than 'remote_path'. So we need
# to make sure the *.md5sum file contains the correct name.
assert '/' in remote_path and not remote_path.endswith('/')
mangled_filename = remote_path[remote_path.rfind('/') + 1:]
local_md5sum = bot_utils.CreateMD5ChecksumFile(local_path,
mangled_filename)
gsutil.upload(local_md5sum, remote_path + '.md5sum', public=True)
local_sha256 = bot_utils.CreateSha256ChecksumFile(
local_path, mangled_filename)
gsutil.upload(local_sha256, remote_path + '.sha256sum', public=True)
def StepsGetArmBinaries(cross_mode, cross_arch):
with bot.BuildStep('Get arm binaries %s' % cross_mode):
build_conf = GetConfigurationName(cross_mode, cross_arch, '', False)
build_dir = os.path.join('out', build_conf)
version = utils.GetSemanticSDKVersion()
gsutil = bot_utils.GSUtil()
namer = GetNamer()
zip_file = os.path.join('out',
namer.arm_binaries_zipfilename(cross_mode))
if os.path.exists(zip_file):
os.remove(zip_file)
if os.path.exists(build_dir):
shutil.rmtree(build_dir)
gs_path = namer.arm_binaries_zipfilepath(version, cross_mode)
gsutil.execute(['cp', gs_path, zip_file])
Unzip(zip_file)
def UploadDartdocApiDocs(dir_name):
apidocs_namer = bot_utils.GCSNamerApiDocs(CHANNEL)
revision = utils.GetArchiveVersion()
dartdocs_destination_gcsdir = apidocs_namer.dartdocs_dirpath(revision)
# Return early if the documents have already been uploaded.
# This can happen if a build was forced, or a commit had no changes in the
# dart repository (e.g. DEPS file update).
if GsutilExists(dartdocs_destination_gcsdir):
print ("Not uploading api docs, since %s is already present."
% dartdocs_destination_gcsdir)
return
# Upload everything inside the built apidocs directory.
gsutil = bot_utils.GSUtil()
gsutil.upload(dir_name, dartdocs_destination_gcsdir, recursive=True,
public=True, multithread=True)
def ArchiveArtifacts(tarfile, builddir, channel):
namer = bot_utils.GCSNamer(channel=channel)
gsutil = bot_utils.GSUtil()
revision = utils.GetArchiveVersion()
# Archive the src tar to the src dir
remote_tarfile = '/'.join(
[namer.src_directory(revision),
os.path.basename(tarfile)])
gsutil.upload(tarfile, remote_tarfile, public=True)
# Archive all files except the tar file to the linux packages dir
for entry in os.listdir(builddir):
full_path = os.path.join(builddir, entry)
# We expect a flat structure, not subdirectories
assert (os.path.isfile(full_path))
if full_path != tarfile:
package_dir = namer.linux_packages_directory(revision)
remote_file = '/'.join([package_dir, os.path.basename(entry)])
gsutil.upload(full_path, remote_file, public=True)
def UploadAPKs(options):
with bot.BuildStep('Upload apk'):
revision = utils.GetSVNRevision()
namer = bot_utils.GCSNamer()
gsutil = bot_utils.GSUtil()
web_link_prefix = 'https://storage.cloud.google.com/'
# Archive content shell
local = os.path.join(options.build_products_dir, CS_LOCATION)
# TODO(whesse): pass in arch and mode from reciepe
remote = namer.dartium_android_apk_filepath(revision,
'content_shell-android',
'arm',
'release')
content_shell_link = string.replace(remote, 'gs://', web_link_prefix)
UploadSetACL(gsutil, local, remote)
print "Uploaded content shell, available from: %s" % content_shell_link
def StepsArchiveSDK(build_dir, system, mode, arch):
with bot.BuildStep('Archive bundle %s' % build_dir):
sdk = os.path.join(build_dir, 'dartino-sdk')
zip_file = 'dartino-sdk.zip'
CreateZip(sdk, zip_file)
version = utils.GetSemanticSDKVersion()
namer = GetNamer()
gsutil = bot_utils.GSUtil()
gs_path = namer.dartino_sdk_zipfilepath(version, system, arch, mode)
http_path = GetDownloadLink(gs_path)
gsutil.upload(os.path.join(build_dir, zip_file), gs_path, public=True)
print '@@@STEP_LINK@download@%s@@@' % http_path
docs_gs_path = namer.docs_filepath(version)
gsutil.upload(os.path.join('out', 'docs'),
docs_gs_path,
recursive=True,
public=True)
docs_http_path = '%s/%s' % (GetDownloadLink(docs_gs_path),
'index.html')
print '@@@STEP_LINK@docs@%s@@@' % docs_http_path
def FTSlave(config):
with bot.BuildStep('Fetching editor'):
revision = int(os.environ['BUILDBOT_GOT_REVISION'])
bot_name, _ = bot.GetBotName()
print bot_name
channel = bot_utils.GetChannelFromName(bot_name)
namer = bot_utils.GCSNamer(channel=channel)
system = config.system
if system == 'mac':
system = 'macos'
editor_path = namer.editor_zipfilepath(revision, system, 'x64')
gsutils = bot_utils.GSUtil()
editor_location = '/home/chrome-bot/Desktop'
if system == 'macos':
editor_location = '/Users/chrome-bot/Desktop'
local_path = os.path.join(editor_location, 'editor.zip')
if os.path.exists(local_path):
os.remove(local_path)
local_extracted = os.path.join(editor_location, 'dart')
shutil.rmtree(local_extracted, ignore_errors=True)
gsutils.execute(['cp', editor_path, local_path])
Run(['unzip', local_path, '-d', editor_location])
def UploadAPKs(options):
with bot.BuildStep('Upload apk'):
bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
on_fyi = 'fyi-' in os.environ.get('BUILDBOT_SCHEDULER')
if '-integration' in bot_name or on_fyi:
return
channel = bot_utils.GetChannelFromName(bot_name)
namer = bot_utils.GCSNamer(channel=channel)
gsutil = bot_utils.GSUtil()
web_link_prefix = 'https://storage.cloud.google.com/'
# Archive content shell
local = os.path.join(options.build_products_dir, CS_LOCATION)
for revision in [utils.GetArchiveVersion(), 'latest']:
# TODO(whesse): pass in arch and mode from reciepe
remote = namer.dartium_android_apk_filepath(revision,
'content_shell-android',
'arm',
'release')
content_shell_link = string.replace(remote, 'gs://', web_link_prefix)
UploadSetACL(gsutil, local, remote)
print "Uploaded content shell, available from: %s" % content_shell_link