def Update(config, active_master, c):
factory = BuildFactory()
factory_commands = commands.FactoryCommands(factory)
factory_commands.AddUpdateScriptStep()
# pylint: disable=W0212
gitzip_exe = os.path.join(factory_commands._script_dir, 'gitzip.py')
cmd = [
'python', gitzip_exe, '--workdir', '.', '--url',
'%schromium/src.git' % config.Master.git_server_url, '--gs_bucket',
'gs://chromium-git-bundles', '--gs_acl', 'public-read', '--timeout',
'%d' % (60 * 60), '--stayalive', '200', '--verbose'
]
factory.addStep(shell.ShellCommand,
name='gitzip',
description='gitzip',
timeout=7200,
workdir='',
command=cmd)
builders = c.setdefault('builders', [])
builders.append({
'name': 'Chromium Git Packager',
'builddir': 'chromium-git-packager',
'factory': factory,
'auto_reboot': False
})
schedulers = c.setdefault('schedulers', [])
schedulers.append(
Nightly(name='gitzip_nightly',
branch=None,
builderNames=['Chromium Git Packager'],
hour=2,
minute=40))
def Update(config, active_master, c):
factory = BuildFactory()
factory_commands = commands.FactoryCommands(factory)
factory_commands.AddUpdateScriptStep()
# pylint: disable=W0212
gitzip_exe = os.path.join(factory_commands._script_dir, 'gitzip.py')
cmd = ['python', gitzip_exe,
'--workdir', '.',
'--url', '%schromium/src.git' % config.Master.git_server_url,
'--gs_bucket', 'gs://chromium-git-bundles',
'--gs_acl', 'public-read',
'--timeout', '%d' % (60*60),
'--stayalive', '200',
'--verbose']
factory.addStep(shell.ShellCommand, name='gitzip', description='gitzip',
timeout=7200, workdir='', command=cmd)
builders = c.setdefault('builders', [])
builders.append({
'name': 'Chromium Git Packager',
'builddir': 'chromium-git-packager',
'factory': factory,
'auto_reboot': False})
schedulers = c.setdefault('schedulers', [])
schedulers.append(Nightly(name='gitzip_nightly',
branch=None,
builderNames=['Chromium Git Packager'],
hour=2,
minute=40))
def BaseFactory(self, recipe, factory_properties=None, triggers=None, timeout=1200, max_time=None):
"""The primary input for the factory is the |recipe|, which specifies the
name of a recipe file to search for. The recipe file will fill in the rest
of the |factory_properties|. This setup allows for major changes to factory
properties to occur on slave-side without master restarts.
NOTE: Please be very discerning with what |factory_properties| you pass to
this method. Ideally, you will pass none, and that will be sufficient in the
vast majority of cases. Think very carefully before adding any
|factory_properties| here, as changing them will require a master restart.
|timeout| refers to the maximum number of seconds a step should be allowed
to run without output. After no output for |timeout| seconds, the step is
forcibly killed.
|max_time| refers to the maximum number of seconds a step should be allowed
to run, regardless of output. After |max_time| seconds, the step is forcibly
killed.
"""
factory_properties = factory_properties or {}
factory_properties.update({"recipe": recipe})
self._factory_properties = factory_properties
factory = BuildFactory()
cmd_obj = annotator_commands.AnnotatorCommands(factory)
cmd_obj.AddAnnotatedScript(factory_properties, timeout=timeout, max_time=max_time)
for t in triggers or []:
factory.addStep(commands.CreateTriggerStep(t, trigger_copy_properties=["swarm_hashes"]))
return factory
def BaseFactory(self, recipe, factory_properties=None, triggers=None,
timeout=1200, max_time=None):
"""The primary input for the factory is the |recipe|, which specifies the
name of a recipe file to search for. The recipe file will fill in the rest
of the |factory_properties|. This setup allows for major changes to factory
properties to occur on slave-side without master restarts.
NOTE: Please be very discerning with what |factory_properties| you pass to
this method. Ideally, you will pass none, and that will be sufficient in the
vast majority of cases. Think very carefully before adding any
|factory_properties| here, as changing them will require a master restart.
|timeout| refers to the maximum number of seconds a step should be allowed
to run without output. After no output for |timeout| seconds, the step is
forcibly killed.
|max_time| refers to the maximum number of seconds a step should be allowed
to run, regardless of output. After |max_time| seconds, the step is forcibly
killed.
"""
factory_properties = factory_properties or {}
factory_properties.update({'recipe': recipe})
self._factory_properties = factory_properties
factory = BuildFactory()
cmd_obj = annotator_commands.AnnotatorCommands(factory)
cmd_obj.AddAnnotatedScript(
factory_properties, timeout=timeout, max_time=max_time)
for t in (triggers or []):
factory.addStep(commands.CreateTriggerStep(
t, trigger_copy_properties=['swarm_hashes']))
return factory
def WebMFactory(self,
target='Release',
clobber=False,
tests=None,
mode=None,
slave_type='BuilderTester',
options=None,
compile_timeout=1200,
build_url=None,
project=None,
factory_properties=None):
factory_properties = factory_properties or {}
tests = tests or []
factory = BuildFactory()
# Get the factory command object to create new steps to the factory.
webm_cmd_obj = webm_commands.WebMCommands(factory, target,
self._build_dir,
self._target_platform)
webm_cmd_obj.AddUpdateScriptStep()
# ADD UPDATE AND COMPILE STEP HERE.
url = 'git://review.webmproject.org/libvpx.git'
webm_cmd_obj.AddCloneOrFetchRepositoryStep(url, checkout_dir='libvpx')
webm_cmd_obj.AddCheckoutRevisionStep(url, checkout_dir='libvpx')
webm_cmd_obj.AddConfigureStep(checkout_dir='libvpx')
webm_cmd_obj.AddMakeStep(checkout_dir='libvpx')
webm_cmd_obj.AddInstallStep(checkout_dir='libvpx')
# Add all the tests.
self._AddTests(webm_cmd_obj, tests, mode, factory_properties)
return factory
def BaseFactory(self, gclient_spec=None, official_release=False,
factory_properties=None, build_properties=None,
delay_compile_step=False, sudo_for_remove=False,
gclient_deps=None, slave_type=None):
if gclient_spec is None:
gclient_spec = self.BuildGClientSpec()
factory_properties = factory_properties or {}
factory = BuildFactory(build_properties)
factory_cmd_obj = commands.FactoryCommands(factory,
target_platform=self._target_platform)
# First kill any svn.exe tasks so we can update in peace, and
# afterwards use the checked-out script to kill everything else.
if (self._target_platform == 'win32' and
not factory_properties.get('no_kill')):
factory_cmd_obj.AddSvnKillStep()
script_solutions = None
if (factory_properties.get('goma_canary') and
factory_properties.get('slave_internal_url')):
script_solutions = [GClientSolution(
factory_properties.get('slave_internal_url'),
name="slave.DEPS",
custom_vars_list=[("goma_linux_revision", "HEAD"),
("goma_mac_revision", "HEAD"),
("goma_win_revision", "HEAD")])]
factory_cmd_obj.AddUpdateScriptStep(
gclient_jobs=factory_properties.get('update_scripts_gclient_jobs'),
solutions=script_solutions)
# Once the script is updated, the zombie processes left by the previous
# run can be killed.
if (self._target_platform == 'win32' and
not factory_properties.get('no_kill')):
factory_cmd_obj.AddTaskkillStep()
# Revert the tree to a clean (unmodified) state.
factory_cmd_obj.AddGClientRevertStep()
env = factory_properties.get('gclient_env', {})
# Allow gclient_deps to also come from the factory_properties.
if gclient_deps == None:
gclient_deps = factory_properties.get('gclient_deps', None)
if gclient_deps == 'ios':
gclient_spec += ';target_os = [\'ios\'];target_os_only = True'
# svn timeout is 2 min; we allow 5
timeout = factory_properties.get('gclient_timeout')
if official_release or factory_properties.get('nuke_and_pave'):
no_gclient_branch = factory_properties.get('no_gclient_branch', False)
factory_cmd_obj.AddClobberTreeStep(gclient_spec, env, timeout,
gclient_deps=gclient_deps, gclient_nohooks=self._nohooks_on_update,
no_gclient_branch=no_gclient_branch)
elif not delay_compile_step:
self.AddUpdateStep(gclient_spec, factory_properties, factory,
slave_type, sudo_for_remove, gclient_deps=gclient_deps)
return factory
def BaseFactory(self, recipe=None, factory_properties=None, triggers=None,
timeout=2400, max_time=None):
"""The primary input for the factory is the |recipe|, which specifies the
name of a recipe file to search for. The recipe file will fill in the rest
of the |factory_properties|. This setup allows for major changes to factory
properties to occur on slave-side without master restarts.
NOTE: Please be very discerning with what |factory_properties| you pass to
this method. Ideally, you will pass none, and that will be sufficient in the
vast majority of cases. Think very carefully before adding any
|factory_properties| here, as changing them will require a master restart.
|recipe| is the name of the recipe to pass to annotated_run. If omitted,
annotated_run will attempt to look up the recipe from builders.pyl in the
master.
|timeout| refers to the maximum number of seconds a build should be allowed
to run without output. After no output for |timeout| seconds, the build is
forcibly killed.
|max_time| refers to the maximum number of seconds a build should be allowed
to run, regardless of output. After |max_time| seconds, the build is
forcibly killed.
"""
factory_properties = factory_properties or {}
if recipe:
factory_properties.update({'recipe': recipe})
self._factory_properties = factory_properties
factory = BuildFactory(build_inherit_factory_properties=False)
factory.properties.update(self._factory_properties, 'AnnotatorFactory')
cmd_obj = annotator_commands.AnnotatorCommands(
factory, active_master=self.active_master)
cmd_obj.AddAnnotatedScript(timeout=timeout, max_time=max_time)
for t in triggers or []:
factory.addStep(commands.CreateTriggerStep(
t, trigger_copy_properties=['swarm_hashes']))
return factory
def BaseFactory(self, recipe, factory_properties=None):
"""The primary input for the factory is the |recipe|, which specifies the
name of a recipe file to search for. The recipe file will fill in the rest
of the |factory_properties|. This setup allows for major changes to factory
properties to occur on slave-side without master restarts.
NOTE: Please be very discerning with what |factory_properties| you pass to
this method. Ideally, you will pass none, and that will be sufficient in the
vast majority of cases. Think very carefully before adding any
|factory_properties| here, as changing them will require a master restart.
"""
factory_properties = factory_properties or {}
factory_properties.update({'recipe': recipe})
self._factory_properties = factory_properties
factory = BuildFactory()
cmd_obj = annotator_commands.AnnotatorCommands(factory)
cmd_obj.AddAnnotatedScript(factory_properties)
return factory
def BaseFactory(self, gclient_spec=None, official_release=False,
factory_properties=None, build_properties=None,
delay_compile_step=False, sudo_for_remove=False,
gclient_deps=None, slave_type=None):
if gclient_spec is None:
gclient_spec = self.BuildGClientSpec()
factory_properties = factory_properties or {}
factory = BuildFactory(build_properties)
factory_cmd_obj = commands.FactoryCommands(factory,
target_platform=self._target_platform)
# First kill any svn.exe tasks so we can update in peace, and
# afterwards use the checked-out script to kill everything else.
if (self._target_platform == 'win32' and
not factory_properties.get('no_kill')):
factory_cmd_obj.AddSvnKillStep()
factory_cmd_obj.AddUpdateScriptStep(
gclient_jobs=factory_properties.get('update_scripts_gclient_jobs'))
# Once the script is updated, the zombie processes left by the previous
# run can be killed.
if (self._target_platform == 'win32' and
not factory_properties.get('no_kill')):
factory_cmd_obj.AddTaskkillStep()
env = factory_properties.get('gclient_env', {})
# Allow gclient_deps to also come from the factory_properties.
if gclient_deps == None:
gclient_deps = factory_properties.get('gclient_deps', None)
# svn timeout is 2 min; we allow 5
timeout = factory_properties.get('gclient_timeout')
if official_release or factory_properties.get('nuke_and_pave'):
no_gclient_branch = factory_properties.get('no_gclient_branch', False)
factory_cmd_obj.AddClobberTreeStep(gclient_spec, env, timeout,
gclient_deps=gclient_deps, gclient_nohooks=self._nohooks_on_update,
no_gclient_branch=no_gclient_branch)
elif not delay_compile_step:
self.AddUpdateStep(gclient_spec, factory_properties, factory,
slave_type, sudo_for_remove, gclient_deps=gclient_deps)
return factory
def __init__(self, other_repos=None, do_upload_render_results=False,
do_upload_bench_results=False, do_patch_step=False,
build_subdir='skia', target_platform=None,
configuration=CONFIG_DEBUG, default_timeout=8*60*60,
deps_target_os=None, environment_variables=None,
perf_output_basedir=None, builder_name=None, flavor=None,
make_flags=None, test_args=None, gm_args=None, bench_args=None,
bench_pictures_cfg='default', compile_warnings_as_errors=False,
gyp_defines=None, build_targets=None):
"""Instantiates a SkiaFactory as appropriate for this target_platform.
do_upload_render_results: whether we should upload render results
do_upload_bench_results: whether we should upload bench results
do_patch_step: whether the build should include a step which applies a
patch. This is only applicable for trybots.
build_subdir: subdirectory to check out and then build within
other_repos: list of other repositories to also check out (or None). Each
repo is specified as a tuple: (name, url), where "name" is the target
directory and "url" is the source code url.
target_platform: a string such as TARGET_PLATFORM_LINUX
configuration: 'Debug' or 'Release'
default_timeout: default timeout for each command, in seconds
deps_target_os: string; the target_os to be specified in the gclient config.
environment_variables: dictionary of environment variables that should
be passed to all commands
perf_output_basedir: path to directory under which to store performance
data, or None if we don't want to store performance data
builder_name: name of the builder associated with this factory
flavor: which "flavor" of slave-side scripts this factory should use
make_flags: list of extra flags to pass to the compile step
test_args: list of extra flags to pass to the 'tests' executable
gm_args: list of extra flags to pass to the 'gm' executable
bench_args: list of extra flags to pass to the 'bench' executable
bench_pictures_cfg: config name to use for bench_pictures
compile_warnings_as_errors: boolean; whether to build with "-Werror" or
some equivalent.
gyp_defines: optional dict; GYP_DEFINES to be used in the build.
build_targets: optional list; the targets to build. Default is set depending
on which Build() function is called.
"""
properties = {}
self.skipsteps = utils.GetListFromEnvVar(
config_private.SKIPSTEPS_ENVIRONMENT_VARIABLE)
self.dontskipsteps = utils.GetListFromEnvVar(
config_private.DONTSKIPSTEPS_ENVIRONMENT_VARIABLE)
if not make_flags:
make_flags = []
self._make_flags = make_flags
# Platform-specific stuff.
if target_platform == TARGET_PLATFORM_WIN32:
self.TargetPath = ntpath
else:
self.TargetPath = posixpath
# Create gclient solutions corresponding to the main build_subdir
# and other directories we also wish to check out.
self._gclient_solutions = [gclient_factory.GClientSolution(
svn_url=SKIA_GIT_URL, name=build_subdir
).GetSpec()]
if not other_repos:
other_repos = []
repos_to_checkout = set(other_repos)
for other_repo in repos_to_checkout:
self._gclient_solutions.append(gclient_factory.GClientSolution(
svn_url=other_repo[1], name=other_repo[0]).GetSpec())
self._deps_target_os = deps_target_os
# Set _default_clobber based on config.Master
self._default_clobber = getattr(config.Master, 'default_clobber', False)
self._do_upload_render_results = do_upload_render_results
self._do_upload_bench_results = (do_upload_bench_results and
perf_output_basedir != None)
self._do_patch_step = do_patch_step
if not environment_variables:
self._env_vars = {}
else:
self._env_vars = dict(environment_variables)
self._gyp_defines = dict(gyp_defines or {})
self._gyp_defines['skia_warnings_as_errors'] = \
'%d' % int(compile_warnings_as_errors)
self._build_targets = list(build_targets or [])
# Get an implementation of SkiaCommands as appropriate for
# this target_platform.
self._workdir = self.TargetPath.join('build', build_subdir)
self._skia_cmd_obj = skia_commands.SkiaCommands(
target_platform=target_platform, factory=self,
configuration=configuration, workdir=self._workdir,
target_arch=None, default_timeout=default_timeout,
environment_variables=self._env_vars)
self._perf_output_basedir = perf_output_basedir
self._configuration = configuration
if self._configuration not in CONFIGURATIONS:
raise ValueError('Invalid configuration %s. Must be one of: %s' % (
self._configuration, CONFIGURATIONS))
self._skia_svn_username_file = '.skia_svn_username'
self._skia_svn_password_file = '.skia_svn_password'
self._builder_name = builder_name
self._flavor = flavor
def _DetermineRevision(build):
""" Get the 'revision' property at build time. WithProperties returns the
empty string if 'revision' is not defined, which causes failures when we
try to pass the revision over a command line, so we use the string "None"
to indicate that the revision is not defined.
build: instance of Build for the current build.
"""
props = build.getProperties().asDict()
if props.has_key('revision'):
if props['revision'][0]:
return props['revision'][0]
return 'None'
if not test_args:
test_args = []
if not gm_args:
gm_args = []
if not bench_args:
bench_args = []
self._common_args = [
# See http://skbug.com/2663 before deleting the next line.
'--autogen_svn_baseurl', 'bogusValueToFixSkBug2663',
'--configuration', configuration,
'--deps_target_os', self._deps_target_os or 'None',
'--builder_name', builder_name,
'--build_number', WithProperties('%(buildnumber)s'),
'--target_platform', target_platform,
'--revision', WithProperties('%(rev)s', rev=_DetermineRevision),
'--got_revision', WithProperties('%(got_revision:-None)s'),
'--perf_output_basedir', perf_output_basedir or 'None',
'--make_flags', '"%s"' % ' '.join(self._make_flags),
'--test_args', '"%s' % ' '.join(test_args),
'--gm_args', '"%s"' % ' '.join(gm_args),
'--bench_args', '"%s"' % ' '.join(bench_args),
'--is_try', str(self._do_patch_step),
'--bench_pictures_cfg', bench_pictures_cfg,
'--issue_number', WithProperties('%(issue:-None)s'),
]
BuildFactory.__init__(self, build_factory_properties=properties)
def __init__(self,
other_repos=None,
do_upload_render_results=False,
do_upload_bench_results=False,
do_patch_step=False,
build_subdir='skia',
target_platform=None,
configuration=CONFIG_DEBUG,
default_timeout=8 * 60 * 60,
deps_target_os=None,
environment_variables=None,
perf_output_basedir=None,
builder_name=None,
flavor=None,
make_flags=None,
test_args=None,
gm_args=None,
bench_args=None,
bench_pictures_cfg='default',
compile_warnings_as_errors=False,
gyp_defines=None,
build_targets=None):
"""Instantiates a SkiaFactory as appropriate for this target_platform.
do_upload_render_results: whether we should upload render results
do_upload_bench_results: whether we should upload bench results
do_patch_step: whether the build should include a step which applies a
patch. This is only applicable for trybots.
build_subdir: subdirectory to check out and then build within
other_repos: list of other repositories to also check out (or None). Each
repo is specified as a tuple: (name, url), where "name" is the target
directory and "url" is the source code url.
target_platform: a string such as TARGET_PLATFORM_LINUX
configuration: 'Debug' or 'Release'
default_timeout: default timeout for each command, in seconds
deps_target_os: string; the target_os to be specified in the gclient config.
environment_variables: dictionary of environment variables that should
be passed to all commands
perf_output_basedir: path to directory under which to store performance
data, or None if we don't want to store performance data
builder_name: name of the builder associated with this factory
flavor: which "flavor" of slave-side scripts this factory should use
make_flags: list of extra flags to pass to the compile step
test_args: list of extra flags to pass to the 'tests' executable
gm_args: list of extra flags to pass to the 'gm' executable
bench_args: list of extra flags to pass to the 'bench' executable
bench_pictures_cfg: config name to use for bench_pictures
compile_warnings_as_errors: boolean; whether to build with "-Werror" or
some equivalent.
gyp_defines: optional dict; GYP_DEFINES to be used in the build.
build_targets: optional list; the targets to build. Default is set depending
on which Build() function is called.
"""
properties = {}
self.skipsteps = utils.GetListFromEnvVar(
config_private.SKIPSTEPS_ENVIRONMENT_VARIABLE)
self.dontskipsteps = utils.GetListFromEnvVar(
config_private.DONTSKIPSTEPS_ENVIRONMENT_VARIABLE)
if not make_flags:
make_flags = []
self._make_flags = make_flags
# Platform-specific stuff.
if target_platform == TARGET_PLATFORM_WIN32:
self.TargetPath = ntpath
else:
self.TargetPath = posixpath
# Create gclient solutions corresponding to the main build_subdir
# and other directories we also wish to check out.
self._gclient_solutions = [
gclient_factory.GClientSolution(svn_url=SKIA_GIT_URL,
name=build_subdir).GetSpec()
]
if not other_repos:
other_repos = []
repos_to_checkout = set(other_repos)
for other_repo in repos_to_checkout:
self._gclient_solutions.append(
gclient_factory.GClientSolution(svn_url=other_repo[1],
name=other_repo[0]).GetSpec())
self._deps_target_os = deps_target_os
# Set _default_clobber based on config.Master
self._default_clobber = getattr(config.Master, 'default_clobber',
False)
self._do_upload_render_results = do_upload_render_results
self._do_upload_bench_results = (do_upload_bench_results
and perf_output_basedir != None)
self._do_patch_step = do_patch_step
if not environment_variables:
self._env_vars = {}
else:
self._env_vars = dict(environment_variables)
self._gyp_defines = dict(gyp_defines or {})
self._gyp_defines['skia_warnings_as_errors'] = \
'%d' % int(compile_warnings_as_errors)
self._build_targets = list(build_targets or [])
# Get an implementation of SkiaCommands as appropriate for
# this target_platform.
self._workdir = self.TargetPath.join('build', build_subdir)
self._skia_cmd_obj = skia_commands.SkiaCommands(
target_platform=target_platform,
factory=self,
configuration=configuration,
workdir=self._workdir,
target_arch=None,
default_timeout=default_timeout,
environment_variables=self._env_vars)
self._perf_output_basedir = perf_output_basedir
self._configuration = configuration
if self._configuration not in CONFIGURATIONS:
raise ValueError('Invalid configuration %s. Must be one of: %s' %
(self._configuration, CONFIGURATIONS))
self._skia_svn_username_file = '.skia_svn_username'
self._skia_svn_password_file = '.skia_svn_password'
self._builder_name = builder_name
self._flavor = flavor
def _DetermineRevision(build):
""" Get the 'revision' property at build time. WithProperties returns the
empty string if 'revision' is not defined, which causes failures when we
try to pass the revision over a command line, so we use the string "None"
to indicate that the revision is not defined.
build: instance of Build for the current build.
"""
props = build.getProperties().asDict()
if props.has_key('revision'):
if props['revision'][0]:
return props['revision'][0]
return 'None'
if not test_args:
test_args = []
if not gm_args:
gm_args = []
if not bench_args:
bench_args = []
self._common_args = [
# See http://skbug.com/2663 before deleting the next line.
'--autogen_svn_baseurl',
'bogusValueToFixSkBug2663',
'--configuration',
configuration,
'--deps_target_os',
self._deps_target_os or 'None',
'--builder_name',
builder_name,
'--build_number',
WithProperties('%(buildnumber)s'),
'--target_platform',
target_platform,
'--revision',
WithProperties('%(rev)s', rev=_DetermineRevision),
'--got_revision',
WithProperties('%(got_revision:-None)s'),
'--perf_output_basedir',
perf_output_basedir or 'None',
'--make_flags',
'"%s"' % ' '.join(self._make_flags),
'--test_args',
'"%s' % ' '.join(test_args),
'--gm_args',
'"%s"' % ' '.join(gm_args),
'--bench_args',
'"%s"' % ' '.join(bench_args),
'--is_try',
str(self._do_patch_step),
'--bench_pictures_cfg',
bench_pictures_cfg,
'--issue_number',
WithProperties('%(issue:-None)s'),
]
BuildFactory.__init__(self, build_factory_properties=properties)