def expand(self, max_executors):
test_stats, avg_test_time = self.get_test_stats()
group_tests = [[] for _ in range(max_executors)]
group_weights = [0 for _ in range(max_executors)]
weights = [0] * max_executors
weighted_tests = [
(self.get_test_duration(t, test_stats) or avg_test_time, t)
for t in self.data['tests']
]
for weight, test in sorted(weighted_tests, reverse=True):
low_index, _ = min(enumerate(weights), key=itemgetter(1))
weights[low_index] += 1 + weight
group_tests[low_index].append(test)
group_weights[low_index] += 1 + weight
for test_list, weight in zip(group_tests, group_weights):
future_command = FutureCommand(
script=self.data['command'].format(test_names=' '.join(test_list)),
path=self.data.get('path'),
env=self.data.get('env'),
artifacts=self.data.get('artifacts'),
)
future_jobstep = FutureJobStep(
label=self.data.get('label') or future_command.label,
commands=[future_command],
data={'weight': weight},
)
yield future_jobstep
def test_create_expanded_jobstep(self, get_vcs):
build = self.create_build(self.create_project())
job = self.create_job(build)
jobphase = self.create_jobphase(job, label='foo')
jobstep = self.create_jobstep(jobphase)
new_jobphase = self.create_jobphase(job, label='bar')
vcs = mock.Mock(spec=Vcs)
vcs.get_buildstep_clone.return_value = 'git clone https://example.com'
get_vcs.return_value = vcs
future_jobstep = FutureJobStep(
label='test',
commands=[
FutureCommand('echo 1'),
FutureCommand('echo "foo"\necho "bar"'),
],
)
buildstep = self.get_buildstep()
new_jobstep = buildstep.create_expanded_jobstep(
jobstep, new_jobphase, future_jobstep)
db.session.flush()
assert new_jobstep.data['expanded'] is True
commands = new_jobstep.commands
assert len(commands) == 4
assert commands[0].script == 'git clone https://example.com'
assert commands[0].cwd == ''
assert commands[0].type == CommandType.infra_setup
assert commands[0].order == 0
assert commands[1].script == 'echo "hello world 2"'
assert commands[1].cwd == '/usr/test/1'
assert commands[1].type == CommandType.setup
assert commands[1].order == 1
assert commands[2].label == 'echo 1'
assert commands[2].script == 'echo 1'
assert commands[2].order == 2
assert commands[2].cwd == DEFAULT_PATH
assert commands[3].label == 'echo "foo"'
assert commands[3].script == 'echo "foo"\necho "bar"'
assert commands[3].order == 3
assert commands[3].cwd == DEFAULT_PATH
def get_future_commands(self, params, commands):
"""Create future commands which are later created as comands.
See models/command.py.
"""
return map(
lambda command: FutureCommand(command['script'],
env=self.params_to_env(params)),
commands)
def expand(self, max_executors):
for cmd_data in self.data['commands']:
future_command = FutureCommand(**cmd_data)
future_jobstep = FutureJobStep(
label=cmd_data.get('label') or future_command.label,
commands=[future_command],
)
yield future_jobstep
def expand(self, max_executors, **kwargs):
for cmd_data in self.data['commands']:
# TODO: group commands with jobsteps so as to respect max_executors
future_command = FutureCommand(**cmd_data)
future_jobstep = FutureJobStep(
label=cmd_data.get('label') or future_command.label,
commands=[future_command],
)
yield future_jobstep
def test_expand_jobstep(self):
build = self.create_build(self.create_project())
job = self.create_job(build)
jobphase = self.create_jobphase(job, label='foo')
jobstep = self.create_jobstep(jobphase)
new_jobphase = self.create_jobphase(job, label='bar')
future_jobstep = FutureJobStep(
label='test',
commands=[
FutureCommand('echo 1'),
FutureCommand('echo "foo"\necho "bar"'),
],
)
buildstep = self.get_buildstep()
new_jobstep = buildstep.expand_jobstep(
jobstep, new_jobphase, future_jobstep)
db.session.flush()
assert new_jobstep.data['generated'] is True
commands = list(Command.query.filter(
Command.jobstep_id == new_jobstep.id,
).order_by(
Command.order.asc(),
))
assert len(commands) == 3
assert commands[0].script == 'echo "hello world 2"'
assert commands[0].cwd == '/usr/test/1'
assert commands[0].type == CommandType.setup
assert commands[0].order == 0
assert commands[1].label == 'echo 1'
assert commands[1].script == 'echo 1'
assert commands[1].order == 1
assert commands[1].cwd == DEFAULT_PATH
assert commands[2].label == 'echo "foo"'
assert commands[2].script == 'echo "foo"\necho "bar"'
assert commands[2].order == 2
assert commands[2].cwd == DEFAULT_PATH
def iter_all_commands(self, job):
source = job.source
repo = source.repository
vcs = repo.get_vcs()
if vcs is not None:
yield FutureCommand(
script=vcs.get_buildstep_clone(source, self.path, self.clean),
env=self.env,
type=CommandType.infra_setup,
)
if source.patch:
yield FutureCommand(
script=vcs.get_buildstep_patch(source, self.path),
env=self.env,
type=CommandType.infra_setup,
)
for command in self.commands:
yield FutureCommand(**command)
def expand(self, max_executors, test_stats_from=None):
test_stats, avg_test_time = self.get_test_stats(test_stats_from
or self.project.slug)
groups = self.shard_tests(self.data['tests'], max_executors,
test_stats, avg_test_time)
for weight, test_list in groups:
future_command = FutureCommand(
script=self.data['cmd'].format(test_names=' '.join(test_list)),
path=self.data.get('path'),
env=self.data.get('env'),
artifacts=self.data.get('artifacts'),
)
future_jobstep = FutureJobStep(
label=self.data.get('label') or future_command.label,
commands=[future_command],
data={
'weight': weight,
'tests': test_list,
'shard_count': len(groups)
},
)
yield future_jobstep
def test_create_replacement_jobstep_expanded(self, get_vcs):
build = self.create_build(self.create_project())
job = self.create_job(build)
jobphase = self.create_jobphase(job, label='foo')
jobstep = self.create_jobstep(jobphase)
new_jobphase = self.create_jobphase(job, label='bar')
vcs = mock.Mock(spec=Vcs)
vcs.get_buildstep_clone.return_value = 'git clone https://example.com'
get_vcs.return_value = vcs
future_jobstep = FutureJobStep(
label='test',
commands=[
FutureCommand('echo 1'),
FutureCommand('echo "foo"\necho "bar"'),
],
data={'weight': 1, 'forceInfraFailure': True},
)
buildstep = self.get_buildstep()
fail_jobstep = buildstep.create_expanded_jobstep(
jobstep, new_jobphase, future_jobstep)
fail_jobstep.result = Result.infra_failed
fail_jobstep.status = Status.finished
db.session.add(fail_jobstep)
db.session.commit()
new_jobstep = buildstep.create_replacement_jobstep(fail_jobstep)
# new jobstep should still be part of same job/phase
assert new_jobstep.job == job
assert new_jobstep.phase == fail_jobstep.phase
# make sure .steps actually includes the new jobstep
assert len(fail_jobstep.phase.steps) == 2
# make sure replacement id is correctly set
assert fail_jobstep.replacement_id == new_jobstep.id
# we want the replacement jobstep to have the same attributes the
# original jobstep would be expected to after expand_jobstep()
assert new_jobstep.data['expanded'] is True
assert new_jobstep.data['weight'] == 1
# make sure non-whitelisted attributes aren't copied over
assert 'forceInfraFailure' not in new_jobstep.data
commands = new_jobstep.commands
assert len(commands) == 4
assert commands[0].script == 'git clone https://example.com'
assert commands[0].cwd == ''
assert commands[0].type == CommandType.infra_setup
assert commands[0].order == 0
assert commands[1].script == 'echo "hello world 2"'
assert commands[1].cwd == '/usr/test/1'
assert commands[1].type == CommandType.setup
assert commands[1].order == 1
assert commands[2].label == 'echo 1'
assert commands[2].script == 'echo 1'
assert commands[2].order == 2
assert commands[2].cwd == DEFAULT_PATH
assert commands[3].label == 'echo "foo"'
assert commands[3].script == 'echo "foo"\necho "bar"'
assert commands[3].order == 3
assert commands[3].cwd == DEFAULT_PATH
def test_simple_expander(self, mock_get_expander,
mock_get_build_step_for_job):
project = self.create_project()
build = self.create_build(project)
job = self.create_job(build)
jobphase = self.create_jobphase(job)
jobstep = self.create_jobstep(jobphase, data={
'max_executors': 10,
})
plan = self.create_plan(project, label='test')
self.create_step(plan)
jobplan = self.create_job_plan(job, plan)
command = self.create_command(jobstep,
type=CommandType.collect_tests,
status=Status.in_progress)
def dummy_expand_jobstep(jobstep, new_jobphase, future_jobstep):
return future_jobstep.as_jobstep(new_jobphase)
dummy_expander = Mock(spec=Expander)
dummy_expander.expand.return_value = [
FutureJobStep(
label='test',
commands=[
FutureCommand(script='echo 1', ),
FutureCommand(script='echo "foo"\necho "bar"', )
],
)
]
mock_get_expander.return_value.return_value = dummy_expander
mock_buildstep = Mock(spec=BuildStep)
mock_buildstep.expand_jobstep.side_effect = dummy_expand_jobstep
mock_get_build_step_for_job.return_value = jobplan, mock_buildstep
path = '/api/0/commands/{0}/'.format(command.id.hex)
# missing output
resp = self.client.post(path, data={
'status': 'finished',
})
assert resp.status_code == 400, resp.data
mock_get_expander.reset_mock()
# valid params
resp = self.client.post(path,
data={
'status': 'finished',
'output': '{"foo": "bar"}',
})
assert resp.status_code == 200, resp.data
mock_get_expander.assert_called_once_with(command.type)
mock_get_expander.return_value.assert_called_once_with(
project=project,
data={'foo': 'bar'},
)
dummy_expander.validate.assert_called_once_with()
dummy_expander.expand.assert_called_once_with(max_executors=10)
new_jobstep = JobStep.query.filter(
JobStep.job_id == job.id,
JobStep.id != jobstep.id,
).first()
assert new_jobstep.label == 'test'
def test_create_expanded_jobstep(self, get_vcs):
build = self.create_build(self.create_project())
job = self.create_job(build)
jobphase = self.create_jobphase(job, label='foo')
jobstep = self.create_jobstep(jobphase)
new_jobphase = self.create_jobphase(job, label='bar')
vcs = mock.Mock(spec=Vcs)
vcs.get_buildstep_clone.return_value = 'git clone https://example.com'
get_vcs.return_value = vcs
future_jobstep = FutureJobStep(
label='test',
commands=[
FutureCommand('echo 1'),
FutureCommand('echo "foo"\necho "bar"', path='subdir'),
],
)
buildstep = self.get_buildstep(cluster='foo')
new_jobstep = buildstep.create_expanded_jobstep(
jobstep, new_jobphase, future_jobstep)
db.session.flush()
assert new_jobstep.data['expanded'] is True
assert new_jobstep.cluster == 'foo'
commands = new_jobstep.commands
assert len(commands) == 4
assert commands[0].script == 'git clone https://example.com'
assert commands[0].cwd == ''
assert commands[0].type == CommandType.infra_setup
assert commands[0].artifacts == []
assert commands[0].env == DEFAULT_ENV
assert commands[0].order == 0
assert commands[1].script == 'echo "hello world 2"'
assert commands[1].cwd == '/usr/test/1'
assert commands[1].type == CommandType.setup
assert tuple(commands[1].artifacts) == ('artifact1.txt',
'artifact2.txt')
assert commands[1].env['PATH'] == '/usr/test/1'
for k, v in DEFAULT_ENV.items():
if k != 'PATH':
assert commands[1].env[k] == v
assert commands[1].order == 1
assert commands[2].label == 'echo 1'
assert commands[2].script == 'echo 1'
assert commands[2].order == 2
assert commands[2].cwd == DEFAULT_PATH
assert commands[2].type == CommandType.default
assert tuple(commands[2].artifacts) == tuple(DEFAULT_ARTIFACTS)
assert commands[2].env == DEFAULT_ENV
assert commands[3].label == 'echo "foo"'
assert commands[3].script == 'echo "foo"\necho "bar"'
assert commands[3].order == 3
assert commands[3].cwd == './source/subdir'
assert commands[3].type == CommandType.default
assert tuple(commands[3].artifacts) == tuple(DEFAULT_ARTIFACTS)
assert commands[3].env == DEFAULT_ENV
def __init__(self,
commands=None,
path=DEFAULT_PATH,
env=None,
artifacts=DEFAULT_ARTIFACTS,
release=DEFAULT_RELEASE,
max_executors=10,
cpus=4,
memory=8 * 1024,
clean=True,
debug_config=None,
test_stats_from=None,
cluster=None,
**kwargs):
"""
Constructor for DefaultBuildStep.
Args:
cpus: How many cpus to limit the container to (not applicable for basic)
memory: How much memory to limit the container to (not applicable for basic)
clean: controls if the repository should be cleaned before
tests are run.
Defaults to true, because False may be unsafe; it may be
useful to set to False if snapshots are in use and they
intentionally leave useful incremental build products in the
repository.
debug_config: A dictionary of debug config options. These are passed through
to changes-client. There is also an infra_failures option, which takes a
dictionary used to force infrastructure failures in builds. The keys of
this dictionary refer to the phase (for DefaultBuildSteps, only possible
value is 'primary'), and the values are the probabilities with which
a JobStep in that phase will fail.
An example: "debug_config": {"infra_failures": {"primary": 0.5}}
This will then cause an infra failure in the primary JobStep with
probability 0.5.
test_stats_from: project to get test statistics from, or
None (the default) to use this project. Useful if the
project runs a different subset of tests each time, so
test timing stats from the parent are not reliable.
cluster: a cluster to associate jobs of this BuildStep with.
Jobsteps will then only be run on slaves of the given cluster.
"""
if commands is None:
raise ValueError("Missing required config: need commands")
if env is None:
env = DEFAULT_ENV.copy()
self.artifacts = artifacts
self.env = env
self.path = path
self.release = release
self.max_executors = max_executors
self.resources = {
'cpus': cpus,
'mem': memory,
}
self.clean = clean
self.debug_config = debug_config or {}
self.test_stats_from = test_stats_from
self.cluster = cluster
future_commands = []
for command in commands:
command_copy = command.copy()
if 'type' in command_copy:
command_copy['type'] = CommandType[command_copy['type']]
future_command = FutureCommand(**command_copy)
self._set_command_defaults(future_command)
future_commands.append(future_command)
self.commands = future_commands
super(DefaultBuildStep, self).__init__(**kwargs)