def test_build_configuration_pinning(storage):
config = dedent("""\
jobs:
- id: job-1
function: jobcontrol.utils.testing:testing_job
kwargs:
retval: "original-retval"
""")
config = JobControlConfig.from_string(config)
jc = JobControl(storage=storage, config=config)
# ------------------------------------------------------------
# Create a build with old configuration
# ------------------------------------------------------------
job = jc.get_job('job-1')
build = job.create_build()
build.run()
build.refresh()
assert build['finished'] and build['success']
assert build['retval'] == 'original-retval'
build = job.create_build()
build_id = build.id # Then stop using this object
# ------------------------------------------------------------
# Update the configuration
# ------------------------------------------------------------
config = dedent("""\
jobs:
- id: job-1
function: jobcontrol.utils.testing:testing_job
kwargs:
retval: "new-retval"
""")
config = JobControlConfig.from_string(config)
jc = JobControl(storage=storage, config=config)
# ------------------------------------------------------------
# Running that build will return the original return value
build = jc.get_build(build_id)
build.run()
build.refresh()
assert build['finished'] and build['success']
assert build['retval'] == 'original-retval'
# ------------------------------------------------------------
# A freshly created build will return the new return value
job = jc.get_job('job-1')
build = job.create_build()
build.run()
build.refresh()
assert build['finished'] and build['success']
assert build['retval'] == 'new-retval'
build = job.create_build()
build_id = build.id # Then stop using this object
def test_build_failure_nonserializable_exception(storage):
"""
It only gets worse when we cannot even serialize the exception..
But still, we can wrap it in a serialization error exception
and be fine with it. Hopefully, we can keep the original traceback..
"""
config = JobControlConfig.from_string("""
jobs:
- id: job-nse
function: jobcontrol.utils.testing:job_raising_nonserializable
""")
jc = JobControl(storage=storage, config=config)
# Run build for RAISE nonserializable
# It should just fail with an exception in the post-run serialization
# todo: We might even check the traceback for that..
job = jc.get_job('job-nse')
build = job.create_build()
build.run()
assert build['started']
assert build['finished']
assert not build['success']
# WARNING! How to tell whether this job failed due to
# the raised exception being serialized properly, or due
# to the exception serialization failed?
assert not isinstance(build['exception'], NonSerializableException)
assert isinstance(build['exception'], ExceptionPlaceholder)
def test_config_all_keys():
config = JobControlConfig.from_string("""
storage: postgresql://localhost/database
webapp:
PORT: 5000
HOST: 0.0.0.0
SECRET_KEY: "super secret key"
celery:
BROKER_URL: redis://
secret:
MY_PASSWORD: '******'
jobs:
- id: foo
- id: bar
""")
assert config.storage == 'postgresql://localhost/database'
assert config.webapp == {
'PORT': 5000,
'HOST': '0.0.0.0',
'SECRET_KEY': 'super secret key',
}
assert config.celery == {'BROKER_URL': 'redis://'}
assert config.secret == {'MY_PASSWORD': '******'}
assert len(config.jobs) == 2
assert isinstance(config.jobs[0], BuildConfig)
assert config.jobs[0]['id'] == 'foo'
assert isinstance(config.jobs[1], BuildConfig)
assert config.jobs[1]['id'] == 'bar'
def test_build_with_skip(storage):
config = JobControlConfig.from_string("""
jobs:
- id: job-to-skip
function: jobcontrol.utils.testing:testing_job
kwargs:
retval: "Foo Retval"
skip: True
""")
jc = JobControl(storage=storage, config=config)
job = jc.get_job('job-to-skip')
build = job.create_build()
build.run()
assert build['started']
assert build['finished']
assert build['skipped']
assert job.has_builds()
assert not job.has_successful_builds() # Skipped builds are ignored
assert not job.has_running_builds()
assert list(job.iter_builds()) == [build]
assert list(job.iter_builds(skipped=False)) == []
assert build['exception'] is None
assert build['exception_tb'] is None
def test_build_with_failure(storage):
config = JobControlConfig.from_string("""
jobs:
- id: foo
function: jobcontrol.utils.testing:testing_job
kwargs:
retval: "Foo Retval"
fail: True
""")
jc = JobControl(storage=storage, config=config)
job = jc.get_job('foo')
build = job.create_build()
build.run()
assert build['started']
assert build['finished']
assert not build['success']
assert not build['skipped']
assert job.has_builds()
assert not job.has_successful_builds()
assert not job.has_running_builds()
assert list(job.iter_builds()) == [build]
assert list(job.iter_builds(success=True)) == []
assert list(job.iter_builds(success=False)) == [build]
assert isinstance(build['exception'], RuntimeError)
assert isinstance(build['exception_tb'], TracebackInfo)
def from_config(cls, config):
"""
Initialize JobControl from some configuration.
:param config:
Either a :py:class:`jobcontrol.config.JobControlConfig`
instance, or a dict to be passed as argument to that
class constructor.
:return:
a :py:class:`JobControl` instance
"""
if not isinstance(config, JobControlConfig):
config = JobControlConfig(config)
obj = cls(storage=config.get_storage(), config=config)
return obj
def test_config_default_values():
config = JobControlConfig.from_string('')
assert config.storage is None
assert config.jobs == []
assert config.webapp == {}
assert config.celery == {}
assert config.secret == {}
assert config.get_storage() is None
def from_config_file(cls, config_file):
"""
Initialize JobControl by loading configuration from a file.
Will also initialize storage taking values from the configuration.
:param config_file:
Path to configuration file, or an open file descriptor
(or file-like object).
:return:
a :py:class:`JobControl` instance
"""
config = JobControlConfig.from_file(config_file)
obj = cls(storage=config.get_storage(), config=config)
return obj
def test_simple_build_run(storage):
config = JobControlConfig.from_string("""
jobs:
- id: foo
function: jobcontrol.utils.testing:testing_job
kwargs:
retval: "Foo Retval"
""")
jc = JobControl(storage=storage, config=config)
job = jc.get_job('foo')
assert job.has_builds() is False
assert job.has_successful_builds() is False
assert job.has_running_builds() is False
assert job.is_outdated() is None
assert job.can_be_built() is True
# Create and run a build
# ------------------------------------------------------------
build = job.create_build()
assert job.has_builds() is False # "finished" builds only
assert job.has_successful_builds() is False
assert job.has_running_builds() is False
assert job.is_outdated() is None
assert job.can_be_built() is True
assert list(job.iter_builds()) == [build]
build.run()
assert build['started'] is True
assert build['finished'] is True
assert build['success'] is True
assert build['skipped'] is False
assert build['retval'] == 'Foo Retval'
assert job.has_builds() is True
assert job.has_successful_builds() is True
assert job.has_running_builds() is False
assert job.is_outdated() is False
assert job.can_be_built() is True
assert list(job.iter_builds()) == [build]
def test_build_failure_due_to_nonserializable_object(storage):
config = JobControlConfig.from_string("""
jobs:
- id: job-nso
function: jobcontrol.utils.testing:job_returning_nonserializable
""")
jc = JobControl(storage=storage, config=config)
job = jc.get_job('job-nso')
build = job.create_build()
build.run()
assert build['started']
assert build['finished']
assert not build['success']
assert isinstance(build['exception'], SerializationError)
assert ( # The original exception message is kept..
"TypeError('a class that defines __slots__ without defining "
"__getstate__ cannot be pickled',)") in build['exception'].message
def test_jobcontrol_config_exceptions():
with pytest.raises(TypeError):
JobControlConfig('This is not a dict')
with pytest.raises(TypeError) as excinfo:
JobControlConfig({'storage': ['not', 'a', 'string']})
assert excinfo.value.message == 'storage must be a string'
with pytest.raises(TypeError) as excinfo:
JobControlConfig.from_string("""
jobs:
- missing: id
""")
assert excinfo.value.message == 'Job id cannot be None'
with pytest.raises(ValueError) as excinfo:
JobControlConfig.from_string("""
jobs:
- id: foo
- id: bar
- id: foo
""")
assert excinfo.value.message == 'Duplicate job id: foo'
with pytest.raises(TypeError) as excinfo:
JobControlConfig.from_string("""
jobs:
- id: something
title: ['not', 'a', 'string']
""")
assert excinfo.value.message == 'title must be a string, got list instead'
with pytest.raises(TypeError) as excinfo:
JobControlConfig.from_string("""
jobs:
- id: something
notes: ['not', 'a', 'string']
""")
assert excinfo.value.message == 'notes must be a string, got list instead'
def test_build_config_pickle_unpickle():
import pickle
config = JobControlConfig.from_string("""
webapp:
PORT: 5000
jobs:
- id: foo
- id: bar
dependencies: ['foo']
args:
- !retval 'foo'
""")
pickled_config = pickle.dumps(config)
unpickled_config = pickle.loads(pickled_config)
assert isinstance(unpickled_config, JobControlConfig)
assert unpickled_config == config
assert not (unpickled_config != config) # test __ne__
assert len(unpickled_config.jobs) == 2
for job in unpickled_config.jobs:
assert isinstance(job, BuildConfig)
def test_simple_build_deletion(storage):
config = JobControlConfig.from_string("""
jobs:
- id: job-to-delete
function: jobcontrol.utils.testing:testing_job
""")
jc = JobControl(storage=storage, config=config)
job = jc.get_job('job-to-delete')
build_1 = job.create_build()
build_1.run()
assert len(list(job.iter_builds())) == 1
build_2 = job.create_build()
build_2.run()
assert len(list(job.iter_builds())) == 2
build_1.delete()
assert len(list(job.iter_builds())) == 1
def test_build_deletion_with_cleanup(storage):
config = JobControlConfig.from_string("""
jobs:
- id: job-to-delete
function: jobcontrol.utils.testing:job_creating_temp_file
cleanup_function: jobcontrol.utils.testing:cleanup_temp_file
""")
jc = JobControl(storage=storage, config=config)
job = jc.get_job('job-to-delete')
build = job.create_build()
build.run()
assert build['finished'] and build['success']
assert isinstance(build.retval, str)
assert os.path.isfile(build.retval)
assert len(list(job.iter_builds())) == 1
build.delete()
assert len(list(job.iter_builds())) == 0
assert not os.path.isfile(build.retval)
def test_job_config_with_dependencies():
config = JobControlConfig.from_string("""
jobs:
- id: foo
function: mymodule.foo
dependencies: []
- id: bar
function: mymodule.bar
dependencies: ['foo']
- id: baz
function: mymodule.baz
dependencies: ['foo', 'bar']
""")
# Make sure they are still the same method.
# Note that comparing with ``is`` will fail as they are *not*
# actually the same method anymore..
assert config.get_job == config.get_job_config
assert JobControlConfig.get_job == JobControlConfig.get_job_config
assert config.get_job('foo')['dependencies'] == []
assert config.get_job('bar')['dependencies'] == ['foo']
assert config.get_job('baz')['dependencies'] == ['foo', 'bar']
assert config.get_job_deps('foo') == []
assert config.get_job_deps('bar') == ['foo']
assert config.get_job_deps('baz') == ['foo', 'bar']
assert config.get_job_revdeps('foo') == ['bar', 'baz']
assert config.get_job_revdeps('bar') == ['baz']
assert config.get_job_revdeps('baz') == []
assert config.get_job_deps('does-not-exist') == []
assert config.get_job_revdeps('does-not-exist') == []
def test_job_config_default_values():
config = JobControlConfig.from_string("""
jobs:
- id: foo
custom_key: 'Custom value'
""")
job = config.get_job('foo')
assert isinstance(job, BuildConfig)
assert job['id'] == 'foo'
assert job['function'] is None
assert job['args'] == ()
assert job['kwargs'] == {}
assert job['dependencies'] == []
assert job['pinned_builds'] == {}
assert job['title'] is None
assert job['notes'] is None
assert job['protected'] is False
assert job['cleanup_function'] is None
assert job['repr_function'] is None
with pytest.raises(KeyError):
job['does_not_exist']
assert job['custom_key'] == 'Custom value'
def test_core_config_jobs(storage):
config = JobControlConfig.from_string("""
jobs:
- id: foo
function: mymodule.foo
dependencies: []
- id: bar
function: mymodule.bar
dependencies: ['foo']
- id: baz
function: mymodule.baz
dependencies: ['foo', 'bar']
""")
jc = JobControl(storage=storage, config=config)
job_foo = jc.get_job('foo')
job_bar = jc.get_job('bar')
job_baz = jc.get_job('baz')
# Check jobs
# ------------------------------------------------------------
assert isinstance(job_foo, JobInfo)
assert job_foo.id == 'foo'
assert job_foo.config['id'] == 'foo'
assert job_foo.config['function'] == 'mymodule.foo'
assert job_foo.config['args'] == ()
assert job_foo.config['kwargs'] == {}
assert job_foo.config['dependencies'] == []
assert list(job_foo.get_deps()) == []
assert list(job_foo.get_revdeps()) == [job_bar, job_baz]
assert job_foo.get_status() == 'not_built'
assert list(job_foo.iter_builds()) == []
assert job_foo.get_latest_successful_build() is None
assert job_foo.has_builds() is False
assert job_foo.has_successful_builds() is False
assert job_foo.has_running_builds() is False
assert job_foo.is_outdated() is None # no builds..
assert job_foo.can_be_built() is True
assert isinstance(job_bar, JobInfo)
assert job_bar.id == 'bar'
assert job_bar.config['id'] == 'bar'
assert job_bar.config['function'] == 'mymodule.bar'
assert job_bar.config['args'] == ()
assert job_bar.config['kwargs'] == {}
assert job_bar.config['dependencies'] == ['foo']
assert list(job_bar.get_deps()) == [job_foo]
assert list(job_bar.get_revdeps()) == [job_baz]
assert job_bar.get_status() == 'not_built'
assert list(job_bar.iter_builds()) == []
assert job_bar.get_latest_successful_build() is None
assert job_bar.has_builds() is False
assert job_bar.has_successful_builds() is False
assert job_bar.has_running_builds() is False
assert job_bar.is_outdated() is None # no builds..
assert job_bar.can_be_built() is False # "foo" has no builds
assert isinstance(job_baz, JobInfo)
assert job_baz.id == 'baz'
assert job_baz.config['id'] == 'baz'
assert job_baz.config['function'] == 'mymodule.baz'
assert job_baz.config['args'] == ()
assert job_baz.config['kwargs'] == {}
assert job_baz.config['dependencies'] == ['foo', 'bar']
assert list(job_baz.get_deps()) == [job_foo, job_bar]
assert list(job_baz.get_revdeps()) == []
assert job_baz.get_status() == 'not_built'
assert list(job_baz.iter_builds()) == []
assert job_baz.get_latest_successful_build() is None
assert job_baz.has_builds() is False
assert job_baz.has_successful_builds() is False
assert job_baz.has_running_builds() is False
assert job_baz.is_outdated() is None # no builds..
assert job_baz.can_be_built() is False # "foo" and "bar" have no builds
# Exception on non-existing job
with pytest.raises(NotFound):
jc.get_job('does-not-exist')
# Iterate jobs
assert list(jc.iter_jobs()) == [job_foo, job_bar, job_baz]
def test_dependency_pinning(storage):
# Test for dependency pinning
# ---------------------------
#
# We want to make sure that a build uses the latest build for
# a dependency at the time it was created; so if we run a build
# for job-1, then create a build for job-2, then run another build
# for job-1, the return values used when running a build for job-2
# will be the one from the *first* build.
# To ensure this, we are going to change the return value
# in the configuration.
config = dedent("""\
jobs:
- id: job-1
function: jobcontrol.utils.testing:testing_job
kwargs:
retval: "original-retval"
- id: job-2
function: jobcontrol.utils.testing:testing_job
kwargs:
retval: !retval 'job-1'
dependencies: ['job-1']
""")
config = JobControlConfig.from_string(config)
jc = JobControl(storage=storage, config=config)
build_1_1 = jc.create_build('job-1')
build_1_1.run()
build_1_1.refresh()
assert build_1_1['finished'] and build_1_1['success']
assert build_1_1['retval'] == 'original-retval'
# This should have pinned dependency on build_1_1
build_2_1 = jc.create_build('job-2')
# Update configuration
# --------------------
config = dedent("""\
jobs:
- id: job-1
function: jobcontrol.utils.testing:testing_job
kwargs:
retval: "new-retval"
- id: job-2
function: jobcontrol.utils.testing:testing_job
kwargs:
retval: !retval 'job-1'
dependencies: ['job-1']
""")
config = JobControlConfig.from_string(config)
jc = JobControl(storage=storage, config=config)
build_1_2 = jc.create_build('job-1')
build_1_2.run()
build_1_2.refresh()
assert build_1_2['finished'] and build_1_2['success']
assert build_1_2['retval'] == 'new-retval'
build_2_1 = jc.get_build(build_2_1.id) # Get from *new* JC
build_2_1.run()
build_2_1.refresh()
assert build_2_1['finished'] and build_2_1['success']
assert build_2_1['retval'] == 'original-retval'
build_2_2 = jc.create_build('job-2')
build_2_2.run()
build_2_2.refresh()
assert build_2_2['finished'] and build_2_2['success']
assert build_2_2['retval'] == 'new-retval'
def test_job_status_reporting(storage):
config = JobControlConfig.from_string("""
jobs:
- id: job-1
function: jobcontrol.utils.testing:testing_job
- id: job-2
function: jobcontrol.utils.testing:testing_job
- id: job-3
function: jobcontrol.utils.testing:testing_job
dependencies: ['job-1', 'job-2']
- id: job-4
function: jobcontrol.utils.testing:testing_job
dependencies: ['job-3']
""")
jc = JobControl(storage=storage, config=config)
# Check status of unbuilt jobs
job_1 = jc.get_job('job-1')
job_2 = jc.get_job('job-2')
job_3 = jc.get_job('job-3')
job_4 = jc.get_job('job-4')
assert job_1.get_status() == 'not_built'
assert job_2.get_status() == 'not_built'
assert job_3.get_status() == 'not_built'
assert job_4.get_status() == 'not_built'
assert list(job_1.iter_builds()) == []
assert job_1.get_latest_successful_build() is None
assert job_1.has_builds() is False
assert job_1.has_successful_builds() is False
assert job_1.has_running_builds() is False
assert job_1.is_outdated() is None # IDK
assert job_1.can_be_built() is True
assert job_2.can_be_built() is True
assert job_3.can_be_built() is False # deps not met
assert job_4.can_be_built() is False # deps not met
# ------------------------------------------------------------
# Manually start a build for job 1, as we want to
# check it is running, etc..
# ------------------------------------------------------------
build_1_1 = job_1.create_build()
assert build_1_1['started'] is False
assert build_1_1['finished'] is False
assert job_1.has_builds() is False
assert job_1.has_running_builds() is False
assert job_1.get_status() == 'not_built'
jc.storage.start_build(build_1_1.id)
build_1_1.refresh()
assert build_1_1['started'] is True
assert build_1_1['finished'] is False
assert job_1.has_builds() is False # **Completed** builds..
assert job_1.has_running_builds() is True
# Note: "running" is not anymore reported as a state
assert job_1.get_status() == 'not_built'
jc.storage.finish_build(build_1_1.id, success=False)
build_1_1.refresh()
assert build_1_1['started'] is True
assert build_1_1['finished'] is True
assert build_1_1['success'] is False
assert job_1.has_builds() is True
assert job_1.has_successful_builds() is False
assert job_1.has_running_builds() is False
assert job_1.get_status() == 'failed'
# ------------------------------------------------------------
# Do it again, with a new build, which should succeed this time
# ------------------------------------------------------------
build_1_2 = job_1.create_build()
build_1_2.run()
build_1_2.refresh()
assert len(list(job_1.iter_builds())) == 2
assert build_1_2['started'] is True
assert build_1_2['finished'] is True
assert build_1_2['success'] is True
assert job_1.has_builds() is True
assert job_1.has_successful_builds() is True
assert job_1.has_running_builds() is False
assert job_1.get_status() == 'success'
# ------------------------------------------------------------
# Now build job 2 and make sure 3 becomes buildable
# ------------------------------------------------------------
assert job_3.can_be_built() is False
build_2_1 = job_2.create_build()
build_2_1.run()
assert job_3.can_be_built() is True
# Job 4 is still missing a build from 3
assert job_4.can_be_built() is False
job_3.create_build().run()
assert job_4.can_be_built() is True
assert job_2.get_status() == 'success'
assert job_3.get_status() == 'success'
assert job_4.get_status() == 'not_built'
# ------------------------------------------------------------
# Rebuild #1 to get #3 to be "outdated"
# ------------------------------------------------------------
assert job_3.is_outdated() is False
job_1.create_build().run()
assert job_3.is_outdated() is True
assert job_3.get_status() == 'outdated'
def test_core_config_jobs(storage):
config = JobControlConfig.from_string("""
jobs:
- id: foo
function: mymodule.foo
dependencies: []
- id: bar
function: mymodule.bar
dependencies: ['foo']
- id: baz
function: mymodule.baz
dependencies: ['foo', 'bar']
""")
jc = JobControl(storage=storage, config=config)
job_foo = jc.get_job('foo')
job_bar = jc.get_job('bar')
job_baz = jc.get_job('baz')
# Check jobs
# ------------------------------------------------------------
assert isinstance(job_foo, JobInfo)
assert job_foo.id == 'foo'
assert job_foo.config['id'] == 'foo'
assert job_foo.config['function'] == 'mymodule.foo'
assert job_foo.config['args'] == ()
assert job_foo.config['kwargs'] == {}
assert job_foo.config['dependencies'] == []
assert list(job_foo.get_deps()) == []
assert list(job_foo.get_revdeps()) == [job_bar, job_baz]
assert job_foo.get_status() == 'not_built'
assert list(job_foo.iter_builds()) == []
assert job_foo.get_latest_successful_build() is None
assert job_foo.has_builds() is False
assert job_foo.has_successful_builds() is False
assert job_foo.has_running_builds() is False
assert job_foo.is_outdated() is None # no builds..
assert job_foo.can_be_built() is True
assert isinstance(job_bar, JobInfo)
assert job_bar.id == 'bar'
assert job_bar.config['id'] == 'bar'
assert job_bar.config['function'] == 'mymodule.bar'
assert job_bar.config['args'] == ()
assert job_bar.config['kwargs'] == {}
assert job_bar.config['dependencies'] == ['foo']
assert list(job_bar.get_deps()) == [job_foo]
assert list(job_bar.get_revdeps()) == [job_baz]
assert job_bar.get_status() == 'not_built'
assert list(job_bar.iter_builds()) == []
assert job_bar.get_latest_successful_build() is None
assert job_bar.has_builds() is False
assert job_bar.has_successful_builds() is False
assert job_bar.has_running_builds() is False
assert job_bar.is_outdated() is None # no builds..
assert job_bar.can_be_built() is False # "foo" has no builds
assert isinstance(job_baz, JobInfo)
assert job_baz.id == 'baz'
assert job_baz.config['id'] == 'baz'
assert job_baz.config['function'] == 'mymodule.baz'
assert job_baz.config['args'] == ()
assert job_baz.config['kwargs'] == {}
assert job_baz.config['dependencies'] == ['foo', 'bar']
assert list(job_baz.get_deps()) == [job_foo, job_bar]
assert list(job_baz.get_revdeps()) == []
assert job_baz.get_status() == 'not_built'
assert list(job_baz.iter_builds()) == []
assert job_baz.get_latest_successful_build() is None
assert job_baz.has_builds() is False
assert job_baz.has_successful_builds() is False
assert job_baz.has_running_builds() is False
assert job_baz.is_outdated() is None # no builds..
assert job_baz.can_be_built() is False # "foo" and "bar" have no builds
# Exception on non-existing job
with pytest.raises(NotFound):
jc.get_job('does-not-exist')
# Iterate jobs
assert list(jc.iter_jobs()) == [job_foo, job_bar, job_baz]
def test_job_status_reporting(storage):
config = JobControlConfig.from_string("""
jobs:
- id: job-1
function: jobcontrol.utils.testing:testing_job
- id: job-2
function: jobcontrol.utils.testing:testing_job
- id: job-3
function: jobcontrol.utils.testing:testing_job
dependencies: ['job-1', 'job-2']
- id: job-4
function: jobcontrol.utils.testing:testing_job
dependencies: ['job-3']
""")
jc = JobControl(storage=storage, config=config)
# Check status of unbuilt jobs
job_1 = jc.get_job('job-1')
job_2 = jc.get_job('job-2')
job_3 = jc.get_job('job-3')
job_4 = jc.get_job('job-4')
assert job_1.get_status() == 'not_built'
assert job_2.get_status() == 'not_built'
assert job_3.get_status() == 'not_built'
assert job_4.get_status() == 'not_built'
assert list(job_1.iter_builds()) == []
assert job_1.get_latest_successful_build() is None
assert job_1.has_builds() is False
assert job_1.has_successful_builds() is False
assert job_1.has_running_builds() is False
assert job_1.is_outdated() is None # IDK
assert job_1.can_be_built() is True
assert job_2.can_be_built() is True
assert job_3.can_be_built() is False # deps not met
assert job_4.can_be_built() is False # deps not met
# ------------------------------------------------------------
# Manually start a build for job 1, as we want to
# check it is running, etc..
# ------------------------------------------------------------
build_1_1 = job_1.create_build()
assert build_1_1['started'] is False
assert build_1_1['finished'] is False
assert job_1.has_builds() is False
assert job_1.has_running_builds() is False
assert job_1.get_status() == 'not_built'
jc.storage.start_build(build_1_1.id)
build_1_1.refresh()
assert build_1_1['started'] is True
assert build_1_1['finished'] is False
assert job_1.has_builds() is False # **Completed** builds..
assert job_1.has_running_builds() is True
# Note: "running" is not anymore reported as a state
assert job_1.get_status() == 'not_built'
jc.storage.finish_build(build_1_1.id, success=False)
build_1_1.refresh()
assert build_1_1['started'] is True
assert build_1_1['finished'] is True
assert build_1_1['success'] is False
assert job_1.has_builds() is True
assert job_1.has_successful_builds() is False
assert job_1.has_running_builds() is False
assert job_1.get_status() == 'failed'
# ------------------------------------------------------------
# Do it again, with a new build, which should succeed this time
# ------------------------------------------------------------
build_1_2 = job_1.create_build()
build_1_2.run()
build_1_2.refresh()
assert len(list(job_1.iter_builds())) == 2
assert build_1_2['started'] is True
assert build_1_2['finished'] is True
assert build_1_2['success'] is True
assert job_1.has_builds() is True
assert job_1.has_successful_builds() is True
assert job_1.has_running_builds() is False
assert job_1.get_status() == 'success'
# ------------------------------------------------------------
# Now build job 2 and make sure 3 becomes buildable
# ------------------------------------------------------------
assert job_3.can_be_built() is False
build_2_1 = job_2.create_build()
build_2_1.run()
assert job_3.can_be_built() is True
# Job 4 is still missing a build from 3
assert job_4.can_be_built() is False
job_3.create_build().run()
assert job_4.can_be_built() is True
assert job_2.get_status() == 'success'
assert job_3.get_status() == 'success'
assert job_4.get_status() == 'not_built'
# ------------------------------------------------------------
# Rebuild #1 to get #3 to be "outdated"
# ------------------------------------------------------------
assert job_3.is_outdated() is False
job_1.create_build().run()
assert job_3.is_outdated() is True
assert job_3.get_status() == 'outdated'
def test_dependency_pinning(storage):
# Test for dependency pinning
# ---------------------------
#
# We want to make sure that a build uses the latest build for
# a dependency at the time it was created; so if we run a build
# for job-1, then create a build for job-2, then run another build
# for job-1, the return values used when running a build for job-2
# will be the one from the *first* build.
# To ensure this, we are going to change the return value
# in the configuration.
config = dedent("""\
jobs:
- id: job-1
function: jobcontrol.utils.testing:testing_job
kwargs:
retval: "original-retval"
- id: job-2
function: jobcontrol.utils.testing:testing_job
kwargs:
retval: !retval 'job-1'
dependencies: ['job-1']
""")
config = JobControlConfig.from_string(config)
jc = JobControl(storage=storage, config=config)
build_1_1 = jc.create_build('job-1')
build_1_1.run()
build_1_1.refresh()
assert build_1_1['finished'] and build_1_1['success']
assert build_1_1['retval'] == 'original-retval'
# This should have pinned dependency on build_1_1
build_2_1 = jc.create_build('job-2')
# Update configuration
# --------------------
config = dedent("""\
jobs:
- id: job-1
function: jobcontrol.utils.testing:testing_job
kwargs:
retval: "new-retval"
- id: job-2
function: jobcontrol.utils.testing:testing_job
kwargs:
retval: !retval 'job-1'
dependencies: ['job-1']
""")
config = JobControlConfig.from_string(config)
jc = JobControl(storage=storage, config=config)
build_1_2 = jc.create_build('job-1')
build_1_2.run()
build_1_2.refresh()
assert build_1_2['finished'] and build_1_2['success']
assert build_1_2['retval'] == 'new-retval'
build_2_1 = jc.get_build(build_2_1.id) # Get from *new* JC
build_2_1.run()
build_2_1.refresh()
assert build_2_1['finished'] and build_2_1['success']
assert build_2_1['retval'] == 'original-retval'
build_2_2 = jc.create_build('job-2')
build_2_2.run()
build_2_2.refresh()
assert build_2_2['finished'] and build_2_2['success']
assert build_2_2['retval'] == 'new-retval'
def __init__(self, storage, config):
self.storage = storage
if not isinstance(config, JobControlConfig):
config = JobControlConfig(config)
self.config = config