def register_job() -> cg_json.JSONResponse:
"""Register a new job.
If needed a runner will be started for this job.
"""
remote_job_id = g.data['job_id']
cg_url = g.cg_instance_url
job = None
if request.method == 'PUT':
job = db.session.query(models.Job).filter_by(
remote_id=remote_job_id,
cg_url=cg_url,
).with_for_update().one_or_none()
will_create = job is None
if will_create and g.data.get('error_on_create', False):
raise NotFoundException
if job is None:
job = models.Job(
remote_id=remote_job_id,
cg_url=cg_url,
)
db.session.add(job)
job.update_metadata(g.data.get('metadata', {}))
if 'wanted_runners' in g.data or will_create:
job.wanted_runners = g.data.get('wanted_runners', 1)
active_runners = models.Runner.get_all_active_runners().filter_by(
job_id=job.id
).with_for_update().all()
too_many = len(active_runners) - job.wanted_runners
logger.info(
'Job was updated',
wanted_runners=job.wanted_runners,
amount_active=len(active_runners),
too_many=too_many,
metadata=job.job_metadata,
)
before_assigned = set(models.RunnerState.get_before_assigned_states())
for runner in active_runners:
if too_many <= 0:
break
if runner.state in before_assigned:
runner.make_unassigned()
too_many -= 1
db.session.flush()
job_id = job.id
callback_after_this_request(
lambda: tasks.maybe_start_runners_for_job.delay(job_id)
)
db.session.commit()
return cg_json.jsonify(job)
def _kill_slow_starter(runner_hex_id: str) -> None:
runner_id = uuid.UUID(hex=runner_hex_id)
runner = db.session.query(
models.Runner).filter_by(id=runner_id).with_for_update().one_or_none()
if runner is None:
logger.info('Cannot find runner')
return
elif runner.state not in models.RunnerState.get_before_started_states():
logger.info('Runner already started',
runner=runner,
state=runner.state)
return
should_run_date = runner.created_at + datetime.timedelta(
minutes=app.config['START_TIMEOUT_TIME'])
if utils.maybe_delay_current_task(should_run_date):
return
logger.info('Killing runner', runner=runner, state_of_runner=runner.state)
if runner.job_id is not None:
# For some strange reason the runner still had a job. In this case we
# should make a new runner for that job after we killed this runner.
job_id = runner.job_id
callback_after_this_request(
lambda: maybe_start_runners_for_job.delay(job_id))
else:
callback_after_this_request(maybe_start_more_runners)
runner.kill(maybe_start_new=False, shutdown_only=False)
def make_unassigned(self) -> None:
"""Make this runner unassigned.
.. note::
This also starts a job to kill this runner after a certain amount
of time if it is still unassigned.
"""
runner_hex_id = self.id.hex
self.job_id = None
if self.state in RunnerState.get_before_running_states():
if self.state.is_assigned:
self.state = RunnerState.started
eta = DatetimeWithTimezone.utcnow() + timedelta(
minutes=app.config['RUNNER_MAX_TIME_ALIVE']
)
callback_after_this_request(
lambda: cg_broker.tasks.maybe_kill_unneeded_runner.apply_async(
(runner_hex_id, ),
eta=eta,
)
)
else:
self.state = RunnerState.cleaning
callback_after_this_request(
lambda: cg_broker.tasks.kill_runner.delay(runner_hex_id)
)
def schedule_attachment_deletion(self) -> None:
"""Delete the attachment of this result after the current request.
The attachment, if present, will be deleted, if not attachment is
present this function does nothing.
:returns: Nothing.
"""
old_attachment = self.attachment
if old_attachment.is_just:
deleter = old_attachment.value.delete
callback_after_this_request(deleter)
def maybe_start_more_runners() -> None:
"""Start more runners for jobs that still need runners.
This assigns all unassigned runners if possible, and if needed it also
starts new runners.
"""
# We might change the amount of unassigned runners during this task, so
# maybe we need to start more.
callback_after_this_request(start_needed_unassigned_runners.delay)
# Do not use count so we can lock all these runners, this makes sure we
# will not create new runners while we are counting.
active_jobs = {
job.id: job
for job in db.session.query(models.Job).filter(
models.Job.state != models.JobState.finished).with_for_update()
}
unassigned_runners = models.Runner.get_all_active_runners().filter_by(
job_id=None).with_for_update().all()
jobs_needed_runners = db.session.query(
models.Job.id,
models.Job.wanted_runners - sql_func.count(models.Runner.id),
).filter(models.Job.id.in_(list(active_jobs.keys())), ).join(
models.Runner,
and_(models.Runner.job_id == models.Job.id,
models.Runner.state.in_(models.RunnerState.get_active_states())),
isouter=True).having(
models.Job.wanted_runners > sql_func.count(
models.Runner.id), ).group_by(models.Job.id).all()
with bound_to_logger(
jobs_needed_runners=jobs_needed_runners,
all_unassigned_runners=unassigned_runners,
):
if jobs_needed_runners:
startable = models.Runner.get_amount_of_startable_runners()
logger.info('More runners are needed')
for job_id, _ in jobs_needed_runners:
# This never raises a key error as only jobs in this dictionary
# are selected.
job = active_jobs[job_id]
startable -= job.add_runners_to_job(unassigned_runners,
startable)
db.session.commit()
elif unassigned_runners:
logger.error('More runners than jobs active')
else:
logger.info('No new runners needed')
def register_job() -> cg_json.JSONResponse:
"""Register a new job.
If needed a runner will be started for this job.
"""
remote_job_id = g.data['job_id']
cg_url = request.headers['CG-Broker-Instance']
job = None
if request.method == 'PUT':
job = db.session.query(models.Job).filter_by(
remote_id=remote_job_id,
cg_url=cg_url,
).one_or_none()
if job is None:
job = models.Job(
remote_id=remote_job_id,
cg_url=cg_url,
)
db.session.add(job)
job.update_metadata(g.data.get('metadata', {}))
job.wanted_runners = g.data.get('wanted_runners', 1)
active_runners = models.Runner.get_all_active_runners().filter_by(
job_id=job.id).with_for_update().all()
too_many = len(active_runners) - job.wanted_runners
logger.info(
'Job was updated',
wanted_runners=job.wanted_runners,
amount_active=len(active_runners),
too_many=too_many,
metadata=job.job_metadata,
)
for runner in active_runners:
if too_many <= 0:
break
if runner.state in models.RunnerState.get_before_running_states():
runner.make_unassigned()
too_many -= 1
db.session.commit()
job_id = job.id
callback_after_this_request(
lambda: tasks.maybe_start_runners_for_job.delay(job_id))
assert job.id is not None
return cg_json.jsonify(job)
def add_runners_to_job(self, unassigned_runners: t.List[Runner],
startable: int) -> int:
"""Add runners to the given job.
This adds runners from the ``unassigned_runners`` list to the current
job, or starts new runners as long as ``startable`` is greater than 0.
.. note:: The ``unassigned_runners`` list may be mutated in place.
:param unassigned_runners: Runners that are not assigned yet and can be
used by this job.
:param startable: The amount of runners we may start.
:returns: The amount of new runners started.
"""
needed = max(0, self.wanted_runners - len(self.get_active_runners()))
to_start: t.List[uuid.UUID] = []
created: t.List[Runner] = []
if needed > 0 and unassigned_runners:
# We will assign runner that were previously unassigned, so we
# might need to start some extra runners.
callback_after_this_request(
cg_broker.tasks.start_needed_unassigned_runners.delay)
for _ in range(needed):
if unassigned_runners:
self.runners.append(unassigned_runners.pop())
elif startable > 0:
runner = Runner.create_of_type(app.config['AUTO_TEST_TYPE'])
self.runners.append(runner)
db.session.add(runner)
created.append(runner)
startable -= 1
else:
break
db.session.flush()
for runner in created:
to_start.append(runner.id)
def start_runners() -> None:
for runner_id in to_start:
cg_broker.tasks.start_runner.delay(runner_id.hex)
callback_after_this_request(start_runners)
return len(created)
def update_backing_file(self,
new_file: cg_object_storage.File,
*,
delete: bool = False) -> None:
"""Replace the backing file of this ``File``.
:param new_file: The new backing file for this row.
:param delete: If ``True`` we will delete the old file at the end of
the request.
:raises AssertionError: When called on a directory.
"""
if self.is_directory: # pragma: no cover
raise AssertionError('Cannot set file of directory')
if delete:
cg_flask_helpers.callback_after_this_request(self._get_deleter())
self._filename = new_file.name
def make_unassigned(self) -> None:
"""Make this runner unassigned.
.. note::
This also starts a job to kill this runner after a certain amount
of time if it is still unassigned.
"""
self.job_id = None
eta = DatetimeWithTimezone.utcnow() + timedelta(
minutes=app.config['RUNNER_MAX_TIME_ALIVE'])
runner_hex_id = self.id.hex
callback_after_this_request(
lambda: cg_broker.tasks.maybe_kill_unneeded_runner.apply_async(
(runner_hex_id, ),
eta=eta,
))
def schedule_attachment_deletion(self) -> None:
"""Delete the attachment of this result after the current request.
The attachment, if present, will be deleted, if not attachment is
present this function does nothing.
:returns: Nothing.
"""
old_filename = self.attachment_filename
if old_filename is not None:
old_path = psef.files.safe_join(current_app.config['UPLOAD_DIR'],
old_filename)
def after_req() -> None:
if os.path.isfile(old_path):
os.unlink(old_path)
callback_after_this_request(after_req)
def __was_updated(
target: types.Base,
_value: object,
_oldvalue: object,
_initiator: object,
) -> None:
item = (fun, target)
is_hash = _hashable(item)
info = self.__session.info
validation_set = info.get(_CG_VALIDATOR_SET, None)
if validation_set is None:
validation_set = info[_CG_VALIDATOR_SET] = set()
callback_after_this_request(self._clear_info_data)
if is_hash and item in validation_set:
return
info.setdefault(_CG_VALIDATOR, []).append(item)
if is_hash:
validation_set.add(item)
def _use_runner(self, runner: Runner) -> None:
self.runners.append(runner)
active_runners = self.get_active_runners()
before_assigned = set(RunnerState.get_before_assigned_states())
too_many_active = len(active_runners) > self.wanted_runners
if too_many_active:
unneeded_runner = next(
(r for r in active_runners if r.state in before_assigned),
None,
)
if unneeded_runner is not None:
# In this case we now have too many runners assigned to us, so
# make one of the runners unassigned. But only do this if we
# have a runner which isn't already running.
unneeded_runner.make_unassigned()
callback_after_this_request(
cg_broker.tasks.maybe_start_more_runners.delay
)
def send(self, value: T) -> None:
"""Send an event to this signal.
:param value: The value you want to emit.
:returns: Nothing.
"""
if self.__celery_todo: # pragma: no cover
raise AssertionError(
'The Signal still has uninitialized celery listeners'
)
for _, callback in self.__immediate_callbacks:
callback(value)
if self.__after_request_callbacks:
def send_dispatch() -> None:
for _, callback in self.__after_request_callbacks:
callback(value)
callback_after_this_request(send_dispatch)
def _after_update_minimum_amount_extra_runners() -> None:
callback_after_this_request(
cg_broker.tasks.start_needed_unassigned_runners.delay)
def maybe_use_runner(self, runner: Runner) -> bool:
"""Maybe use the given ``runner`` for this job.
This function checks if this job is allowed to use the given
runner. This is the case if the runner is assigned to us, if it is
unassigned or we can steal it from another job.
:param runner: The runner we might want to use.
:returns: ``True`` if we can use the runner, and ``False`` if we
cannot.
"""
if runner in self.runners:
return True
active_runners = self.get_active_runners()
before_active = set(RunnerState.get_before_running_states())
# In this case we have enough running runners for this job.
if sum(r.state not in before_active
for r in active_runners) >= self.wanted_runners:
logger.info('Too many runners assigned to job')
return False
# We want more, but this should only be given if the runner is not
# needed elsewhere.
# Runner is unassigned, so get it.
if runner.job is None:
self.runners.append(runner)
# However, we might assume that this runner can be used for other
# jobs, so we might need to start more runners.
callback_after_this_request(
cg_broker.tasks.maybe_start_more_runners.delay)
return True
# Runner is assigned but we maybe can steal it.
if self._can_steal_runner(runner):
logger.info(
'Stealing runner from job',
new_job_id=self.id,
other_job_id=runner.job.id,
runner=runner,
)
self.runners.append(runner)
unneeded_runner = next(
(r for r in active_runners if r.state in before_active), None)
too_many_active = len(active_runners) + 1 > self.wanted_runners
if too_many_active and unneeded_runner:
# In this case we now have too many runners assigned to us, so
# make one of the runners unassigned. But only do this if we
# have a runner which isn't already running.
unneeded_runner.make_unassigned()
# The runner we stole might be useful for the other job, as it
# might have requested extra runners. So we might want to start
# extra runners.
callback_after_this_request(
cg_broker.tasks.maybe_start_more_runners.delay)
return True
return False