def _republish_for_estimation(self, task_declaration: TaskDeclaration):
assert task_declaration.estimators_needed > 0
task_declaration.state = TaskDeclaration.State.ESTIMATE_IS_REQUIRED
task_declaration.save()
for ei in task_declaration.get_estimation_assignments(states=(EstimationAssignment.State.REJECTED,)):
ei.state = EstimationAssignment.State.REASSIGN
# return back ownership
ei.save(recipients=ei.estimator.address)
def _process_estimate_is_in_progress(self, task_declaration: TaskDeclaration):
assert task_declaration.state == TaskDeclaration.State.ESTIMATE_IS_IN_PROGRESS
estimation_assignments = task_declaration.get_estimation_assignments(
states=(
EstimationAssignment.State.ESTIMATING,
EstimationAssignment.State.FINISHED
)
)
finished_assignments = []
count_timeout = 0
with async_commit():
for ea in estimation_assignments:
if ea.state == EstimationAssignment.State.ESTIMATING:
if ea.estimation_result.state == EstimationResult.State.FINISHED:
ea.state = EstimationAssignment.State.FINISHED
ea.save()
else:
estimate_timeout = settings.WAIT_ESTIMATE_TIMEOUT
now = datetime.datetime.utcnow().replace(tzinfo=ea.estimation_result.modified_at.tzinfo)
if (now - ea.estimation_result.modified_at).total_seconds() > estimate_timeout:
ea.state = EstimationAssignment.State.TIMEOUT
ea.save()
logger.info('Timeout of waiting for {}'.format(ea))
count_timeout += 1
if ea.state == EstimationAssignment.State.FINISHED:
finished_assignments.append(ea)
if count_timeout:
task_declaration.estimators_needed += count_timeout
self._republish_for_estimation(task_declaration)
return
if len(finished_assignments) == task_declaration.estimators_requested:
task_declaration.state = TaskDeclaration.State.ESTIMATED
task_declaration.estimated_tflops, failed = Estimator.estimate(task_declaration, finished_assignments)
if failed:
logger.info('{} is failed'.format(task_declaration))
task_declaration.state = TaskDeclaration.State.FAILED
task_declaration.save()
return
logger.info('Wait of finish for estimation {}, finished: {}, requested: {}'.format(
task_declaration, len(finished_assignments), task_declaration.estimators_requested
))
def _process_estimate_is_required(self, task_declaration: TaskDeclaration):
assert task_declaration.state == TaskDeclaration.State.ESTIMATE_IS_REQUIRED
with async_commit():
save = False
for ea in task_declaration.get_estimation_assignments(states=(EstimationAssignment.State.READY,)):
if self._is_estimation_assignment_allowed(task_declaration, ea):
ea.state = EstimationAssignment.State.ACCEPTED
ea.save()
task_declaration.estimators_needed -= 1
save = True
else:
ea.state = EstimationAssignment.State.REJECTED
ea.save()
# save changes
if save:
task_declaration.save()
if task_declaration.estimators_needed == 0:
# in assign changes will be saved
self._assign_estimate_data(task_declaration)
def _assign_estimate_data(self, task_declaration: TaskDeclaration):
estimation_assignments = task_declaration.get_estimation_assignments(
states=(EstimationAssignment.State.ACCEPTED, EstimationAssignment.State.TIMEOUT)
)
# split accepted and overdue
accepted_estimation_assignments = []
timeout_estimation_assignments = []
for ea in estimation_assignments:
if ea.state == EstimationAssignment.State.ACCEPTED:
accepted_estimation_assignments.append(ea)
continue
if ea.state == EstimationAssignment.State.TIMEOUT:
timeout_estimation_assignments.append(ea)
continue
assert False and 'Check query!'
if len(timeout_estimation_assignments):
# its reassign
assert len(timeout_estimation_assignments) == len(accepted_estimation_assignments)
for index, ea in enumerate(accepted_estimation_assignments):
timeout_ea = timeout_estimation_assignments[index]
# reassign estimation data
# retrieve data which producer is able to encrypt
estimation_data = EstimationData.get_with_initial_data(
asset_id=timeout_ea.estimation_data_id,
db=self.db,
encryption=self.encryption
)
estimation_data.estimation_assignment_id = ea.asset_id
# share data with new estimator
estimation_data.set_encryption_key(ea.estimator.enc_key)
estimation_data.save()
ea.estimation_data_id = estimation_data.asset_id
ea.state = EstimationAssignment.State.ESTIMATING
ea.save()
timeout_ea.state = EstimationAssignment.State.FORGOTTEN
timeout_ea.save()
else:
estimation_data_params = Estimator.get_data_for_estimate(task_declaration)
for ea in accepted_estimation_assignments:
# create initial state with encrypted data which producer will be able to decrypt
estimation_data = EstimationData.create(
db=self.db,
encryption=self.encryption,
**estimation_data_params
)
# share data with estimator
estimation_data.estimation_assignment_id = ea.asset_id
estimation_data.set_encryption_key(ea.estimator.enc_key)
estimation_data.save()
ea.estimation_data_id = estimation_data.asset_id
ea.state = EstimationAssignment.State.ESTIMATING
ea.save()
task_declaration.state = TaskDeclaration.State.ESTIMATE_IS_IN_PROGRESS
task_declaration.save()