def compute_speeds():
# Process each installed crowd.
registry = CrowdRegistry.get_registry()
for crowd_name, (crowd_interface, crowd_model_spec) in registry.iteritems():
# Skip crowds that don't support retainer pools.
if not crowd_model_spec.retainer_pool_model:
logger.info("Crowd %s doesn't support retainer pools, not computing worker speeds."
% crowd_name)
continue
# Compute stats needed for speed computation
s = compute_speed_stats.si(crowd_model_spec)
# Find pools that are still active
pool_tasks =[]
logger.info("Crowd %s supports retainer pools, computing worker speeds."
% crowd_name)
valid_states = (RetainerPoolStatus.IDLE, RetainerPoolStatus.ACTIVE,
RetainerPoolStatus.REFILLING)
for pool in crowd_model_spec.retainer_pool_model.objects.filter(
status__in=valid_states):
logger.info("Setting up subtasks for speed computation of %s" % pool)
# Compute all the individual worker speeds as subtasks
g = group([compute_worker_speed.si(w.worker_id, crowd_model_spec)
for w in pool.active_workers])
# Then aggregate them to compute the pool speed
pool_tasks.append(g | compute_pool_speed.si(pool.id, crowd_model_spec))
# Execute all of our subtasks
(s | group(pool_tasks))()
def post_retainer_tasks():
# Process each installed crowd.
registry = CrowdRegistry.get_registry()
for crowd_name, (crowd_interface, crowd_model_spec) in registry.iteritems():
# Skip crowds that don't support retainer pools.
if not crowd_model_spec.retainer_pool_model:
logger.info("Crowd %s doesn't support retainer pools, not posting "
"tasks." % crowd_name)
continue
# Find pools that need more workers.
logger.info("Crowd %s supports retainer pools, looking for pools that "
"need more workers." % crowd_name)
valid_states = (RetainerPoolStatus.RECRUITING, RetainerPoolStatus.IDLE,
RetainerPoolStatus.ACTIVE, RetainerPoolStatus.REFILLING)
for pool in crowd_model_spec.retainer_pool_model.objects.filter(
status__in=valid_states):
group = pool.task_groups.order_by('created_at')[0]
exp_config = json.loads(group.global_config).get('experimental')
if exp_config:
churn_thresh = exp_config.get('churn_threshold')
else:
churn_thresh = None
# Check if the pool needs more workers.
num_active_workers = pool.active_workers.count()
need_more_active = num_active_workers < pool.capacity
logging.info("%s needs workers? %s" % (pool, need_more_active))
num_reserve_workers = pool.reserve_workers.count()
need_more_reserve = (churn_thresh is not None
and num_reserve_workers < settings.CHURN_RESERVE_SIZE)
logging.info("%s needs more reserve? %s" % (pool, need_more_reserve))
# if a pool has finished recruiting, start tasks appropriately
if (pool.status in (RetainerPoolStatus.RECRUITING, RetainerPoolStatus.REFILLING)
and not need_more_active):
logger.info("%s is done recruiting" % pool)
waiting_task_groups = crowd_model_spec.group_model.objects.filter(
retainer_pool=pool,
retainer_pool_status__in=(TaskGroupRetainerStatus.WAITING,
TaskGroupRetainerStatus.RUNNING))
if not waiting_task_groups.exists():
logger.info("No waiting task groups, pool is idle")
pool.status = RetainerPoolStatus.IDLE
else:
logger.info("Waiting task groups found, starting work.")
pool.status = RetainerPoolStatus.ACTIVE
for task_group in waiting_task_groups:
logger.info("%s now running." % task_group)
task_group.retainer_pool_status = (
TaskGroupRetainerStatus.RUNNING)
task_group.save()
pool.save()
# Create new recruitment tasks if necessary
elif need_more_active or need_more_reserve:
logger.info("Posting tasks for %s" % pool)
if (pool.status in (RetainerPoolStatus.ACTIVE, RetainerPoolStatus.IDLE)
and need_more_active):
pool.status = RetainerPoolStatus.REFILLING
now = timezone.now()
if (pool.last_recruited_at > now - timedelta(
seconds=settings.RETAINER_TASK_EXPIRATION_SECONDS)):
logger.info("Pool was recruited recently... skipping.")
continue
pool.last_recruited_at = now
pool.save()
# Create dummy tasks on the crowd platform
dummy_content = json.dumps({})
dummy_config = {
'num_assignments': 1,
'task_type': 'retainer',
'task_batch_size': 1,
'callback_url': '',
crowd_name: json.loads(group.crowd_config),
}
for i in range(1, settings.NUM_RETAINER_RECRUITMENT_TASKS + 1):
task_config = copy.deepcopy(dummy_config)
task_config[crowd_name]['title'] += " [" + str(i) + "]"
task_id = crowd_interface.create_task(task_config, dummy_content)
# skip interface.task_pre_save because this isn't a real task.
task = crowd_model_spec.task_model.objects.create(
task_type=task_config['task_type'],
data=dummy_content,
create_time=timezone.now(),
task_id=task_id,
group=pool.task_groups.order_by('-created_at')[0],
num_assignments=task_config['num_assignments'],
)
logger.info("Created Task %s" % task_id)
# Create the retainer task to remember it.
retainer_task = RetainerTask.objects.create(
task=task, crowd_name=crowd_name)
logger.info("Created %s" % retainer_task)
else:
logger.info("%s has status %s, nothing to do." % (pool, pool.get_status_display()))
# Delete old retainerTasks to keep the listings fresh
logger.info('Removing old retainer tasks...')
for retainer_task in RetainerTask.objects.filter(active=True).select_related(
'task__group__retainer_pool'):
# Did we already process this retainer_task?
session_task = retainer_task.task
if not session_task:
continue
# Make sure we're actually recruiting
group = session_task.group
retainer_pool = group.retainer_pool
# Always kill off recruitment tasks for finished pools
pool_finished = (retainer_pool.status == RetainerPoolStatus.FINISHED)
exp_config = json.loads(group.global_config).get('experimental')
if exp_config:
churn_thresh = exp_config.get('churn_threshold')
else:
churn_thresh = None
num_reserve_workers = retainer_pool.reserve_workers.count()
need_more_reserve = (churn_thresh is not None
and num_reserve_workers < settings.CHURN_RESERVE_SIZE)
not_recruiting = (retainer_pool.status not in (RetainerPoolStatus.RECRUITING,
RetainerPoolStatus.REFILLING)
and not need_more_reserve) or pool_finished
if not_recruiting:
# reset the last_recruited timestamp in case we start recruiting again.
retainer_pool.last_recruited_at = timezone.now() - timedelta(
seconds=settings.RETAINER_TASK_EXPIRATION_SECONDS)
retainer_pool.save()
# Kill off tasks that have been around for too long
old_task_cutoff = (
timezone.now()
- timedelta(seconds=settings.RETAINER_TASK_EXPIRATION_SECONDS))
if not_recruiting or retainer_task.created_at < old_task_cutoff:
try:
was_assigned = session_task.assignments.exists()
interface, _ = CrowdRegistry.get_registry_entry(
retainer_task.crowd_name)
# delete the crowd platform task if no one has accepted it.
if not was_assigned:
interface.delete_tasks([session_task,])
logger.info("Deleted platform task %s" % session_task.task_id)
session_task.delete()
logger.info("Deleted ampcrowd task object %s" % session_task)
retainer_task.active = False
retainer_task.save()
# expire the crowd platform task if the pool is finished
elif pool_finished:
try:
interface.expire_tasks([session_task,])
except Exception as e:
logger.exception("Couldn't expire task %s, ignoring..." % session_task)
logger.info("Expired platform task %s" % session_task.task_id)
retainer_task.active = False
retainer_task.save()
else:
logger.info("Not deleting %s, it has a worker." % session_task)
except Exception, e:
logger.warning('Could not remove task %s: %s' % (session_task, str(e)))
def churn_workers():
# Process each installed crowd.
registry = CrowdRegistry.get_registry()
for crowd_name, (crowd_interface, crowd_model_spec) in registry.iteritems():
# Skip crowds that don't support retainer pools.
if not crowd_model_spec.retainer_pool_model:
logger.info("Crowd %s doesn't support retainer pools, not churning "
"workers" % crowd_name)
continue
# Find pools that support churn
logger.info("Crowd %s supports retainer pools, looking for workers to "
"churn." % crowd_name)
valid_states = (RetainerPoolStatus.IDLE, RetainerPoolStatus.ACTIVE,
RetainerPoolStatus.REFILLING)
for pool in crowd_model_spec.retainer_pool_model.objects.filter(
status__in=valid_states):
group = pool.task_groups.order_by('created_at')[0]
exp_config = json.loads(group.global_config).get('experimental')
if exp_config:
churn_thresh = exp_config.get('churn_threshold')
else:
churn_thresh = None
# Don't churn pools that don't support churn
if churn_thresh is None:
logger.info("Pool doesn't support churn--not churning.")
continue
# Don't churn pools if there are no available reserve workers
num_reserve_workers = pool.reserve_workers.count()
if num_reserve_workers == 0:
logger.info("No reserve workers available--not churning.")
continue
logger.info("%d workers on reserve--looking for workers to replace." %
num_reserve_workers)
# Log the pool average
logger.info("Churning pool %s" % pool)
if pool.mean_speed:
logger.info("Average pool speed: %f. std: %f" % (
pool.mean_speed, pool.mean_speed_std))
# Pick the workers above the threshold
bad_workers = sorted(
[w for w in pool.active_workers
if w.mean_speed and w.mean_speed - w.mean_speed_ci > churn_thresh],
reverse=True,
key=lambda w:w.mean_speed - w.mean_speed_ci)
num_bad_workers = len(bad_workers)
logger.info("Found %d workers above the threshold" % num_bad_workers)
if num_bad_workers == 0:
logger.info("No workers to release--not churning.")
continue
# Perform the churn
n_to_churn = min(num_bad_workers, num_reserve_workers)
logger.info("Churning %d workers" % n_to_churn)
workers_to_churn = bad_workers[:n_to_churn]
workers_to_add = pool.reserve_workers.order_by('?')[:n_to_churn]
assert len(workers_to_churn) == len(workers_to_add)
# Activate workers on reserve
logger.info("Activating reserve workers: %s" % workers_to_add)
(crowd_model_spec.assignment_model.objects
.filter(worker__in=workers_to_add)
.update(on_reserve=False))
# Release slow workers
now = timezone.now()
logger.info("Releasing active workers: %s" % workers_to_churn)
(crowd_model_spec.assignment_model.objects
.filter(worker__in=workers_to_churn)
.update(worker_released_at=now))
pool.num_churns += 1
pool.save()
logger.info('Churn complete.')
def retire_workers():
# Process each installed crowd.
registry = CrowdRegistry.get_registry()
for crowd_name, (crowd_interface, crowd_model_spec) in registry.iteritems():
# Skip crowds that don't support retainer pools.
if not crowd_model_spec.retainer_pool_model:
logger.info("Crowd %s doesn't support retainer pools, not expiring "
"workers" % crowd_name)
continue
# Find pools with expired workers.
logger.info("Crowd %s supports retainer pools, looking for workers to "
"retire." % crowd_name)
for pool in crowd_model_spec.retainer_pool_model.objects.all():
for expired_task in pool.new_expired_tasks(crowd_model_spec.task_model):
logger.info("%s has expired. Cleaning up and paying the "
"workers." % expired_task)
success = True
# Tally the work done by workers on this session task
assignments = expired_task.assignments.all()
logger.info("%d workers on session %s" %
(assignments.count(), expired_task))
for assignment in assignments:
if (assignment.is_active or
(assignment.last_ping is None
and pool.status != RetainerPoolStatus.FINISHED)):
success = False # come back to this assignment later.
continue # assignment hasn't been finished yet.
logger.info("Proccessing assignment %s" % assignment)
worker = assignment.worker
assignment.finished_at = timezone.now()
assignment.finish_waiting_session()
wait_time = assignment.time_waited
logger.info("%s waited %f seconds in this session." %
(worker, wait_time))
# Count tasks the worker has completed during this session.
work_assignments = worker.assignments_for_pool_session(expired_task)
num_tasks = work_assignments.count()
completed_assignments = worker.completed_assignments_for_pool_session(
expired_task)
num_completed_tasks = completed_assignments.count()
logger.info("%s completed %d tasks." % (worker,
num_completed_tasks))
# Make sure the worker has completed the required number of tasks
reject = False
pay = True
group_config = json.loads(expired_task.group.global_config)
retain_config = group_config['retainer_pool']
not_enough_tasks = (num_completed_tasks < retain_config['min_tasks_per_worker']
and num_tasks > 0)
if not_enough_tasks:
logger.info("%s was assigned tasks, but didn't compelete the minimum."
% worker)
if assignment.pool_ended_mid_assignment:
logger.info("work by %s was cut off by the end of the pool, "
"so not rejecting %s." % (worker, assignment))
elif assignment.worker_released_at is not None:
logger.info("work by %s was cut off by churn, so not "
" rejecting %s." % (worker, assignment))
else:
logger.info("Work wasn't cut off by pool, will reject.")
reject = True
pay = False
else:
logger.info("%s completed enough tasks or wasn't assigned any."
% worker)
# Reject the work if they haven't
if reject and assignment.rejected_at is None:
logger.info("Rejecting %s." % assignment)
try:
logger.warning("WOULD REJECT %s, BUT BEING NICE" % assignment)
#crowd_interface.reject_task(
# assignment, worker, "You must complete at least %d "
# "tasks to be approved for this assignment, as stated "
# "in the HIT instructions." %
# retain_config['min_tasks_per_worker'])
assignment.rejected_at = timezone.now()
except Exception, e:
logger.error('Error rejecting %s' % assignment)
success = False
elif assignment.rejected_at is not None:
logger.info("%s already rejected, no need to reject twice." % assignment)
else:
logger.info("%s was valid, and doesn't need rejection." % assignment)
# Pay the worker if we haven't already.
if pay and assignment.paid_at is None and assignment.rejected_at is None:
waiting_rate = retain_config['waiting_rate']
per_task_rate = retain_config['task_rate']
list_rate = retain_config['list_rate']
bonus_amount, message, log_msg = assignment.compute_bonus(
waiting_rate, per_task_rate, list_rate, logger)
logger.info(log_msg)
try:
if bonus_amount <= 0.01:
logger.info("Bonus amount is too small--not giving bonus.")
else:
crowd_interface.pay_worker_bonus(
worker, assignment, bonus_amount, message)
assignment.amount_paid_bonus = bonus_amount
assignment.amount_paid_list = list_rate
assignment.paid_at = timezone.now()
except Exception, e:
logger.error('Error paying for %s' % assignment)
success = False
# payment is important--make it explicit when there's an issue
with open('PAYMENT_ERRORS.txt', 'a') as f:
print >>f, str(assignment), str(e)
elif assignment.paid_at is not None:
logger.info("%s already paid, no need to pay twice." % assignment)
else:
logger.info("%s was rejected, not paying worker." % assignment)
assignment.save()