def postHITs():
global active_hits
startTime = datetime.utcnow()
seq = 0
while unixtime(datetime.utcnow()) < (unixtime(startTime) + JUST_UNDER_A_MINUTE):
flushTransaction()
seq = (seq + 1) % ACTIVE_HIT_NUM # what's a good number of HITs?
time.sleep(2)
#print "seq " + str(seq) + " at " + str(unixtime(datetime.utcnow()))
protos = ProtoHit.objects.all()
for proto in protos:
reservations = unfulfilledReservationsForProto(proto)
for reservation in reservations:
numOnRetainer = len(reservation.workers)
if unixtime(datetime.now()) > (reservation.start_time + settings.RESERVATION_TIMEOUT_MINUTES * 60 * 20):
#print 'not yet expiring: ' + str(reservation)
break
# how many HITs are we already maintaining for this reservation?
existing_hits = Hit.objects.filter(reservation = reservation, removed = False)
num_existing_hits = len(existing_hits)
#print 'existing hits: ' + str(existing_hits)
hit = postHIT(reservation, seq)
active_hits.append(hit)
def postHITs():
global active_hits
startTime = datetime.utcnow()
seq = 0
while unixtime(
datetime.utcnow()) < (unixtime(startTime) + JUST_UNDER_A_MINUTE):
flushTransaction()
seq = (seq + 1) % ACTIVE_HIT_NUM # what's a good number of HITs?
time.sleep(2)
#print "seq " + str(seq) + " at " + str(unixtime(datetime.utcnow()))
protos = ProtoHit.objects.all()
for proto in protos:
reservations = unfulfilledReservationsForProto(proto)
for reservation in reservations:
numOnRetainer = len(reservation.workers)
if unixtime(datetime.now()) > (
reservation.start_time +
settings.RESERVATION_TIMEOUT_MINUTES * 60 * 20):
#print 'not yet expiring: ' + str(reservation)
break
# how many HITs are we already maintaining for this reservation?
existing_hits = Hit.objects.filter(reservation=reservation,
removed=False)
num_existing_hits = len(existing_hits)
#print 'existing hits: ' + str(existing_hits)
hit = postHIT(reservation, seq)
active_hits.append(hit)
def clean_up_old_hits(conn, verbose=True, dry_run=False):
counts = {'cleaned':0, 'remaining':0, 'errors':0}
hits = get_all_hits(conn)
now = timeutils.unixtime(datetime.now())
for h in hits:
try:
num_to_be_reviewed = len(conn.get_assignments(h.HITId, status="Submitted")) # yet to be reviewed
if verbose:
print "------------------"
print "HIT ID: " + h.HITId
expiration = timeutils.parseISO(h.Expiration)
duration = Decimal(h.AssignmentDurationInSeconds)
print "Expiration: " + str(expiration)
print "isExpired: " + str(h.expired)
print "AssignmentDuration: " + str(duration)
print "Seconds until last assignment can be turned in: " + str((float(expiration)+float(duration))-float(now))
print "HITStatus: " + h.HITStatus
print "Yet-to-be-reviewed assignments: " + str(num_to_be_reviewed)
print "Approved assignments: " + str(len(conn.get_assignments(h.HITId, status="Approved")))
print "Rejected assignments: " + str(len(conn.get_assignments(h.HITId, status="Rejected")))
if (h.HITStatus == 'Reviewable' or h.HITStatus == 'Reviewing') and num_to_be_reviewed == 0:
if not dry_run:
conn.dispose_hit(h.HITId)
print "DISPOSED OF HIT"
counts['cleaned'] += 1
if dry_run:
print "(dry run -- HIT not actually disposed)"
else:
counts['remaining'] += 1
except Exception, e:
print "exception processing HIT:\n" + str(e)
counts['errors'] += 1
def workers(self):
flushTransaction() # may no longer be needed after console->cron script conversion
timeCutoff = datetime.now() - timedelta(seconds = PING_TIMEOUT_SECONDS)
waitingWorkers = Assignment.objects.filter( \
event__event_type = 'waiting', \
event__server_time__gte = unixtime(timeCutoff), \
hit_id__reservation = self, \
task_id = None).distinct()
return waitingWorkers
def waiting_workers():
flush_transaction()
time_ping_floor = datetime.now() - timedelta(seconds=PING_TIMEOUT_SECONDS)
waiting_assignments = Assignment.objects.filter(
event__event_type=PING_TYPES['waiting'],
event__server_time__gte=unixtime(time_ping_floor),
task_id=None).distinct()
return waiting_assignments
def postHIT(resv, seq):
proto = resv.proto
qual_arr = []
if proto.worker_locale != '' and proto.worker_locale is not None:
qual_arr.append(LocaleRequirement('EqualTo', proto.worker_locale))
if proto.approval_rating > 0:
qual_arr.append(
PercentAssignmentsApprovedRequirement('GreaterThan',
proto.approval_rating))
quals = Qualifications(qual_arr)
seq_str = " [" + str(seq) + "]"
eh = ExternalHit(title=proto.title + seq_str,
description=proto.description + seq_str,
keywords=proto.keywords,
url=proto.url,
frame_height=proto.frame_height,
reward_as_usd_float=float(proto.reward),
assignment_duration=proto.assignment_duration,
lifetime=proto.lifetime,
max_assignments=proto.max_assignments,
qualifications=quals,
auto_approval_delay=proto.auto_approval_delay)
try:
if proto.sandbox:
turk_hit = eh.post(mt_conn_sandbox)
else:
turk_hit = eh.post(mt_conn_real)
print "Posted HIT ID " + turk_hit.HITId + " for reservation: " + str(
resv)
django_hit = Hit(proto_id=proto.id,
hit_id=turk_hit.HITId,
reservation=resv,
hit_type_id=proto.hit_type_id,
title=proto.title + seq_str,
description=proto.description + seq_str,
keywords=proto.keywords,
url=proto.url,
reward=proto.reward,
assignment_duration=proto.assignment_duration,
lifetime=proto.lifetime,
max_assignments=proto.max_assignments,
auto_approval_delay=proto.auto_approval_delay,
worker_locale=proto.worker_locale,
approval_rating=proto.approval_rating,
creation_time=unixtime(datetime.utcnow()),
sandbox=proto.sandbox)
django_hit.save()
return django_hit
except Exception, e:
print "Got exception posting HIT:\n" + str(e)
def postHIT(resv, seq):
proto = resv.proto
qual_arr = []
if proto.worker_locale != '' and proto.worker_locale is not None:
qual_arr.append(LocaleRequirement('EqualTo', proto.worker_locale))
if proto.approval_rating > 0:
qual_arr.append(PercentAssignmentsApprovedRequirement('GreaterThan', proto.approval_rating))
quals = Qualifications(qual_arr)
seq_str = " [" + str(seq) + "]"
eh = ExternalHit(title=proto.title + seq_str,
description = proto.description + seq_str,
keywords = proto.keywords,
url = proto.url,
frame_height = proto.frame_height,
reward_as_usd_float = float(proto.reward),
assignment_duration = proto.assignment_duration,
lifetime = proto.lifetime,
max_assignments = proto.max_assignments,
qualifications = quals,
auto_approval_delay = proto.auto_approval_delay)
try:
if proto.sandbox:
turk_hit = eh.post(mt_conn_sandbox)
else:
turk_hit = eh.post(mt_conn_real)
print "Posted HIT ID " + turk_hit.HITId + " for reservation: " + str(resv)
django_hit = Hit(proto_id = proto.id,
hit_id = turk_hit.HITId,
reservation = resv,
hit_type_id = proto.hit_type_id,
title = proto.title + seq_str,
description = proto.description + seq_str,
keywords = proto.keywords,
url = proto.url,
reward = proto.reward,
assignment_duration = proto.assignment_duration,
lifetime = proto.lifetime,
max_assignments = proto.max_assignments,
auto_approval_delay = proto.auto_approval_delay,
worker_locale = proto.worker_locale,
approval_rating = proto.approval_rating,
creation_time = unixtime(datetime.utcnow()),
sandbox = proto.sandbox)
django_hit.save()
return django_hit
except Exception, e:
print "Got exception posting HIT:\n" + str(e)
def ping(request, worker_id, assignment_id, hit_id, ping_type):
assignment = makeOrGetAssignment(assignment_id, worker_id, hit_id)
if assignment is None:
return HttpResponse('Bad assignment')
ping = PING_TYPES[ping_type]
event = Event(assignment = assignment, event_type = ping,
ip = request.META['REMOTE_ADDR'],
user_agent = request.META['HTTP_USER_AGENT'],
server_time = unixtime(datetime.now()),
client_time = 0, # TODO: fix. add to url params or switch to POST
detail = '')
event.save()
return HttpResponse('OK')
def ping(request, worker_id, assignment_id, hit_id, ping_type):
assignment = get_assignment(assignment_id, worker_id, hit_id)
if assignment is None:
return HttpResponse('Bad assignment')
ping = PING_TYPES[ping_type]
event = Event(
assignment=assignment,
event_type=ping,
ip=request.META['REMOTE_ADDR'],
user_agent=request.META['HTTP_USER_AGENT'],
server_time=unixtime(datetime.now()),
client_time=0, # TODO: fix. add to url params or switch to POST
detail='')
event.save()
return HttpResponse('OK')
def trimHITs():
global active_hits
new_active_hits = []
for hit in active_hits:
if unixtime(datetime.utcnow()) > hit.creation_time + 60 * 3:
try:
print 'removing old HIT: ' + hit.hit_id
if hit.sandbox:
mt_conn_sandbox.expire_hit(hit.hit_id)
else:
mt_conn_real.expire_hit(hit.hit_id)
hit.removed = True
hit.save()
except Exception, e:
print 'could not remove the HIT for some reason' + str(e)
else:
new_active_hits.append(hit)
def cron():
mt_conn = get_mt_conn()
cron_start = datetime.utcnow()
print cron_start.strftime("%m.%d.%Y %I:%M:%S %p") + ' --- retainer cron job ---'
protos = ProtoHit.objects.all()
for proto in protos:
reservations = unfulfilledReservationsForProto(proto)
for reservation in reservations:
numOnRetainer = len(reservation.workers)
print 'num on retainer: ' + str(numOnRetainer) + ' for ' + str(reservation)
if unixtime(datetime.now()) > (reservation.start_time + settings.RESERVATION_TIMEOUT_MINUTES * 60):
print 'expired, not posting more HITs for this reservation'
break
# postHITs for this reservation, if needed
if numOnRetainer < reservation.assignments:
postHITs(mt_conn, reservation)
def trimHITs():
global active_hits
new_active_hits = []
for hit in active_hits:
if unixtime(datetime.utcnow()) > hit.creation_time + 60 * 3:
try:
print 'removing old HIT: ' + hit.hit_id
if hit.sandbox:
mt_conn_sandbox.expire_hit(hit.hit_id)
else:
mt_conn_real.expire_hit(hit.hit_id)
hit.removed = True
hit.save()
except Exception, e:
print 'could not remove the HIT for some reason' + str(e)
else:
new_active_hits.append(hit)
def make(request):
# initialize a WorkReservation, for some time in the future
if not checkRequestParams(request, ['apiKey', 'hitType', 'foreignID', 'delay', 'numAssignments']):
return HttpResponse('Bad params')
params = request.POST
try:
proto = ProtoHit.objects.get(hit_type_id=params['hitType'])
foreign_id = int(params['foreignID'])
payload = params['payload'] if 'payload' in params else ''
resv = WorkReservation(proto = proto, \
foreign_id = foreign_id, \
payload = payload, \
start_time = unixtime(datetime.now() + timedelta(seconds = int(params['delay']))), \
assignments = int(params['numAssignments']), \
done = False, \
invoked = False)
resv.save()
return HttpResponse(resv.id)
except ProtoHit.DoesNotExist:
return HttpResponse('ProtoHit.DoesNotExist')
def clean_up_old_hits(conn, verbose=True, dry_run=False):
counts = {'cleaned': 0, 'remaining': 0, 'errors': 0}
hits = get_all_hits(conn)
now = timeutils.unixtime(datetime.now())
for h in hits:
try:
num_to_be_reviewed = len(
conn.get_assignments(h.HITId,
status="Submitted")) # yet to be reviewed
if verbose:
print "------------------"
print "HIT ID: " + h.HITId
expiration = timeutils.parseISO(h.Expiration)
duration = Decimal(h.AssignmentDurationInSeconds)
print "Expiration: " + str(expiration)
print "isExpired: " + str(h.expired)
print "AssignmentDuration: " + str(duration)
print "Seconds until last assignment can be turned in: " + str(
(float(expiration) + float(duration)) - float(now))
print "HITStatus: " + h.HITStatus
print "Yet-to-be-reviewed assignments: " + str(
num_to_be_reviewed)
print "Approved assignments: " + str(
len(conn.get_assignments(h.HITId, status="Approved")))
print "Rejected assignments: " + str(
len(conn.get_assignments(h.HITId, status="Rejected")))
if (h.HITStatus == 'Reviewable'
or h.HITStatus == 'Reviewing') and num_to_be_reviewed == 0:
if not dry_run:
conn.dispose_hit(h.HITId)
print "DISPOSED OF HIT"
counts['cleaned'] += 1
if dry_run:
print "(dry run -- HIT not actually disposed)"
else:
counts['remaining'] += 1
except Exception, e:
print "exception processing HIT:\n" + str(e)
counts['errors'] += 1
def make(request):
# initialize a WorkReservation, for some time in the future
if not checkRequestParams(
request,
['apiKey', 'hitType', 'foreignID', 'delay', 'numAssignments']):
return HttpResponse('Bad params')
params = request.POST
try:
proto = ProtoHit.objects.get(hit_type_id=params['hitType'])
foreign_id = int(params['foreignID'])
payload = params['payload'] if 'payload' in params else ''
resv = WorkReservation(proto = proto, \
foreign_id = foreign_id, \
payload = payload, \
start_time = unixtime(datetime.now() + timedelta(seconds = int(params['delay']))), \
assignments = int(params['numAssignments']), \
done = False, \
invoked = False)
resv.save()
return HttpResponse(resv.id)
except ProtoHit.DoesNotExist:
return HttpResponse('ProtoHit.DoesNotExist')
