def run(self, *args, **kwargs):
kw = {
'tagasaurisjobs__%s' % self.hit_name: None,
'tagasaurisjobs__%s__isnull' % self.key_name: False,
}
jobs = Job.objects.select_related('tagasaurisjobs').filter(
status=JOB_STATUS_ACTIVE, tagasaurisjobs__isnull=False,
**kw).iterator()
client = make_tagapi_client()
for job in jobs:
old_hit = getattr(job.tagasaurisjobs, self.hit_name)
new_hit = get_hit(client,
getattr(job.tagasaurisjobs, self.key_name),
job.get_hit_type())
if not new_hit or old_hit == new_hit:
continue
kw = {self.hit_name: new_hit}
TagasaurisJobs.objects.filter(urlannotator_job=job).update(**kw)
send_event(
self.event_name,
job_id=job.id,
old_hit=old_hit,
new_hit=new_hit,
)
def run(self, *args, **kwargs):
""" Main task function.
"""
try:
unmapped_samples = self.get_unmapped_samples()
jobs = self.get_jobs(unmapped_samples)
tc = make_tagapi_client()
log.info('SampleVotingManager: Gathered samples and jobs.')
for job, new_samples, initialized in jobs:
try:
handler = get_job_handler(job)
if initialized:
handler.update_voting(tc=tc, samples=new_samples)
else:
handler.init_voting(tc=tc, samples=new_samples)
except Exception, e:
log.exception(
'SampleVotingManager: Error in job %d: %s.' % (job.id, e)
)
continue
except Exception, e:
log.exception(
'SampleVotingManager: exception while all jobs: %s.' % e
)
def run(self, *args, **kwargs):
kw = {
'tagasaurisjobs__%s' % self.hit_name: None,
'tagasaurisjobs__%s__isnull' % self.key_name: False,
}
jobs = Job.objects.select_related('tagasaurisjobs').filter(
status=JOB_STATUS_ACTIVE,
tagasaurisjobs__isnull=False, **kw).iterator()
client = make_tagapi_client()
for job in jobs:
old_hit = getattr(job.tagasaurisjobs, self.hit_name)
new_hit = get_hit(client,
getattr(job.tagasaurisjobs, self.key_name), job.get_hit_type())
if not new_hit or old_hit == new_hit:
continue
kw = {self.hit_name: new_hit}
TagasaurisJobs.objects.filter(urlannotator_job=job).update(
**kw
)
send_event(
self.event_name,
job_id=job.id,
old_hit=old_hit,
new_hit=new_hit,
)
def init_btm_voting(self, samples, *args, **kwargs):
lock_name = 'job_handler_init_btm_vot_%d' % self.job.id
with POSIXLock(lock_name):
if self.job.tagasaurisjobs.voting_btm_hit is not None:
log.info(
'Tried to create new btm voting job, but it already exists'
)
return True
tc = make_tagapi_client()
create_btm_voting_job(tc, self.job, samples)
def init_btm_voting(self, samples, *args, **kwargs):
lock_name = 'job_handler_init_btm_vot_%d' % self.job.id
with POSIXLock(lock_name):
if self.job.tagasaurisjobs.voting_btm_hit is not None:
log.info(
'Tried to create new btm voting job, but it already exists'
)
return True
tc = make_tagapi_client()
create_btm_voting_job(tc, self.job, samples)
def update_btm(self, btm_samples, *args, **kwargs):
lock_name = 'job_handler_upd_btm_%d' % self.job.id
with POSIXLock(lock_name):
tc = make_tagapi_client()
samples = (btm.sample for btm in btm_samples)
tag_jobs = self.job.tagasaurisjobs
# Job has no BTM key - create a new job
if not tag_jobs.voting_btm_key:
# Creates sample to mediaobject mapping
self.init_btm_voting(samples)
else:
update_btm(tc, self.job, samples)
def init_btm_gather(self, topic, description, no_of_urls, *args, **kwargs):
lock_name = 'job_handler_init_btm_%d' % self.job.id
with POSIXLock(lock_name):
if self.job.tagasaurisjobs.beatthemachine_hit is not None:
log.info(
'Tried to create new btm gathering job, but it already exists'
)
return True
tc = make_tagapi_client()
key, hit = create_btm(tc, self.job, topic, description, no_of_urls)
self.job.tagasaurisjobs.beatthemachine_key = key
self.job.tagasaurisjobs.save()
return key is not None
def update_btm(self, btm_samples, *args, **kwargs):
lock_name = 'job_handler_upd_btm_%d' % self.job.id
with POSIXLock(lock_name):
tc = make_tagapi_client()
samples = (btm.sample for btm in btm_samples)
tag_jobs = self.job.tagasaurisjobs
# Job has no BTM key - create a new job
if not tag_jobs.voting_btm_key:
# Creates sample to mediaobject mapping
self.init_btm_voting(samples)
else:
update_btm(tc, self.job, samples)
def init_btm_gather(self, topic, description, no_of_urls, *args, **kwargs):
lock_name = 'job_handler_init_btm_%d' % self.job.id
with POSIXLock(lock_name):
if self.job.tagasaurisjobs.beatthemachine_hit is not None:
log.info(
'Tried to create new btm gathering job, but it already exists'
)
return True
tc = make_tagapi_client()
key, hit = create_btm(tc, self.job, topic, description, no_of_urls)
self.job.tagasaurisjobs.beatthemachine_key = key
self.job.tagasaurisjobs.save()
return key is not None
def setUp(self):
self.u = User.objects.create_user(username='******', password='******')
self.job = Job.objects.create_active(
title='urlannotator_test_tagapi_client',
description='test_description',
no_of_urls=2,
account=self.u.get_profile(),
gold_samples=[{'url': '10clouds.com', 'label': LABEL_YES}])
self.sample = Sample.objects.all()[0]
self.mock = mock.patch(
'urlannotator.crowdsourcing.tagasauris_helper.BACKOFF_GENERATOR',
backoff
)
self.mock.start()
self.tc = make_tagapi_client()
def setUp(self):
self.u = User.objects.create_user(username='******', password='******')
self.job = Job.objects.create_active(
title='urlannotator_test_tagapi_client',
description='test_description',
no_of_urls=2,
account=self.u.get_profile(),
gold_samples=[{
'url': '10clouds.com',
'label': LABEL_YES
}])
self.sample = Sample.objects.all()[0]
self.mock = mock.patch(
'urlannotator.crowdsourcing.tagasauris_helper.BACKOFF_GENERATOR',
backoff)
self.mock.start()
self.tc = make_tagapi_client()
def run(self, *args, **kwargs):
""" Main task function.
"""
try:
unmapped_samples = self.get_unmapped_samples()
jobs = self.get_jobs(unmapped_samples)
tc = make_tagapi_client()
log.info('SampleVotingManager: Gathered samples and jobs.')
for job, new_samples, initialized in jobs:
try:
handler = get_job_handler(job)
if initialized:
handler.update_voting(tc=tc, samples=new_samples)
else:
handler.init_voting(tc=tc, samples=new_samples)
except Exception, e:
log.exception('SampleVotingManager: Error in job %d: %s.' %
(job.id, e))
continue
except Exception, e:
log.exception(
'SampleVotingManager: exception while all jobs: %s.' % e)
def _pay_tagasauris_bonus(self):
tc = make_tagapi_client()
tc.pay_worker_bonus(
worker_id=int(self.worker.external_id),
amount=float(self.amount),
reason='Bonus for Beat The Machine job no %s' % self.job.id)
def _pay_tagasauris_bonus(self):
tc = make_tagapi_client()
tc.pay_worker_bonus(worker_id=int(self.worker.external_id),
amount=float(self.amount),
reason='Bonus for Beat The Machine job no %s' %
self.job.id)
