def add_uri_visit(self, request, uri):
# only create objects and score for desired URLs
if not is_ignored(uri):
# lock down the table
with ClassLock('checking for uri'):
hitcount, created = self.select_for_update().get_or_create(
uri=uri)
# If this request comes from a cache,
# it may include a custom field: Obj-Cache-Hits.
# This field stores a temporary hit count (as string),
# which should be added to the running total.
cache_hits_str = request.META.get('HTTP_OBJ_CACHE_HITS', '0')
try:
cache_hits = int(cache_hits_str)
except (TypeError, ValueError):
cache_hits = 0
hitcount_increment = cache_hits + 1
with TransactionLock(hitcount, 'updating count'):
# Update hitcount object for this URI.
hitcount.last_hit = now()
hitcount.hits += hitcount_increment
hitcount.save()
def create(cls, table, criteria, update_progress=True, parent=None):
# Adjust the criteria for this specific table, locking
# down start/end times as needed
criteria = criteria.build_for_table(table)
try:
criteria.compute_times()
except ValueError:
# Ignore errors, this table may not have start/end times
pass
# Compute the handle -- this will take into account
# cacheability
handle = Job._compute_handle(table, criteria)
# Grab a lock on the row associated with the table
with TransactionLock(table, "Job.create"):
# Look for another job by the same handle in any state except ERROR
master = Job.objects.get_master(handle)
job = Job(table=table,
criteria=criteria,
actual_criteria=None,
status=Job.NEW,
pid=os.getpid(),
handle=handle,
parent=parent,
master=master,
update_progress=update_progress,
message='',
exception='')
job.save()
if master:
master.reference("Master link from job %s" % job)
now = datetime.datetime.now(tz=pytz.utc)
master.safe_update(touched=now)
logger.info("%s: New job for table %s, linked to master %s" %
(job, table.name, master))
else:
logger.info("%s: New job for table %s" % (job, table.name))
# Create new instance in progressd as part of same Transaction
p = {
'job_id': job.id,
'status': job.status,
'progress': 0,
'master_id': job.master.id if job.master else 0,
'parent_id': job.parent.id if job.parent else 0
}
logger.debug('***Creating Job to progressd: %s' % p)
progressd.post(**p)
# End of TransactionLock
logger.debug("%s: criteria = %s" % (job, criteria))
return job
def check_children(self, objlock=None):
# get a lock on the child that's called us to ensure any status
# from its transaction will be seen.
if objlock is None:
objlock = self
with TransactionLock(objlock, '%s.checking_children' % self):
running_children = Job.objects.filter(
parent=self, status__in=[Job.NEW, Job.RUNNING])
logger.info("%s: %d running children" % (self, len(running_children)))
logger.debug(
"%s: all children: %s" %
(self, ';'.join('%s - %s' % (j.status, j)
for j in Job.objects.filter(parent=self))))
if len(running_children) > 0:
# Not done yet, do nothing
return
# Grab a lock on this job to make sure only one caller
# gets the callback
with TransactionLock(self, '%s.check_children' % self):
# Now that we have the lock, make sure we have latest Job
# details
self.refresh()
logger.info("%s: checking callback %s" % (self, self.callback))
if self.callback is None:
# Some other child got to it first
return
# Save off the callback, we'll call it outside the transaction
callback = self.callback
# Clear the callback while still in lockdown
self.callback = None
self.save()
t = Task(self, callback=callback)
logger.info("%s: Created callback task %s" % (self, t))
t.start()
def safe_update(self, **kwargs):
""" Update the job with the passed dictionary in a database safe way.
This method updates only the requested paraemters and refreshes
the rest from the database. This should be used for all updates
to Job's to ensure that unmodified keys are not accidentally
clobbered by doing a blanket job.save().
"""
logger.debug("%s safe_update %s" % (self, kwargs))
with TransactionLock(self, '%s.safe_update' % str(self)):
Job.objects.filter(pk=self.pk).update(**kwargs)
self.refresh()
def mark_done(self, status, **kwargs):
with TransactionLock(self, '%s.mark_done' % self):
self.refresh()
old_status = self.status
if old_status in (Job.COMPLETE, Job.ERROR):
# Status was already set to a done state, avoid
# double action and return now
return
self.status = status
for k, v in kwargs.iteritems():
setattr(self, k, v)
self.save()
# On status change, do more...
self.mark_progress(status=status, progress=100)
if not self.is_follower:
# Notify followers of this job
followers = Job.objects.filter(master=self)
for follower in followers:
if self.status == Job.COMPLETE:
kwargs['actual_criteria'] = self.actual_criteria
follower.mark_complete(status=status, **kwargs)
elif self.status == Job.ERROR:
follower.mark_done(status=status, **kwargs)
if self.parent:
logger.info("%s: Asking parent %s to check children" %
(self, self.parent))
t = Task(self.parent,
callback=Callable(self.parent.check_children,
called_kwargs={'objlock': self}),
generic=True)
logger.info("%s: Created check_children task %s" % (self, t))
t.start()
return True
def start(self, method=None, method_args=None):
""" Start this job. """
with TransactionLock(self.table, '%s.start' % self):
logger.info("%s: Job starting" % self)
self.refresh()
if self.is_follower:
logger.debug("%s: Shadowing master job %s" %
(self, self.master))
if self.master.status == Job.COMPLETE:
self.mark_complete()
elif self.master.status == Job.ERROR:
self.mark_error(self.master.message, self.master.exception)
return
if method is None:
method = self.table.queryclass.run
# Create an task to do the work
task = Task(self, Callable(method, method_args))
logger.debug("%s: Created task %s" % (self, task))
task.start()
def dereference(self, message=""):
with TransactionLock(self, '%s.dereference' % self):
pk = self.pk
Job.objects.filter(pk=pk).update(refcount=F('refcount') - 1)