def task_check_interval_source(source_id, test=False):
Source = apps.get_model(app_label='sources', model_name='Source')
lock_id = 'task_check_interval_source-%d' % source_id
try:
logger.debug('trying to acquire lock: %s', lock_id)
lock = locking_backend.acquire_lock(lock_id,
DEFAULT_SOURCE_LOCK_EXPIRE)
except LockError:
logger.debug('unable to obtain lock: %s' % lock_id)
else:
logger.debug('acquired lock: %s', lock_id)
try:
source = Source.objects.get_subclass(pk=source_id)
if source.enabled or test:
source.check_source(test=test)
except Exception as exception:
logger.error('Error processing source: %s; %s', source, exception)
source.logs.create(message=_('Error processing source: %s') %
exception)
else:
source.logs.all().delete()
finally:
lock.release()
def create_file(self, filename):
lock_id = 'cache_partition-create_file-{}-{}'.format(self.pk, filename)
try:
logger.debug('trying to acquire lock: %s', lock_id)
lock = locking_backend.acquire_lock(lock_id)
logger.debug('acquired lock: %s', lock_id)
try:
self.cache.prune()
# Since open "wb+" doesn't create files force the creation of an
# empty file.
self.cache.storage.delete(name=self.get_full_filename(
filename=filename))
self.cache.storage.save(
name=self.get_full_filename(filename=filename),
content=ContentFile(content=''))
try:
with transaction.atomic():
partition_file = self.files.create(filename=filename)
yield partition_file.open(mode='wb')
partition_file.update_size()
except Exception as exception:
logger.error(
'Unexpected exception while trying to save new '
'cache file; %s', exception)
self.cache.storage.delete(name=self.get_full_filename(
filename=filename))
raise
finally:
lock.release()
except LockError:
logger.debug('unable to obtain lock: %s' % lock_id)
raise
def task_do_ocr(self, document_version_pk):
DocumentVersion = apps.get_model(app_label='documents',
model_name='DocumentVersion')
DocumentPageOCRContent = apps.get_model(
app_label='ocr', model_name='DocumentPageOCRContent')
lock_id = 'task_do_ocr_doc_version-%d' % document_version_pk
try:
logger.debug('trying to acquire lock: %s', lock_id)
# Acquire lock to avoid doing OCR on the same document version more
# than once concurrently
lock = locking_backend.acquire_lock(name=lock_id, timeout=LOCK_EXPIRE)
logger.debug('acquired lock: %s', lock_id)
document_version = None
try:
document_version = DocumentVersion.objects.get(
pk=document_version_pk)
logger.info('Starting document OCR for document version: %s',
document_version)
DocumentPageOCRContent.objects.process_document_version(
document_version=document_version)
except OperationalError as exception:
logger.warning('OCR error for document version: %d; %s. Retrying.',
document_version_pk, exception)
raise self.retry(exc=exception)
finally:
lock.release()
except LockError:
logger.debug('unable to obtain lock: %s' % lock_id)
def task_check_expired_check_outs():
DocumentCheckout = apps.get_model(app_label='checkouts',
model_name='DocumentCheckout')
logger.debug(msg='executing...')
lock_id = 'task_expired_check_outs'
try:
logger.debug('trying to acquire lock: %s', lock_id)
lock = locking_backend.acquire_lock(
name=lock_id, timeout=CHECKOUT_EXPIRATION_LOCK_EXPIRE)
logger.debug('acquired lock: %s', lock_id)
DocumentCheckout.objects.check_in_expired_check_outs()
lock.release()
except LockError:
logger.debug(msg='unable to obtain lock')
def remove_document(self, document):
"""
The argument `acquire_lock` controls whether or not this method
acquires or lock. The case for this is to acquire when called directly
or not to acquire when called as part of a larger index process
that already has a lock
"""
# Prevent another process to work on this node. We use the node's
# parent template node for the lock
try:
lock = locking_backend.acquire_lock(
self.index_template_node.get_lock_string())
except LockError:
raise
else:
try:
self.documents.remove(document)
finally:
lock.release()
def task_process_document_version(document_version_id):
DocumentVersion = apps.get_model(app_label='documents',
model_name='DocumentVersion')
document_version = DocumentVersion.objects.get(pk=document_version_id)
lock_id = 'task_process_document_version-%d' % document_version_id
try:
logger.debug('trying to acquire lock: %s', lock_id)
# Acquire lock to avoid processing the same document version more
# than once concurrently
lock = locking_backend.acquire_lock(name=lock_id, timeout=LOCK_EXPIRE)
logger.debug('acquired lock: %s', lock_id)
except LockError:
logger.debug('unable to obtain lock: %s' % lock_id)
else:
FileMetadataDriver.process_document_version(
document_version=document_version)
lock.release()
def delete_empty(self):
"""
Method to delete all empty node instances in a recursive manner.
"""
# Prevent another process to delete this node.
try:
lock = locking_backend.acquire_lock(
self.index_template_node.get_lock_string())
except LockError:
raise
else:
try:
if self.documents.count() == 0 and self.get_children().count(
) == 0:
if not self.is_root_node():
# I'm not a root node, I can be deleted
self.delete()
if self.parent.is_root_node():
# My parent is not a root node, it can be deleted
self.parent.delete_empty()
finally:
lock.release()
def index_document(self,
document,
acquire_lock=True,
index_instance_node_parent=None):
# Start transaction after the lock in case the locking backend uses
# the database.
try:
if acquire_lock:
lock = locking_backend.acquire_lock(self.get_lock_string())
except LockError:
raise
else:
try:
logger.debug('IndexTemplateNode; Indexing document: %s',
document)
if not index_instance_node_parent:
# I'm the root
with transaction.atomic():
index_instance_root_node = self.get_instance_root_node(
)
for child in self.get_children():
child.index_document(document=document,
acquire_lock=False,
index_instance_node_parent=
index_instance_root_node)
elif self.enabled:
with transaction.atomic():
logger.debug(
'IndexTemplateNode; non parent: evaluating')
logger.debug('My parent template is: %s', self.parent)
logger.debug('My parent instance node is: %s',
index_instance_node_parent)
logger.debug(
'IndexTemplateNode; Evaluating template: %s',
self.expression)
try:
context = Context({'document': document})
template = Template(self.expression)
result = template.render(context=context)
except Exception as exception:
logger.debug('Evaluating error: %s', exception)
error_message = _(
'Error indexing document: %(document)s; expression: '
'%(expression)s; %(exception)s') % {
'document': document,
'expression': self.expression,
'exception': exception
}
logger.debug(error_message)
else:
logger.debug('Evaluation result: %s', result)
if result:
index_instance_node, created = self.index_instance_nodes.get_or_create(
parent=index_instance_node_parent,
value=result)
if self.link_documents:
index_instance_node.documents.add(document)
for child in self.get_children():
child.index_document(
document=document,
acquire_lock=False,
index_instance_node_parent=
index_instance_node)
finally:
if acquire_lock:
lock.release()
