def extract_from_lead(lead_id):
"""
A task to auto extract text and images from a lead.
* Lead should have a valid attachment or url.
* It needs to be checked whether this task from the same lead
is already going on.
* On extraction complete, subscribed users through websocket
need to be notified.
"""
# Use redis store to keep track of leads currently being extracted
# and try to prevent useless parallel extraction of same lead that
# that might happen.
r = redis.get_connection()
key = 'lead_extraction_{}'.format(lead_id)
lock = 'lock_{}'.format(key)
# Check if key exists and if so return, otherwise set the key ourself
# Also use lock while doing this.
with redis.get_lock(lock):
if r.exists(key):
logger.error('Lead Redis Locked')
return False
r.set(key, '1')
try:
# Actual extraction process
return_value = _extract_from_lead_core(lead_id)
# Send signal to all pending websocket clients
# that the lead extraction has completed.
code = SubscriptionConsumer.encode({
'channel': 'leads',
'event': 'onPreviewExtracted',
'leadId': lead_id,
})
# TODO: Discuss and decide the notification response format
# Also TODO: Should a handler be added during subscription
# to immediately reply with already extracted lead?
Group(code).send(
json.loads(JSONRenderer().render({
'code': code,
'timestamp': timezone.now(),
'type': 'notification',
'status': return_value,
}).decode('utf-8')))
except Exception as e:
logger.error(traceback.format_exc())
return_value = False
# Once done, we delete the key to say that this task is done.
r.delete(key)
return return_value
def extract_from_lead(lead_id):
"""
A task to auto extract text and images from a lead.
* Lead should have a valid attachment or url.
* It needs to be checked whether this task from the same lead
is already going on.
* On extraction complete, subscribed users through websocket
need to be notified.
"""
# Use redis lock to keep track of leads currently being extracted
# and try to prevent useless parallel extraction of same lead that
# that might happen.
key = 'lead_extraction_{}'.format(lead_id)
lock = redis.get_lock(key, 60 * 60 * 4) # Lock lifetime 4 hours
have_lock = lock.acquire(blocking=False)
if not have_lock:
return False
try:
# Actual extraction process
return_value = _extract_from_lead_core(lead_id)
# Send signal to all pending websocket clients
# that the lead extraction has completed.
code = SubscriptionConsumer.encode({
'channel': 'leads',
'event': 'onPreviewExtracted',
'leadId': lead_id,
})
# TODO: Discuss and decide the notification response format
# Also TODO: Should a handler be added during subscription
# to immediately reply with already extracted lead?
Group(code).send(
json.loads(JSONRenderer().render({
'code': code,
'timestamp': timezone.now(),
'type': 'notification',
'status': return_value,
}).decode('utf-8')))
except Exception:
logger.error(traceback.format_exc())
return_value = False
lock.release()
return return_value
def extract_from_file(file_preview_id):
key = 'file_extraction_{}'.format(file_preview_id)
lock = redis.get_lock(key, 60 * 60 * 24) # Lock lifetime 24 hours
have_lock = lock.acquire(blocking=False)
if not have_lock:
return False
try:
return_value = _extract_from_file_core(file_preview_id)
except Exception:
logger.error('gallery.extract_from_file', exc_info=True)
return_value = False
lock.release()
return return_value
def generate_project_stats_cache(force=False):
"""
Generate stats data for Home
"""
key = STATS_WAIT_LOCK_KEY
lock = redis.get_lock(key, STATS_WAIT_TIMEOUT)
have_lock = lock.acquire(blocking=False)
if not have_lock and not force:
logger.warning(f'GENERATE_PROJECT_STATS:: Waiting for timeout {key}')
return False
logger.info(f'GENERATE_PROJECT_STATS:: Processing for {key}')
_generate_project_stats_cache()
lock.release()
return True
def load_geo_areas(region_id):
key = 'load_geo_areas_{}'.format(region_id)
lock = redis.get_lock(key, 60 * 30) # Lock lifetime 30 minutes
have_lock = lock.acquire(blocking=False)
if not have_lock:
return False
try:
return_value = _load_geo_areas(region_id)
except Exception:
logger.error(traceback.format_exc())
return_value = False
lock.release()
return return_value
def remaining_tabular_generate_columns_image():
"""
Scheduled task
NOTE: Only use it through schedular
"""
key = 'remaining_tabular_generate_columns_image'
lock = redis.get_lock(key, 60 * 60 * 2) # Lock lifetime 2 hours
have_lock = lock.acquire(blocking=False)
if not have_lock:
return '{} Locked'.format(key)
tabular_generate_columns_image(
Field.objects.filter(cache__status=Field.CACHE_PENDING,
).distinct().order_by('id').values_list(
'id', flat=True)[:300])
lock.release()
return True
def generate_viz_stats(project_id, force=False):
"""
Generate stats data for Vizualization (Entry/Ary Viz)
"""
key = VIZ_STATS_WAIT_LOCK_KEY.format(project_id)
lock = redis.get_lock(key, STATS_WAIT_TIMEOUT)
have_lock = lock.acquire(blocking=False)
if not have_lock and not force:
logger.warning(
f'GENERATE_PROJECT_VIZ_STATS:: Waiting for timeout {key}')
return False
logger.info(f'GENERATE_PROJECT_STATS:: Processing for {key}')
_generate_project_viz_stats(project_id)
# NOTE: lock.release() is not called so that another process waits for timeout
return True
def _caller(*args, **kwargs):
key = lock_key.format(*args, **kwargs)
lock = redis.get_lock(key, timeout)
have_lock = lock.acquire(blocking=False)
if not have_lock:
logger.warning(
f'Unable to get lock for {key}(ttl: {get_redis_lock_ttl(lock)})'
)
return False
try:
return_value = func(*args, **kwargs) or True
except Exception:
logger.error('{}.{}'.format(func.__module__, func.__name__),
exc_info=True)
return_value = False
lock.release()
return return_value
def extract_from_file(file_preview_id):
r = redis.get_connection()
key = 'file_extraction_{}'.format(file_preview_id)
lock = 'lock_{}'.format(key)
with redis.get_lock(lock):
if r.exists(key):
return False
r.set(key, '1')
try:
return_value = _extract_from_file_core(file_preview_id)
except Exception as e:
logger.error(traceback.format_exc())
return_value = False
r.delete(key)
return return_value
def load_geo_areas(region_id):
r = redis.get_connection()
key = 'load_geo_areas_{}'.format(region_id)
lock = 'lock_{}'.format(key)
with redis.get_lock(lock):
if r.exists(key):
logger.error('Geo Area Redis Locked')
return False
r.set(key, '1')
try:
return_value = _load_geo_areas(region_id)
except Exception as e:
logger.error(traceback.format_exc())
return_value = False
r.delete(key)
return return_value
def tabular_extract_geo(geodata_pk):
key = 'tabular_meta_extract_geo_{}'.format(geodata_pk)
lock = redis.get_lock(key, 60 * 60 * 24) # Lock lifetime 24 hours
have_lock = lock.acquire(blocking=False)
if not have_lock:
return False
geodata = Geodata.objects.get(pk=geodata_pk)
try:
with transaction.atomic():
return_value = _tabular_meta_extract_geo(geodata)
geodata.status = Geodata.SUCCESS
except Exception:
logger.error('Tabular Extract Geo Failed!!', exc_info=True)
geodata.status = Geodata.FAILED
return_value = False
geodata.save()
lock.release()
return return_value
def tabular_extract_book(book_pk):
key = 'tabular_extract_book_{}'.format(book_pk)
lock = redis.get_lock(key, 60 * 60 * 24) # Lock lifetime 24 hours
have_lock = lock.acquire(blocking=False)
if not have_lock:
return False
book = Book.objects.get(pk=book_pk)
try:
with transaction.atomic():
return_value = _tabular_extract_book(book)
book.status = Book.SUCCESS
except Exception:
logger.error('Tabular Extract Book Failed!!', exc_info=True)
book.status = Book.FAILED
book.error = Book.UNKNOWN_ERROR # TODO: handle all type of error
return_value = False
book.save()
lock.release()
return return_value