def extract_from_lead(lead_id): """ A task to auto extract text and images from a lead. * Lead should have a valid attachment or url. * It needs to be checked whether this task from the same lead is already going on. * On extraction complete, subscribed users through websocket need to be notified. """ # Use redis store to keep track of leads currently being extracted # and try to prevent useless parallel extraction of same lead that # that might happen. r = redis.get_connection() key = 'lead_extraction_{}'.format(lead_id) lock = 'lock_{}'.format(key) # Check if key exists and if so return, otherwise set the key ourself # Also use lock while doing this. with redis.get_lock(lock): if r.exists(key): logger.error('Lead Redis Locked') return False r.set(key, '1') try: # Actual extraction process return_value = _extract_from_lead_core(lead_id) # Send signal to all pending websocket clients # that the lead extraction has completed. code = SubscriptionConsumer.encode({ 'channel': 'leads', 'event': 'onPreviewExtracted', 'leadId': lead_id, }) # TODO: Discuss and decide the notification response format # Also TODO: Should a handler be added during subscription # to immediately reply with already extracted lead? Group(code).send( json.loads(JSONRenderer().render({ 'code': code, 'timestamp': timezone.now(), 'type': 'notification', 'status': return_value, }).decode('utf-8'))) except Exception as e: logger.error(traceback.format_exc()) return_value = False # Once done, we delete the key to say that this task is done. r.delete(key) return return_value
def extract_from_lead(lead_id): """ A task to auto extract text and images from a lead. * Lead should have a valid attachment or url. * It needs to be checked whether this task from the same lead is already going on. * On extraction complete, subscribed users through websocket need to be notified. """ # Use redis lock to keep track of leads currently being extracted # and try to prevent useless parallel extraction of same lead that # that might happen. key = 'lead_extraction_{}'.format(lead_id) lock = redis.get_lock(key, 60 * 60 * 4) # Lock lifetime 4 hours have_lock = lock.acquire(blocking=False) if not have_lock: return False try: # Actual extraction process return_value = _extract_from_lead_core(lead_id) # Send signal to all pending websocket clients # that the lead extraction has completed. code = SubscriptionConsumer.encode({ 'channel': 'leads', 'event': 'onPreviewExtracted', 'leadId': lead_id, }) # TODO: Discuss and decide the notification response format # Also TODO: Should a handler be added during subscription # to immediately reply with already extracted lead? Group(code).send( json.loads(JSONRenderer().render({ 'code': code, 'timestamp': timezone.now(), 'type': 'notification', 'status': return_value, }).decode('utf-8'))) except Exception: logger.error(traceback.format_exc()) return_value = False lock.release() return return_value
def extract_from_file(file_preview_id): key = 'file_extraction_{}'.format(file_preview_id) lock = redis.get_lock(key, 60 * 60 * 24) # Lock lifetime 24 hours have_lock = lock.acquire(blocking=False) if not have_lock: return False try: return_value = _extract_from_file_core(file_preview_id) except Exception: logger.error('gallery.extract_from_file', exc_info=True) return_value = False lock.release() return return_value
def generate_project_stats_cache(force=False): """ Generate stats data for Home """ key = STATS_WAIT_LOCK_KEY lock = redis.get_lock(key, STATS_WAIT_TIMEOUT) have_lock = lock.acquire(blocking=False) if not have_lock and not force: logger.warning(f'GENERATE_PROJECT_STATS:: Waiting for timeout {key}') return False logger.info(f'GENERATE_PROJECT_STATS:: Processing for {key}') _generate_project_stats_cache() lock.release() return True
def load_geo_areas(region_id): key = 'load_geo_areas_{}'.format(region_id) lock = redis.get_lock(key, 60 * 30) # Lock lifetime 30 minutes have_lock = lock.acquire(blocking=False) if not have_lock: return False try: return_value = _load_geo_areas(region_id) except Exception: logger.error(traceback.format_exc()) return_value = False lock.release() return return_value
def remaining_tabular_generate_columns_image(): """ Scheduled task NOTE: Only use it through schedular """ key = 'remaining_tabular_generate_columns_image' lock = redis.get_lock(key, 60 * 60 * 2) # Lock lifetime 2 hours have_lock = lock.acquire(blocking=False) if not have_lock: return '{} Locked'.format(key) tabular_generate_columns_image( Field.objects.filter(cache__status=Field.CACHE_PENDING, ).distinct().order_by('id').values_list( 'id', flat=True)[:300]) lock.release() return True
def generate_viz_stats(project_id, force=False): """ Generate stats data for Vizualization (Entry/Ary Viz) """ key = VIZ_STATS_WAIT_LOCK_KEY.format(project_id) lock = redis.get_lock(key, STATS_WAIT_TIMEOUT) have_lock = lock.acquire(blocking=False) if not have_lock and not force: logger.warning( f'GENERATE_PROJECT_VIZ_STATS:: Waiting for timeout {key}') return False logger.info(f'GENERATE_PROJECT_STATS:: Processing for {key}') _generate_project_viz_stats(project_id) # NOTE: lock.release() is not called so that another process waits for timeout return True
def _caller(*args, **kwargs): key = lock_key.format(*args, **kwargs) lock = redis.get_lock(key, timeout) have_lock = lock.acquire(blocking=False) if not have_lock: logger.warning( f'Unable to get lock for {key}(ttl: {get_redis_lock_ttl(lock)})' ) return False try: return_value = func(*args, **kwargs) or True except Exception: logger.error('{}.{}'.format(func.__module__, func.__name__), exc_info=True) return_value = False lock.release() return return_value
def extract_from_file(file_preview_id): r = redis.get_connection() key = 'file_extraction_{}'.format(file_preview_id) lock = 'lock_{}'.format(key) with redis.get_lock(lock): if r.exists(key): return False r.set(key, '1') try: return_value = _extract_from_file_core(file_preview_id) except Exception as e: logger.error(traceback.format_exc()) return_value = False r.delete(key) return return_value
def load_geo_areas(region_id): r = redis.get_connection() key = 'load_geo_areas_{}'.format(region_id) lock = 'lock_{}'.format(key) with redis.get_lock(lock): if r.exists(key): logger.error('Geo Area Redis Locked') return False r.set(key, '1') try: return_value = _load_geo_areas(region_id) except Exception as e: logger.error(traceback.format_exc()) return_value = False r.delete(key) return return_value
def tabular_extract_geo(geodata_pk): key = 'tabular_meta_extract_geo_{}'.format(geodata_pk) lock = redis.get_lock(key, 60 * 60 * 24) # Lock lifetime 24 hours have_lock = lock.acquire(blocking=False) if not have_lock: return False geodata = Geodata.objects.get(pk=geodata_pk) try: with transaction.atomic(): return_value = _tabular_meta_extract_geo(geodata) geodata.status = Geodata.SUCCESS except Exception: logger.error('Tabular Extract Geo Failed!!', exc_info=True) geodata.status = Geodata.FAILED return_value = False geodata.save() lock.release() return return_value
def tabular_extract_book(book_pk): key = 'tabular_extract_book_{}'.format(book_pk) lock = redis.get_lock(key, 60 * 60 * 24) # Lock lifetime 24 hours have_lock = lock.acquire(blocking=False) if not have_lock: return False book = Book.objects.get(pk=book_pk) try: with transaction.atomic(): return_value = _tabular_extract_book(book) book.status = Book.SUCCESS except Exception: logger.error('Tabular Extract Book Failed!!', exc_info=True) book.status = Book.FAILED book.error = Book.UNKNOWN_ERROR # TODO: handle all type of error return_value = False book.save() lock.release() return return_value