def calc_block_similarity(task,
df1_redis_key,
df2_redis_key,
search_similar_documents,
engine_kwargs,
project_id: Optional[int]):
task.log_info(
f'Calculate similarity for feature_df blocks with keys: '
f'"{df1_redis_key}" and "{df2_redis_key}"')
engine_class = DocumentSimilarityEngine if search_similar_documents else TextUnitSimilarityEngine
df1_data, df1_index, df1_columns = redis.pop(df1_redis_key)
df2_data, df2_index, df2_columns = redis.pop(df2_redis_key)
df1 = pd.DataFrame(df1_data, index=df1_index, columns=df1_columns)
df2 = pd.DataFrame(df2_data, index=df2_index, columns=df2_columns)
engine_kwargs['log_routine'] = lambda m: task.log_info(m)
sim_obj = engine_class(**engine_kwargs)
records = sim_obj.calc_block_similarity(df1, df2)
if not search_similar_documents:
SimilarityObjectReference.ensure_unit_similarity_model_refs(records, project_id)
sim_obj.similarity_model.objects.bulk_create(records)
count = len(records)
task.log_info(f'Created {count} records')
def calc_block_similarity(task, df1_redis_key, df2_redis_key,
search_similar_documents, engine_kwargs):
task.log_info(f'Calculate similarity for feature_df blocks with keys: '
f'"{df1_redis_key}" and "{df2_redis_key}"')
engine_class = DocumentSimilarityEngine if search_similar_documents else TextUnitSimilarityEngine
df1 = redis.pop(df1_redis_key)
df2 = redis.pop(df2_redis_key)
engine_class(**engine_kwargs).calc_block_similarity(df1, df2)
def get_cached(cls):
cached_key = redis.pop(f'{cls.CACHE_KEY}_hash', unpickle_value=False)
if cls.CACHED_DATA:
if cached_key == cls.LAST_CACHED_HASH:
return cls.CACHED_DATA
cls.CACHED_DATA = redis.pop(f'{cls.CACHE_KEY}_data', unpickle_value=True)
if cls.CACHED_DATA is None:
return cls.rewrite_cache()
cls.LAST_CACHED_HASH = cached_key
return cls.CACHED_DATA
def try_delete(self, user):
token_user_key = self.get_token_user_key(user.id)
try:
token_key = redis.pop(token_user_key)
redis.r.delete(token_user_key, token_key)
except DataError:
pass
def get_geo_config():
# return DbCache.get(CACHE_KEY_GEO_CONFIG)
config = redis.pop(CACHE_KEY_GEO_CONFIG)
if config:
return config
# either it doesn't exist OR it's empty
return cache_geo_config()
def delete(self, user):
token_user_key = self.get_token_user_key(user.id)
token_key = redis.pop(token_user_key)
if not token_key:
msg = f'TokenCache.delete("{token_user_key}"): token_user_key was not found'
logger.error(msg)
raise RuntimeError(msg)
redis.r.delete(token_user_key, token_key)
def calc_block_similarity(task, df1_redis_key, df2_redis_key,
search_similar_documents,
engine_kwargs):
task.log_info(
f'Calculate similarity for feature_df blocks with keys: '
f'"{df1_redis_key}" and "{df2_redis_key}"')
engine_class = DocumentSimilarityEngine if search_similar_documents else TextUnitSimilarityEngine
df1_data, df1_index, df1_columns = redis.pop(df1_redis_key)
df2_data, df2_index, df2_columns = redis.pop(df2_redis_key)
df1 = pd.DataFrame(df1_data, index=df1_index, columns=df1_columns)
df2 = pd.DataFrame(df2_data, index=df2_index, columns=df2_columns)
count = engine_class(**engine_kwargs).calc_block_similarity(df1, df2)
task.log_info(f'Created {count} records')
def get_token_keys(self, user):
token_user_key = self.get_token_user_key(user.id)
token_key = redis.pop(token_user_key)
if not token_key:
msg = f'TokenCache.get_token_keys("{token_user_key}"): token_user_key is not stored'
logger.error(msg)
raise RuntimeError(msg)
token = self.get_token_from_key(token_key)
return token_user_key, token_key, token
def get_cached_user(self, user_id):
users = redis.pop(self.users_cache_key)
if not users:
users = {str(i.pk): i for i in User.objects.all()}
self.cache_users(users)
# for new user
if user_id not in users:
users[user_id] = User.objects.get(id=user_id)
self.cache_users(users)
return users[user_id]
def getEffectiveLevel(self):
"""
Get the effective level for this logger.
Loop through this logger and its parents in the logger hierarchy,
looking for a non-zero logging level. Return the first one found.
"""
logger = self
# custom logic
# cannot use AppVar due to circular import issue
app_var_level = redis.pop(
f'app_var:Logging:logger:{logger.name}:log_level')
if app_var_level:
app_var_level = app_var_level.upper()
if app_var_level in logging._levelToName.values():
logger.setLevel(app_var_level)
# log sql to console
if logger.name == 'django.db.backends':
handler = logging.StreamHandler()
handler.setLevel(logger.level)
logger_handler_classes = [
i.__class__ for i in logger.handlers
]
if redis.pop(
f'app_var:Logging:logger:{logger.name}:log_to_console'
) is True:
if handler.__class__ not in logger_handler_classes:
logger.addHandler(handler)
else:
logger.handlers = [
i for i in logger.handlers
if i.__class__ != handler.__class__
]
# end custom logic
while logger:
if logger.level:
return logger.level
logger = logger.parent
return logging.NOTSET
def val(self):
# if redis.exists(self.cache_key) is True:
value = redis.pop(self.cache_key)
if value is not None:
return value
try:
self.refresh_from_db()
except AppVar.DoesNotExist:
self.save()
self.cache()
return self.value
def val(self):
if is_migrating or settings.TEST_RUN_MODE:
return self.value
if redis.exists(self.cache_key) is True:
return redis.pop(self.cache_key)
try:
self.refresh_from_db()
except AppVar.DoesNotExist:
self.save()
self.cache()
return self.value
def get_term_config(project_id=None):
# res = None
# if project_id is not None:
# key = CACHE_KEY_TERM_STEMS_PROJECT_PTN.format(project_id)
# res = DbCache.get(key)
# if res is None:
# res = DbCache.get(CACHE_KEY_TERM_STEMS)
# return res
# 1. if project config needed
if project_id is not None:
key = CACHE_KEY_TERM_STEMS_PROJECT_PTN.format(project_id)
config = redis.pop(key)
# 1.1. if project config exists
if config:
return config
# 2. either for global config or if project config requested, but it doesn't exist or empty
config = redis.pop(CACHE_KEY_TERM_STEMS)
if config:
return config
return cache_term_stems()
def get_user(self, token):
token_key = self.get_token_key(token)
token_user_key = redis.pop(token_key)
if token_user_key is None:
logger.warning(
f'Not found cached token user key for token "{token}"')
return
try:
user_id = self.token_re.findall(token_user_key)[0]
# try to avoid querying db users in each API request
user = self.get_cached_user(user_id)
# user = User.objects.get(id=user_id)
return user
except:
logger.warning(
f'Not found cached user for token "{token_user_key}:{token}"')
return