def delete_dangling_stats(id_list, container):
"""
This method will delete dangling statistics from Redis for each of three containers
:param id_list: List of IDs
:param container: Container's name
:return:
"""
if container == 'smartlist':
redist_key = 'smartlists_growth_stat_v2_'
elif container == 'talent-pool':
redist_key = 'pools_growth_stat_v2_'
elif container == 'talent-pipeline':
redist_key = 'pipelines_growth_stat_v2_'
else:
raise Exception("Container %s is not supported" % container)
for key in redis_store.keys(redist_key + '*'):
hashed_key = redis_store.get(key)
if int(key.replace(redist_key, '')) not in id_list:
redis_store.delete(hashed_key)
redis_store.delete(key)
else:
# Delete all stats which are older than 90 days
for date_time_key in redis_store.hkeys(hashed_key):
if datetime.strptime(
date_time_key,
'%m/%d/%Y') < datetime.utcnow() - timedelta(days=90):
redis_store.delete(
redis_store.hget(hashed_key, date_time_key))
redis_store.hdel(hashed_key, date_time_key)
logger.info("Dangling Statistics have been deleted for %s" % container)
def update_pipeline_engagement_score():
with app.app_context():
# Updating SmartList Statistics
logger.info(
"TalentPipeline Engagement Score update process has been started "
"at %s" % datetime.utcnow().date().isoformat())
talent_pipelines = TalentPipeline.query.with_entities(
TalentPipeline.id).all()
talent_pipeline_ids = map(lambda talent_pipeline: talent_pipeline[0],
talent_pipelines)
talent_pipeline_cache = redis_dict(redis_store,
'pipelines_engagement_score')
for talent_pipeline_id in talent_pipeline_ids:
try:
pipeline_cache_key = 'pipelines_engagement_score_%s' % talent_pipeline_id
with Timeout(seconds=120):
talent_pipeline_cache[
pipeline_cache_key] = engagement_score_of_pipeline(
talent_pipeline_id)
logger.info(
"Engagement Score for TalentPipeline %s have been updated successfully"
% talent_pipeline_id)
except TimeoutException:
logger.exception(
"Timelimit exceeded for updating Engagement Score for "
"TalentPipeline %s" % talent_pipeline_id)
def update_smartlist_stats():
with app.app_context():
# Updating SmartList Statistics
logger.info(
"SmartList statistics update process has been started at %s" %
datetime.utcnow().date().isoformat())
smartlists = Smartlist.query.with_entities(Smartlist.id).all()
for smartlist_tuple in smartlists:
try:
get_stats_generic_function(Smartlist.query.get(
smartlist_tuple[0]),
'SmartList',
is_update=True,
offset=0)
logger.info(
"Statistics for Smartlist %s have been updated successfully"
% smartlist_tuple[0])
except Exception as e:
db.session.rollback()
logger.exception(
"Update statistics for SmartList %s is not successful because: "
"%s" % (smartlist_tuple[0], e.message))
smartlist_ids = map(lambda smartlist: smartlist[0], smartlists)
delete_dangling_stats(smartlist_ids, container='smartlist')
def update_talent_pipeline_stats():
with app.app_context():
# Updating TalentPipeline Statistics
logger.info(
"TalentPipeline statistics update process has been started at %s" %
datetime.utcnow().isoformat())
talent_pipelines = TalentPipeline.query.with_entities(
TalentPipeline.id).all()
for talent_pipeline_tuple in talent_pipelines:
try:
get_stats_generic_function(TalentPipeline.query.get(
talent_pipeline_tuple[0]),
'TalentPipeline',
is_update=True,
offset=0)
logger.info(
"Statistics for TalentPipeline %s have been updated successfully"
% talent_pipeline_tuple[0])
except Exception as e:
db.session.rollback()
logger.exception(
"Update statistics for TalentPipeline %s is not successful because: "
"%s" % (talent_pipeline_tuple[0], e.message))
talent_pipeline_ids = map(lambda talent_pipeline: talent_pipeline[0],
talent_pipelines)
delete_dangling_stats(talent_pipeline_ids, container='talent-pipeline')
def delete(self, **kwargs):
"""
Deletes (hides) the smartlist
:return: Id of deleted smartlist.
"""
list_id = kwargs.get('id')
if not list_id:
return InvalidUsage("List id is required for deleting a list")
smartlist = Smartlist.query.get(list_id)
if not smartlist or smartlist.is_hidden:
raise NotFoundError("List id does not exists")
if request.user.role.name == 'USER' and smartlist.user.id != request.user.id:
raise ForbiddenError(
"Logged-in user doesn't have appropriate permissions to delete this smartlist"
)
# check whether smartlist belongs to user's domain
if request.user.role.name != 'TALENT_ADMIN' and smartlist.user.domain_id != request.user.domain_id:
raise ForbiddenError(
"Logged-in user doesn't have appropriate permissions to delete this smartlist"
)
smartlist_candidate_ids = SmartlistCandidate.query.with_entities(
SmartlistCandidate.candidate_id).filter(
SmartlistCandidate.smartlist_id == list_id).all()
smartlist_candidate_ids = [
smartlist_candidate_id[0]
for smartlist_candidate_id in smartlist_candidate_ids
]
smartlist.delete()
if smartlist_candidate_ids:
logger.info(
"Candidates %s of SmartList %s are going to be updated in Amazon Cloud "
"Search" % (smartlist_candidate_ids, list_id))
update_candidates_on_cloudsearch(request.oauth_token,
smartlist_candidate_ids)
return {'smartlist': {'id': smartlist.id}}
"""
Run Celery Worker
For Celery to run from command line, script runs as separate process with celery command
Usage: open terminal cd to talent-flask-services directory
Run the following command to start celery worker:
celery -A candidate_pool_service.candidate_pool_app.celery_app worker --concurrency=4 --loglevel=info
"""
# Service Specific
from candidate_pool_service.common.talent_celery import CELERY_WORKER_ARGS
from candidate_pool_service.candidate_pool_app import logger, celery_app as celery
logger.info("Celery worker has been started successfully")
celery.start(argv=CELERY_WORKER_ARGS + ['celery_stats_scheduler'] +
['-n', 'celery_stats_scheduler'])
from datetime import datetime
from candidate_pool_service.candidate_pool_app import logger
from candidate_pool_service.common.redis_cache import redis_store
from candidate_pool_service.candidate_pool_app.talent_pools_pipelines_utilities import (update_smartlist_stats,
update_talent_pool_stats,
update_talent_pipeline_stats,
update_pipeline_engagement_score)
stats_update_key = 'stats-update-timestamp-%s' % datetime.utcnow().date().strftime('%m/%d/%Y')
if not redis_store.exists(stats_update_key):
redis_store.setex(stats_update_key, 1, 86400)
logger.info("CRON JOB: Stats update process has been started")
update_smartlist_stats.delay()
update_talent_pool_stats.delay()
update_talent_pipeline_stats.delay()
update_pipeline_engagement_score.delay()