def fail_proof_set(key, value, timeout):
"""
:param key: key to save the item with
:param value: item to save in the cache
:param timeout: time for which the item will be valid
"""
try:
CACHE.set(key=key, value=value, timeout=timeout)
# pylint: disable=no-member
except pylibmc.TooBig as error:
app_logging.info(
f"Error while writing to cache({str(error)}). Continuing. Don't worry"
)
def fail_proof_get(key):
"""
:param key: the key of the item to get
:return: the stored item if it extists None if it doesn't or there was a connection failure
"""
try:
item = CACHE.get(key=key)
return item
# pylint: disable=no-member
except pylibmc.ConnectionError as error:
app_logging.info(
f"Error while reading from cache({str(error)}). Returning None and continuing. Don't worry"
)
return None
def run(self):
dice = random.randint(0, 9)
dice_must_be = [0, 1, 2]
do_this = dice in dice_must_be
if not do_this:
app_logging.info(
f'Not trying to delete expired urls. Dice was {dice}. Must be {str(dice_must_be)}'
)
return
now = datetime.utcnow().replace(tzinfo=timezone.utc)
index_name = RUN_CONFIG.get('url_shortening').get('index_name')
query = {
"query": {
"range": {
"expires": {
"lte": str(int(now.timestamp() * 1000))
}
}
}
}
expired_urls_response = es_data.get_es_response(index_name,
query,
ignore_cache=True)
num_items = expired_urls_response['hits']['total']['value']
max_items = RUN_CONFIG.get('url_shortening').get(
'expired_urls_lazy_deletion_threshold')
must_do_deletion = num_items > max_items
if must_do_deletion:
# pylint: disable=unexpected-keyword-arg
ES.delete_by_query(index=index_name,
body=query,
conflicts='proceed')
app_logging.info(f'Deleted {num_items} expired shortened urls.')
statistics_saver.record_expired_urls_were_deleted()
else:
app_logging.info(
f'Not deleting expired urls because there are just {num_items}, '
f'will do deletion when more than {max_items}')
def run(self):
dice = random.randint(0, 1000)
dice_must_be = 0
do_this = dice == dice_must_be
if not do_this:
app_logging.info(f'Not trying to delete expired urls. Dice was {dice}. Must be {dice_must_be}')
return
indexes_and_timestamps = RUN_CONFIG.get('lazy_old_record_deletion').get('index_names_and_timestamps')
for config in indexes_and_timestamps:
days_old = RUN_CONFIG.get('lazy_old_record_deletion').get('delete_records_older_than_days')
for current_days_old in range(680, days_old, -1):
app_logging.info(f'---')
delete_old_index_data(config['index_name'], config['timestamp_field'], current_days_old)
app_logging.info(f'---')
def __init__(self):
threading.Thread.__init__(self, )
app_logging.info('Initialising thread to delete expired urls')
os.makedirs(JOBS_RUN_DIR, exist_ok=True)
JOBS_TMP_DIR = RUN_CONFIG.get('jobs_tmp_dir',
str(Path().absolute()) + '/jobs_tmp_dir')
if not os.path.isabs(JOBS_TMP_DIR):
JOBS_TMP_DIR = Path(JOBS_TMP_DIR).resolve()
os.makedirs(JOBS_TMP_DIR, exist_ok=True)
JOBS_OUTPUT_DIR = RUN_CONFIG.get('jobs_output_dir',
str(Path().absolute()) + '/jobs_output')
if not os.path.isabs(JOBS_OUTPUT_DIR):
JOBS_OUTPUT_DIR = Path(JOBS_OUTPUT_DIR).resolve()
os.makedirs(JOBS_OUTPUT_DIR, exist_ok=True)
app_logging.info(
'------------------------------------------------------------------------------'
)
app_logging.info(f'JOBS_RUN_DIR: {JOBS_RUN_DIR}')
app_logging.info(
'------------------------------------------------------------------------------'
)
app_logging.info(
'------------------------------------------------------------------------------'
)
app_logging.info(f'JOBS_OUTPUT_DIR: {JOBS_OUTPUT_DIR}')
app_logging.info(
'------------------------------------------------------------------------------'
)
JOBS_SCRIPTS_DIR = str(Path().absolute()) + '/jobs_scripts'
def delete_old_index_data(index_name, timestamp_field, days_old):
"""
Deletes the old data of the index given as parameter
:param index_name: name of the index for which to delete the old data
:param timestamp_field: field to be used to consider the record as old
:param days_old: how many days old must the records be to be deleted
"""
now = datetime.utcnow().replace(tzinfo=timezone.utc)
time_delta = timedelta(days=days_old)
expiration_date = now - time_delta
app_logging.info(
f'Deleting old index data of index {index_name} using field {timestamp_field} that are older than '
f'{expiration_date} ({days_old} days old)')
query = {
"query": {
"range": {
f'{timestamp_field}': {
"lte": str(int(expiration_date.timestamp() * 1000))
}
}
}
}
app_logging.info(f'index_name: {index_name}')
app_logging.info(f'query: {query}')
expired_docs_response = ES_MONITORING.search(index=index_name, body=query)
num_items = expired_docs_response['hits']['total']['value']
max_items = RUN_CONFIG.get('lazy_old_record_deletion').get('expired_docs_lazy_deletion_threshold')
must_do_deletion = num_items > max_items
app_logging.info(f'there are {num_items} old records in {index_name}')
if must_do_deletion:
# pylint: disable=unexpected-keyword-arg
app_logging.info('Deleting!!!')
ES_MONITORING.delete_by_query(index=index_name, body=query, conflicts='proceed')
app_logging.info(f'Deleted {num_items} monitoring records olddr than {expiration_date} in index {index_name}')
else:
app_logging.info(
f'Not deleting expired docs of because there are just {num_items}, '
f'will do deletion when more than {max_items}')
def __init__(self):
threading.Thread.__init__(self, )
app_logging.info('Initialising thread to delete old monitoring records')