def update_log_checkpoint(log_type, log_offset, child_account_id):
"""
Save log_offset to the checkpoint file for log_type.
@param log_type Used to determine which checkpoint file to open
@param log_offset Information to save in the checkpoint file
"""
Program.log(
f"{log_type} consumer: saving latest log offset to a "
"checkpointing file", logging.INFO)
file_path = os.path.join(
Config.get_checkpoint_dir(),
f"{log_type}_checkpoint_data_" + child_account_id + ".txt")\
if child_account_id else os.path.join(
Config.get_checkpoint_dir(),
f"{log_type}_checkpoint_data.txt")
checkpoint_filename = file_path
# Open file checkpoint_filename in writing mode only
checkpoint_file = open(checkpoint_filename, 'w')
checkpoint_file.write(json.dumps(log_offset) + '\n')
# According to Python docs, closing a file also flushes the file
checkpoint_file.close()
def test_get_checkpoint_dir(self):
config = {'dls_settings': {'checkpointing': {'directory': '/tmp'}}}
Config.set_config(config)
checkpoint_dir = Config.get_checkpoint_dir()
self.assertEqual(checkpoint_dir, '/tmp')
def __init__(self, api_call, log_queue, log_type):
self.api_call = api_call
self.log_queue = log_queue
self.log_type = log_type
self.log_offset = get_log_offset(self.log_type,
Config.get_checkpointing_enabled(),
Config.get_checkpoint_dir())
def __init__(self, api_call, log_queue, log_type, account_id=None, url_path=None):
self.api_call = api_call
self.log_queue = log_queue
self.log_type = log_type
self.account_id = account_id
self.log_offset = get_log_offset(
self.log_type,
Config.get_checkpointing_enabled(),
Config.get_checkpoint_dir(),
self.account_id)
self.url_path = url_path
