def flush_state(self):
"""
Flushes state/config to the state file
:return:
"""
self.state['rate_limit_remaining'] = self.rate_limit_remaining
self.state['rate_limit_reset'] = self.rate_limit_reset
utils.flush_json(self.state, self.state_file_path)
def state_save(self):
"""
saves the state
:return:
"""
try:
js_q = self.state_gen()
utils.flush_json(js_q, self.state_file_path)
except Exception as e:
self.trace_logger.log(e)
logger.error('Exception in state: %s', e)
def store_checkpoint(self, iobj, idx=None):
"""
Stores checkpoint for the current input object
:param iobj:
:param idx:
:return:
"""
state_file = self.cur_state_file
input_name = self.iobj_name(iobj)
# Most importantly, flush data file buffers now so the state is in sync with the checkpoint.
if self.cur_copy_fh is not None:
self.cur_copy_fh.flush()
js = collections.OrderedDict()
js['iobj_name'] = input_name
js['time'] = time.time()
js['read_raw'] = self.read_data
js['read_processor'] = self.processor.total_len
js['block_idx'] = idx
# Serialize input object state
js['iobj'] = iobj.to_state()
# New decompressor checkpoint for random access read
if self.cur_decompressor is not None and self.cur_decompressor.last_read_aligned:
try:
total_read_dec = self.cur_decompressor.data_read
decctx = lz4framed.marshal_decompression_context(
self.cur_decompressor.ctx)
logger.debug('Decompressor state marshalled, size: %s B' %
len(decctx))
checkpoint = DecompressorCheckpoint(
pos=total_read_dec,
rec_pos=self.read_data,
plain_pos=self.processor.total_len,
ctx=decctx)
self.decompressor_checkpoints[total_read_dec] = checkpoint
decctx_str = base64.b16encode(decctx)
js['dec_ctx'] = decctx_str
except Exception as e:
logger.error('Exception when storing decompressor state: %s' %
e)
logger.warning(traceback.format_exc())
js['dec_checks'] = [
x.to_json() for x in self.decompressor_checkpoints.values()
]
utils.flush_json(js, state_file)