def test_start_stop_dumper(self): executions_queue = self.get_queue() sleep_interval = 0.01 dumper = Dumper(queue=executions_queue, sleep_interval=sleep_interval, export_dir='/tmp', batch_size=1, max_files_per_sleep=5, file_prefix='st2-stuff-', file_format='json') dumper.start() # Call stop after at least one batch was written to disk. eventlet.sleep(10 * sleep_interval) dumper.stop()
class ExecutionsExporter(consumers.MessageHandler): message_type = ActionExecutionDB def __init__(self, connection, queues): super(ExecutionsExporter, self).__init__(connection, queues) self.pending_executions = Queue.Queue() self._dumper = Dumper(queue=self.pending_executions, export_dir=cfg.CONF.exporter.dump_dir) self._consumer_thread = None def start(self, wait=False): LOG.info('Bootstrapping executions from db...') try: self._bootstrap() except: LOG.exception('Unable to bootstrap executions from db. Aborting.') raise self._consumer_thread = eventlet.spawn(super(ExecutionsExporter, self).start, wait=True) self._dumper.start() if wait: self.wait() def wait(self): self._consumer_thread.wait() self._dumper.wait() def shutdown(self): self._dumper.stop() super(ExecutionsExporter, self).shutdown() def process(self, execution): LOG.debug('Got execution from queue: %s', execution) if execution.status not in COMPLETION_STATUSES: return execution_api = ActionExecutionAPI.from_model(execution, mask_secrets=True) self.pending_executions.put_nowait(execution_api) LOG.debug("Added execution to queue.") def _bootstrap(self): marker = self._get_export_marker_from_db() LOG.info('Using marker %s...' % marker) missed_executions = self._get_missed_executions_from_db(export_marker=marker) LOG.info('Found %d executions not exported yet...', len(missed_executions)) for missed_execution in missed_executions: if missed_execution.status not in COMPLETION_STATUSES: continue execution_api = ActionExecutionAPI.from_model(missed_execution, mask_secrets=True) try: LOG.debug('Missed execution %s', execution_api) self.pending_executions.put_nowait(execution_api) except: LOG.exception('Failed adding execution to in-memory queue.') continue LOG.info('Bootstrapped executions...') def _get_export_marker_from_db(self): try: markers = DumperMarker.get_all() except: return None else: if len(markers) >= 1: marker = markers[0] return isotime.parse(marker.marker) else: return None def _get_missed_executions_from_db(self, export_marker=None): if not export_marker: return self._get_all_executions_from_db() # XXX: Should adapt this query to get only executions with status # in COMPLETION_STATUSES. filters = {'end_timestamp__gt': export_marker} LOG.info('Querying for executions with filters: %s', filters) return ActionExecution.query(**filters) def _get_all_executions_from_db(self): return ActionExecution.get_all() # XXX: Paginated call.