def __init__(self, cassandra_cluster, elasticsearch_client, settings,
state_file_name):
self._logger = logging.getLogger()
self._state_store = SyncStateStore(state_file_name)
self._interval_between_runs = settings.interval_between_runs
self._cassandra_to_elasticsearch_river = CassandraToElasticsearchRiver(
cassandra_cluster, elasticsearch_client, settings)
self._elasticsearch_to_cassandra_river = ElasticsearchToCassandraRiver(
elasticsearch_client, cassandra_cluster, settings)
def __init__(self, cassandra_cluster, elasticsearch_client, settings, state_file_name):
self._logger = logging.getLogger()
self._state_store = SyncStateStore(state_file_name)
self._interval_between_runs = settings.interval_between_runs
self._cassandra_to_elasticsearch_river = CassandraToElasticsearchRiver(
cassandra_cluster, elasticsearch_client, settings)
self._elasticsearch_to_cassandra_river = ElasticsearchToCassandraRiver(
elasticsearch_client, cassandra_cluster, settings)
class SyncLoop:
def __init__(self, cassandra_cluster, elasticsearch_client, settings,
state_file_name):
self._logger = logging.getLogger()
self._state_store = SyncStateStore(state_file_name)
self._interval_between_runs = settings.interval_between_runs
self._cassandra_to_elasticsearch_river = CassandraToElasticsearchRiver(
cassandra_cluster, elasticsearch_client, settings)
self._elasticsearch_to_cassandra_river = ElasticsearchToCassandraRiver(
elasticsearch_client, cassandra_cluster, settings)
def run(self):
try:
state = self._state_store.load()
self.__initial_sync_if_necessary(state)
while True:
self.__incremental_sync(state)
sleep(self._interval_between_runs)
except Exception as e:
self._logger.error(str(e))
self._logger.error("Aborting...")
return
def __initial_sync_if_necessary(self, state):
if not state.last_cassandra_to_elasticsearch_sync:
self.__initial_cassandra_to_elasticsearch_sync(state)
if not state.last_elasticsearch_to_cassandra_sync:
self.__initial_elasticsearch_to_cassandra_sync(state)
def __initial_cassandra_to_elasticsearch_sync(self, state):
self._logger.warning(
"Initial Cassandra to Elasticsearch sync. This might take a while..."
)
state.last_cassandra_to_elasticsearch_sync = self._cassandra_to_elasticsearch_river.propagate_updates(
)
state.save()
def __initial_elasticsearch_to_cassandra_sync(self, state):
self._logger.warning(
"Initial Elasticsearch to Cassandra sync. This will take a while..."
)
state.last_elasticsearch_to_cassandra_sync = self._elasticsearch_to_cassandra_river.propagate_updates(
)
state.save()
def __incremental_sync(self, state):
self.__incremental_cassandra_to_elasticsearch_sync(state)
sleep(_INTERVAL_BETWEEN_RIVER_SYNCS)
self.__incremental_elasticsearch_to_cassandra_sync(state)
def __incremental_cassandra_to_elasticsearch_sync(self, state):
self._logger.info(
"Syncing Cassandra to Elasticsearch since %s...",
self.__format_timestamp(
state.last_cassandra_to_elasticsearch_sync))
timestamp = self._cassandra_to_elasticsearch_river.propagate_updates(
state.last_cassandra_to_elasticsearch_sync)
state.last_cassandra_to_elasticsearch_sync = timestamp
state.save()
self._logger.info("...synced until %s.",
self.__format_timestamp(timestamp))
def __incremental_elasticsearch_to_cassandra_sync(self, state):
self._logger.info(
"Syncing Elasticsearch to Cassandra since %s...",
self.__format_timestamp(
state.last_elasticsearch_to_cassandra_sync))
timestamp = self._elasticsearch_to_cassandra_river.propagate_updates(
state.last_elasticsearch_to_cassandra_sync)
state.last_elasticsearch_to_cassandra_sync = timestamp
state.save()
self._logger.info("...synced until %s.",
self.__format_timestamp(timestamp))
@staticmethod
def __format_timestamp(timestamp):
return arrow.get(timestamp).to("local").format(
"YYYY-MM-DD HH:mm:ss.SSSSSSZZ")
def river(cassandra_cluster, elasticsearch_client, settings):
return ElasticsearchToCassandraRiver(elasticsearch_client,
cassandra_cluster, settings)
class SyncLoop:
def __init__(self, cassandra_cluster, elasticsearch_client, settings, state_file_name):
self._logger = logging.getLogger()
self._state_store = SyncStateStore(state_file_name)
self._interval_between_runs = settings.interval_between_runs
self._cassandra_to_elasticsearch_river = CassandraToElasticsearchRiver(
cassandra_cluster, elasticsearch_client, settings)
self._elasticsearch_to_cassandra_river = ElasticsearchToCassandraRiver(
elasticsearch_client, cassandra_cluster, settings)
def run(self):
try:
state = self._state_store.load()
self.__initial_sync_if_necessary(state)
while True:
self.__incremental_sync(state)
sleep(self._interval_between_runs)
except Exception as e:
self._logger.error(str(e))
self._logger.error("Aborting...")
return
def __initial_sync_if_necessary(self, state):
if not state.last_cassandra_to_elasticsearch_sync:
self.__initial_cassandra_to_elasticsearch_sync(state)
if not state.last_elasticsearch_to_cassandra_sync:
self.__initial_elasticsearch_to_cassandra_sync(state)
def __initial_cassandra_to_elasticsearch_sync(self, state):
self._logger.warning("Initial Cassandra to Elasticsearch sync. This might take a while...")
state.last_cassandra_to_elasticsearch_sync = self._cassandra_to_elasticsearch_river.propagate_updates()
state.save()
def __initial_elasticsearch_to_cassandra_sync(self, state):
self._logger.warning("Initial Elasticsearch to Cassandra sync. This will take a while...")
state.last_elasticsearch_to_cassandra_sync = self._elasticsearch_to_cassandra_river.propagate_updates()
state.save()
def __incremental_sync(self, state):
self.__incremental_cassandra_to_elasticsearch_sync(state)
sleep(_INTERVAL_BETWEEN_RIVER_SYNCS)
self.__incremental_elasticsearch_to_cassandra_sync(state)
def __incremental_cassandra_to_elasticsearch_sync(self, state):
self._logger.info("Syncing Cassandra to Elasticsearch since %s...",
self.__format_timestamp(state.last_cassandra_to_elasticsearch_sync))
timestamp = self._cassandra_to_elasticsearch_river.propagate_updates(state.last_cassandra_to_elasticsearch_sync)
state.last_cassandra_to_elasticsearch_sync = timestamp
state.save()
self._logger.info("...synced until %s.", self.__format_timestamp(timestamp))
def __incremental_elasticsearch_to_cassandra_sync(self, state):
self._logger.info("Syncing Elasticsearch to Cassandra since %s...",
self.__format_timestamp(state.last_elasticsearch_to_cassandra_sync))
timestamp = self._elasticsearch_to_cassandra_river.propagate_updates(state.last_elasticsearch_to_cassandra_sync)
state.last_elasticsearch_to_cassandra_sync = timestamp
state.save()
self._logger.info("...synced until %s.", self.__format_timestamp(timestamp))
@staticmethod
def __format_timestamp(timestamp):
return arrow.get(timestamp).to("local").format("YYYY-MM-DD HH:mm:ss.SSSSSSZZ")