def should_run_scheduled_action(self, action_name, scheduled_at, cleanup_fn): if scheduled_at: # If the scheduled action is in the far future, we shouldn't do anything and just return. # If the scheduled action is in the past, we consider the value to be stale and we remove # the value. # If the value is close to now, we initiate the scheduled action # Additionally, if the scheduled action cannot be executed altogether, i.e. there is an error # or the action is in the past - we take care of cleaning it up. now = datetime.datetime.now(pytz.utc) try: delta = (scheduled_at - now).total_seconds() if delta > self.dcs.loop_wait: logger.info('Awaiting %s at %s (in %.0f seconds)', action_name, scheduled_at.isoformat(), delta) return False elif delta < - int(self.dcs.loop_wait * 1.5): logger.warning('Found a stale %s value, cleaning up: %s', action_name, scheduled_at.isoformat()) cleanup_fn() self.dcs.manual_failover('', '', index=self.cluster.failover.index) return False # The value is very close to now sleep(max(delta, 0)) logger.info('Manual scheduled {0} at %s'.format(action_name), scheduled_at.isoformat()) return True except TypeError: logger.warning('Incorrect value of scheduled_at: %s', scheduled_at) cleanup_fn() return False
def initialize(self): # wait for etcd to be available while not self.touch_member(): logger.info('waiting on DCS') sleep(5) # is data directory empty? if self.postgresql.data_directory_empty(): while True: try: cluster = self.ha.dcs.get_cluster() if not cluster.is_unlocked(): # the leader already exists if not cluster.initialize: self.ha.dcs.initialize() self.postgresql.bootstrap(cluster.leader) break # racing to initialize elif not cluster.initialize and self.ha.dcs.initialize(): try: self.postgresql.bootstrap() except: # bail out and clean the initialize flag. self.cleanup_on_failed_initialization() raise self.ha.dcs.take_leader() break except DCSError: logger.info('waiting on DCS') sleep(5) elif self.postgresql.is_running(): self.postgresql.load_replication_slots()
def process_unhealthy_cluster(self): """Cluster has no leader key""" if self.is_healthiest_node(): if self.acquire_lock(): failover = self.cluster.failover if failover: if self.is_paused() and failover.leader and failover.candidate: logger.info('Updating failover key after acquiring leader lock...') self.dcs.manual_failover('', failover.candidate, failover.scheduled_at, failover.index) else: logger.info('Cleaning up failover key after acquiring leader lock...') self.dcs.manual_failover('', '') self.load_cluster_from_dcs() return self.enforce_master_role('acquired session lock as a leader', 'promoted self to leader by acquiring session lock') else: return self.follow('demoted self after trying and failing to obtain lock', 'following new leader after trying and failing to obtain lock') else: # when we are doing manual failover there is no guaranty that new leader is ahead of any other node # node tagged as nofailover can be ahead of the new leader either, but it is always excluded from elections need_rewind = bool(self.cluster.failover) or self.patroni.nofailover if need_rewind: sleep(2) # Give a time to somebody to take the leader lock if self.patroni.nofailover: return self.follow('demoting self because I am not allowed to become master', 'following a different leader because I am not allowed to promote', need_rewind=need_rewind) return self.follow('demoting self because i am not the healthiest node', 'following a different leader because i am not the healthiest node', need_rewind=need_rewind)
def create_session(self): while not self._session: try: self.refresh_session() except ConsulError: logger.info('waiting on consul') sleep(5)
def should_run_scheduled_action(self, action_name, scheduled_at, cleanup_fn): if scheduled_at and not self.is_paused(): # If the scheduled action is in the far future, we shouldn't do anything and just return. # If the scheduled action is in the past, we consider the value to be stale and we remove # the value. # If the value is close to now, we initiate the scheduled action # Additionally, if the scheduled action cannot be executed altogether, i.e. there is an error # or the action is in the past - we take care of cleaning it up. now = datetime.datetime.now(pytz.utc) try: delta = (scheduled_at - now).total_seconds() if delta > self.dcs.loop_wait: logger.info('Awaiting %s at %s (in %.0f seconds)', action_name, scheduled_at.isoformat(), delta) return False elif delta < -int(self.dcs.loop_wait * 1.5): logger.warning('Found a stale %s value, cleaning up: %s', action_name, scheduled_at.isoformat()) cleanup_fn() return False # The value is very close to now sleep(max(delta, 0)) logger.info('Manual scheduled {0} at %s'.format(action_name), scheduled_at.isoformat()) return True except TypeError: logger.warning('Incorrect value of scheduled_at: %s', scheduled_at) cleanup_fn() return False
def process_manual_failover_from_leader(self): failover = self.cluster.failover if failover.scheduled_at: # If the failover is in the far future, we shouldn't do anything and just return. # If the failover is in the past, we consider the value to be stale and we remove # the value. # If the value is close to now, we initiate the failover now = datetime.datetime.now(pytz.utc) try: delta = (failover.scheduled_at - now).total_seconds() if delta > self.patroni.nap_time: logging.info('Awaiting failover at %s (in %.0f seconds)', failover.scheduled_at.isoformat(), delta) return elif delta < -int(self.patroni.nap_time * 1.5): logger.warning( 'Found a stale failover value, cleaning up: %s', failover.scheduled_at) self.dcs.manual_failover('', '', index=self.cluster.failover.index) return # The value is very close to now sleep(max(delta, 0)) logger.info('Manual scheduled failover at {}'.format( failover.scheduled_at.isoformat())) except TypeError: logger.warning('Incorrect value in of scheduled_at: %s', failover.scheduled_at) if not failover.leader or failover.leader == self.state_handler.name: if not failover.candidate or failover.candidate != self.state_handler.name: members = [ m for m in self.cluster.members if not failover.candidate or m.name == failover.candidate ] if self.is_failover_possible( members): # check that there are healthy members self._async_executor.schedule('manual failover: demote') self._async_executor.run_async(self.demote) return 'manual failover: demoting myself' else: logger.warning( 'manual failover: no healthy members found, failover is not possible' ) else: logger.warning( 'manual failover: I am already the leader, no need to failover' ) else: logger.warning( 'manual failover: leader name does not match: %s != %s', self.cluster.failover.leader, self.state_handler.name) logger.info('Trying to clean up failover key') self.dcs.manual_failover('', '', index=self.cluster.failover.index)
def get_etcd_client(config): client = None while not client: try: client = Client(config) except etcd.EtcdException: logger.info('waiting on etcd') sleep(5) return client
def watch(self, timeout): """If the current node is a master it should just sleep. Any other node should watch for changes of leader key with a given timeout :param timeout: timeout in seconds :returns: `!True` if you would like to reschedule the next run of ha cycle""" sleep(timeout) return False
def process_sync_replication(self): """Process synchronous standby beahvior. Synchronous standbys are registered in two places postgresql.conf and DCS. The order of updating them must be right. The invariant that should be kept is that if a node is master and sync_standby is set in DCS, then that node must have synchronous_standby set to that value. Or more simple, first set in postgresql.conf and then in DCS. When removing, first remove in DCS, then in postgresql.conf. This is so we only consider promoting standbys that were guaranteed to be replicating synchronously. """ if self.is_synchronous_mode(): current = self.cluster.sync.leader and self.cluster.sync.sync_standby picked, allow_promote = self.state_handler.pick_synchronous_standby( self.cluster) if picked != current: # We need to revoke privilege from current before replacing it in the config if current: logger.info("Removing synchronous privilege from %s", current) if not self.dcs.write_sync_state( self.state_handler.name, None, index=self.cluster.sync.index): logger.info( 'Synchronous replication key updated by someone else.' ) return logger.info("Assigning synchronous standby status to %s", picked) self.state_handler.set_synchronous_standby(picked) if picked and not allow_promote: # Wait for PostgreSQL to enable synchronous mode and see if we can immediately set sync_standby sleep(2) picked, allow_promote = self.state_handler.pick_synchronous_standby( self.cluster) if allow_promote: cluster = self.dcs.get_cluster() if cluster.sync.leader and cluster.sync.leader != self.state_handler.name: logger.info( "Synchronous replication key updated by someone else" ) return if not self.dcs.write_sync_state(self.state_handler.name, picked, index=cluster.sync.index): logger.info( "Synchronous replication key updated by someone else" ) return logger.info("Synchronous standby status assigned to %s", picked) else: if self.cluster.sync.leader and self.dcs.delete_sync_state( index=self.cluster.sync.index): logger.info("Disabled synchronous replication") self.state_handler.set_synchronous_standby(None)
def create_or_restore_session(self): while not self._session: try: _, member = self._client.kv.get(self.member_path) self._session = (member or {}).get('Session') if self.refresh_session(): self._client.kv.delete(self.member_path) except (ConsulException, RequestException): logger.info('waiting on consul') sleep(5)
def demote(self, delete_leader=True): if delete_leader: self.state_handler.stop() self.dcs.delete_leader() self.touch_member() self.dcs.reset_cluster() sleep(2) # Give a time to somebody to promote self.recover() else: self.state_handler.follow(None)
def __init__(self, hosts, port, uri_path='/exhibitor/v1/cluster/list', poll_interval=300): self._exhibitor_port = port self._uri_path = uri_path self._poll_interval = poll_interval self._exhibitors = hosts self._master_exhibitors = hosts self._zookeeper_hosts = '' self._next_poll = None while not self.poll(): logger.info('waiting on exhibitor') sleep(5)
def demote(self, delete_leader=True): if delete_leader: self.state_handler.stop() self.dcs.delete_leader() self.touch_member() self.dcs.reset_cluster() sleep(2) # Give a time to somebody to promote cluster = self.dcs.get_cluster() node_to_follow = self._get_node_to_follow(cluster) self.state_handler.follow(node_to_follow, cluster.leader, True) else: self.state_handler.follow(None, None)
def demote(self, delete_leader=True): if delete_leader: self.state_handler.stop() self.state_handler.set_role('demoted') self.dcs.delete_leader() self.dcs.reset_cluster() sleep(2) # Give a time to somebody to take the leader lock cluster = self.dcs.get_cluster() node_to_follow = self._get_node_to_follow(cluster) self.state_handler.follow(node_to_follow, cluster.leader, recovery=True, need_rewind=True) else: self.state_handler.follow(None, None)
def process_manual_failover_from_leader(self): failover = self.cluster.failover if failover.scheduled_at: # If the failover is in the far future, we shouldn't do anything and just return. # If the failover is in the past, we consider the value to be stale and we remove # the value. # If the value is close to now, we initiate the failover now = datetime.datetime.now(pytz.utc) try: delta = (failover.scheduled_at - now).total_seconds() if delta > self.patroni.nap_time: logging.info('Awaiting failover at %s (in %.0f seconds)', failover.scheduled_at.isoformat(), delta) return elif delta < - int(self.patroni.nap_time * 1.5): logger.warning('Found a stale failover value, cleaning up: %s', failover.scheduled_at) self.dcs.manual_failover('', '', index=self.cluster.failover.index) return # The value is very close to now sleep(max(delta, 0)) logger.info('Manual scheduled failover at {}'.format(failover.scheduled_at.isoformat())) except TypeError: logger.warning('Incorrect value in of scheduled_at: %s', failover.scheduled_at) if not failover.leader or failover.leader == self.state_handler.name: if not failover.candidate or failover.candidate != self.state_handler.name: members = [m for m in self.cluster.members if not failover.candidate or m.name == failover.candidate] if self.is_failover_possible(members): # check that there are healthy members self._async_executor.schedule('manual failover: demote') self._async_executor.run_async(self.demote) return 'manual failover: demoting myself' else: logger.warning('manual failover: no healthy members found, failover is not possible') else: logger.warning('manual failover: I am already the leader, no need to failover') else: logger.warning('manual failover: leader name does not match: %s != %s', self.cluster.failover.leader, self.state_handler.name) logger.info('Trying to clean up failover key') self.dcs.manual_failover('', '', index=self.cluster.failover.index)
def query(self, sql, *params): max_attempts = 0 while True: ex = None try: cursor = self._cursor() cursor.execute(sql, params) return cursor except psycopg2.InterfaceError as e: ex = e except psycopg2.OperationalError as e: if self._connection and self._connection.closed == 0: raise e ex = e if ex: self.disconnect() max_attempts += 1 if max_attempts >= 3: raise ex sleep(5)
def test_sleep(self): sleep(0.01)
def watch(self, timeout): sleep(timeout)
def test_sleep(self): self.assertIsNone(sleep(0.01))
def test_sleep(self): time.sleep = time_sleep sleep(0.01)