def find_shard_mismatches(instance=False):
""" Find shards that are missing or unexpected in a sharded dataset
Args:
instance - If supplied, only check this instance.
Returns:
orphaned - A dict of unexpected and (according to table statistics)
unused dbs. Key is master instance, value is a set.
orphaned_but_used - A dict of unexpected and but used dbs.
Data structure is the same as orphaned.
missing - A dict of expected but missing dbs.
Data structure is the same as orphaned.
"""
orphaned = dict()
orphaned_but_used = dict()
missing_dbs = dict()
zk = host_utils.MysqlZookeeper()
rs_dbs_map = zk.get_sharded_dbs_by_replica_set()
if instance:
rs = zk.get_replica_set_from_instance(instance)
rs_dbs_map = {rs: rs_dbs_map[rs]}
for rs in rs_dbs_map:
# non-sharded replica sets
if not len(rs_dbs_map[rs]):
continue
expected_dbs = rs_dbs_map[rs]
instance = zk.get_mysql_instance_from_replica_set(rs)
activity = mysql_lib.get_dbs_activity(instance)
actual_dbs = mysql_lib.get_dbs(instance)
unexpected_dbs = actual_dbs.difference(expected_dbs)
missing = expected_dbs.difference(actual_dbs)
if missing:
missing_dbs[instance] = expected_dbs.difference(actual_dbs)
for db in unexpected_dbs:
if activity[db]['ROWS_CHANGED'] != 0:
if instance not in orphaned_but_used:
orphaned_but_used[instance] = set()
orphaned_but_used[instance].add(db)
else:
if instance not in orphaned:
orphaned[instance] = set()
orphaned[instance].add(db)
return orphaned, orphaned_but_used, missing_dbs
def find_shard_mismatches(instance=False):
""" Find shards that are missing or unexpected in modhsarddb and sharddb
Args:
instance - If supplied, only check this instance.
Returns:
orphaned - A dict of unexpected and (according to table statistics)
unused shards. Key is master instance, value is a set.
orphaned_but_used - A dict of unexpected and but used shards.
Data strucutre is the same as orphaned.
missing - A dict of expected but missing shards.
Data strucutre is the same as orphaned.
"""
orphaned = dict()
orphaned_but_used = dict()
missing_shards = dict()
zk = host_utils.MysqlZookeeper()
host_shard_map = zk.get_host_shard_map()
if instance:
new_host_shard_map = dict()
new_host_shard_map[instance.__str__()] = host_shard_map[
instance.__str__()]
host_shard_map = new_host_shard_map
for master in host_shard_map:
expected_shards = host_shard_map[master]
instance = host_utils.HostAddr(master)
conn = mysql_lib.connect_mysql(instance)
activity = mysql_lib.get_dbs_activity(conn)
actual_shards = mysql_lib.get_dbs(conn)
unexpected_shards = actual_shards.difference(expected_shards)
missing = expected_shards.difference(actual_shards)
if missing:
missing_shards[master] = expected_shards.difference(actual_shards)
for db in unexpected_shards:
if activity[db]['ROWS_CHANGED'] != 0:
if master not in orphaned_but_used:
orphaned_but_used[master] = set()
orphaned_but_used[master].add(db)
else:
if master not in orphaned:
orphaned[master] = set()
orphaned[master].add(db)
return orphaned, orphaned_but_used, missing_shards
def find_shard_mismatches(instance=False):
""" Find shards that are missing or unexpected in modhsarddb and sharddb
Args:
instance - If supplied, only check this instance.
Returns:
orphaned - A dict of unexpected and (according to table statistics)
unused shards. Key is master instance, value is a set.
orphaned_but_used - A dict of unexpected and but used shards.
Data strucutre is the same as orphaned.
missing - A dict of expected but missing shards.
Data strucutre is the same as orphaned.
"""
orphaned = dict()
orphaned_but_used = dict()
missing_shards = dict()
zk = host_utils.MysqlZookeeper()
host_shard_map = zk.get_host_shard_map()
if instance:
new_host_shard_map = dict()
new_host_shard_map[instance.__str__()] = host_shard_map[instance.__str__()]
host_shard_map = new_host_shard_map
for master in host_shard_map:
expected_shards = host_shard_map[master]
instance = host_utils.HostAddr(master)
conn = mysql_lib.connect_mysql(instance)
activity = mysql_lib.get_dbs_activity(conn)
actual_shards = mysql_lib.get_dbs(conn)
unexpected_shards = actual_shards.difference(expected_shards)
missing = expected_shards.difference(actual_shards)
if missing:
missing_shards[master] = expected_shards.difference(actual_shards)
for db in unexpected_shards:
if activity[db]['ROWS_CHANGED'] != 0:
if master not in orphaned_but_used:
orphaned_but_used[master] = set()
orphaned_but_used[master].add(db)
else:
if master not in orphaned:
orphaned[master] = set()
orphaned[master].add(db)
return orphaned, orphaned_but_used, missing_shards
def confirm_no_writes(conn):
""" Confirm that a server is not receiving any writes
Args:
conn - A mysql connection
"""
mysql_lib.enable_and_flush_activity_statistics(conn)
log.info('Waiting {length} seconds to confirm instance is no longer '
'accepting writes'.format(length=WAIT_TIME_CONFIRM_QUIESCE))
time.sleep(WAIT_TIME_CONFIRM_QUIESCE)
db_activity = mysql_lib.get_dbs_activity(conn)
active_db = set()
for db in db_activity:
if db_activity[db]['ROWS_CHANGED'] != 0:
active_db.add(db)
if active_db:
raise Exception('DB {dbs} has been modified when it should have '
'no activity'.format(dbs=active_db))
log.info('No writes after sleep, looks like we are good to go')
def confirm_no_writes(instance):
""" Confirm that a server is not receiving any writes
Args:
conn - A mysql connection
"""
mysql_lib.enable_and_flush_activity_statistics(instance)
log.info('Waiting {length} seconds to confirm instance is no longer '
'accepting writes'.format(length=WAIT_TIME_CONFIRM_QUIESCE))
time.sleep(WAIT_TIME_CONFIRM_QUIESCE)
db_activity = mysql_lib.get_dbs_activity(instance)
active_db = set()
for db in db_activity:
if db_activity[db]['ROWS_CHANGED'] != 0:
active_db.add(db)
if active_db:
raise Exception('DB {dbs} has been modified when it should have '
'no activity'.format(dbs=active_db))
log.info('No writes after sleep, looks like we are good to go')