def pool_update(self, pool_name, var, val):
log.info("pool_update %s %s %s" % (pool_name, var, val))
pool = [p for p in self._objects['osd_map']['pools'] if p['pool_name'] == pool_name][0]
if var in ['pg_num', 'pgp_num']:
pgs = [p for p in self._objects['pg_brief'] if p['pgid'].startswith("{0}.".format(pool['pool']))]
states = set()
for p in pgs:
states |= set(p['state'].split("+"))
if 'creating' in states:
raise RuntimeError("Cannot modify pg_num while PGs are creating")
if var == 'pg_num':
log.debug("pool_update creating pgs %s->%s" % (
pool['pg_num'], val
))
# Growing a pool, creating PGs
new_pg_count = val - pool['pg_num']
osd_count = min(pool['pg_num'], len(self._objects['osd_map']['osds']))
if new_pg_count > osd_count * int(self._objects['config']['mon_osd_max_split_count']):
raise RuntimeError("Exceeded mon_osd_max_split_count")
self._create_pgs(pool['pool'], range(pool['pg_num'], val))
if var == 'pgp_num':
# On the way in it's called pgp_num, on the way out it's called pg_placement_num
var = 'pg_placement_num'
if pool[var] != val:
pool[var] = val
self._objects['osd_map']['epoch'] += 1
def main():
parser = argparse.ArgumentParser(description='Start simulated salt minions.')
parser.add_argument('--count', dest='count', type=int, default=3, help='Number of simulated minions')
parser.add_argument('--osds-per-host', dest='osds_per_host', type=int, default=4,
help='Number of OSDs on each simulated server')
args = parser.parse_args()
config_path = os.getcwd()
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(name)s %(message)s"))
log.addHandler(handler)
handler = logging.FileHandler("minion_sim.log")
handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(name)s %(message)s"))
log.addHandler(handler)
sim = MinionSim(config_path, args.count, args.osds_per_host)
log.debug("Starting simulator...")
sim.start()
try:
log.debug("Waiting for simulator...")
while True:
time.sleep(0.5)
except KeyboardInterrupt:
log.debug("Terminating simulator...")
sim.stop()
sim.join()
log.debug("Complete.")
def _create_pgs(self, pool_id, new_ids):
pool = [p for p in self._objects['osd_map']['pools'] if p['pool'] == pool_id][0]
for i in new_ids:
pg_id = "%s.%s" % (pool['pool'], i)
log.debug("_create_pgs created pg %s" % pg_id)
osds = pseudorandom_subset(range(0, len(self._objects['osd_map']['osds'])),
pool['size'], pg_id)
self._objects['pg_brief'].append({
'pgid': pg_id,
'state': 'creating',
'up': osds,
'acting': osds
})
self._objects['pg_map']['version'] += 1
def _update_health(self):
"""
Update the 'health' object based on the cluster maps
"""
old_health = self._objects['health']['overall_status']
if any([pg['state'] != 'active+clean' for pg in self._objects['pg_brief']]):
health = "HEALTH_WARN"
self._objects['health']['summary'] = [{
'severity': "HEALTH_WARN",
'summary': "Unclean PGs"
}]
else:
health = "HEALTH_OK"
if old_health != health:
self._objects['health']['overall_status'] = health
log.debug("update_health: %s->%s" % (old_health, health))
def _pg_monitor(self, recovery_credits=0, creation_credits=0):
"""
Crude facimile of the PG monitor. For each PG, based on its
current state and the state of its OSDs, update it: usually do
nothing, maybe mark it stale, maybe remap it.
"""
osds = dict([(osd['osd'], osd) for osd in self._objects['osd_map']['osds']])
changes = False
for pg in self._objects['pg_brief']:
states = set(pg['state'].split('+'))
primary_osd_id = pg['acting'][0]
# Call a PG is stale if its primary OSD is down
if osds[primary_osd_id]['in'] == 1 and osds[primary_osd_id]['up'] == 0:
states.add('stale')
else:
states.discard('stale')
# Call a PG active if any of its OSDs are in
if any([osds[i]['in'] == 1 for i in pg['acting']]):
states.add('active')
else:
states.discard('active')
# Remap a PG if any of its OSDs are out
if any([osds[i]['in'] == 0 for i in pg['acting']]):
states.add('remapped')
osd_ids = self._pg_id_to_osds(pg['pgid'])
pg['up'] = osd_ids
pg['acting'] = osd_ids
# Call a PG clean if its not remapped and all its OSDs are in
if all([osds[i]['in'] == 1 for i in pg['acting']]) and 'remapped' not in states:
states.add('clean')
else:
states.discard('clean')
if recovery_credits > 0 and 'remapped' in states:
states.discard('remapped')
recovery_credits -= 1
log.debug("Recovered PG %s" % pg['pgid'])
if creation_credits > 0 and 'creating' in states:
states.discard('creating')
creation_credits -= 1
log.debug("Completed creation PG %s" % pg['pgid'])
new_state = "+".join(sorted(list(states)))
if pg['state'] != new_state:
log.debug("New PG state %s: %s" % (pg['pgid'], new_state))
changes = True
pg['state'] = new_state
if changes:
self._objects['pg_map']['version'] += 1
self._update_health()
def set_osd_state(self, osd_id, up=None, osd_in=None):
log.debug("set_osd_state: '%s' %s %s %s" % (osd_id, osd_id.__class__, up, osd_in))
# Update OSD map
dirty = False
osd = [o for o in self._objects['osd_map']['osds'] if o['osd'] == osd_id][0]
if up is not None and osd['up'] != up:
log.debug("Mark OSD %s up=%s" % (osd_id, up))
osd['up'] = up
dirty = True
if osd_in is not None and osd['in'] != osd_in:
log.debug("Mark OSD %s in=%s" % (osd_id, osd_in))
osd['in'] = osd_in
dirty = True
if not dirty:
return
log.debug("Advancing OSD map")
self._objects['osd_map']['epoch'] += 1
self._pg_monitor()
self._update_health()
def run(self):
# A thread to generate some synthetic activity on the synthetic cluster
load_gen = LoadGenerator(self.cluster)
load_gen.start()
self.start_minions()
self._server_available.set()
log.debug("Starting XMLRPC server...")
self._server.serve_forever()
self._server.server_close()
log.debug("XMLRPC server terminated, stopping threads")
log.debug("Stopping load gen")
load_gen.stop()
load_gen.join()
log.debug("Stopping minions")
self.halt_minions()
log.debug("Saving state")
self.cluster.save()
log.debug("Complete.")
