def pool_update(self, pool_name, var, val): log.info("pool_update %s %s %s" % (pool_name, var, val)) pool = [p for p in self._objects['osd_map']['pools'] if p['pool_name'] == pool_name][0] if var in ['pg_num', 'pgp_num']: pgs = [p for p in self._objects['pg_brief'] if p['pgid'].startswith("{0}.".format(pool['pool']))] states = set() for p in pgs: states |= set(p['state'].split("+")) if 'creating' in states: raise RuntimeError("Cannot modify pg_num while PGs are creating") if var == 'pg_num': log.debug("pool_update creating pgs %s->%s" % ( pool['pg_num'], val )) # Growing a pool, creating PGs new_pg_count = val - pool['pg_num'] osd_count = min(pool['pg_num'], len(self._objects['osd_map']['osds'])) if new_pg_count > osd_count * int(self._objects['config']['mon_osd_max_split_count']): raise RuntimeError("Exceeded mon_osd_max_split_count") self._create_pgs(pool['pool'], range(pool['pg_num'], val)) if var == 'pgp_num': # On the way in it's called pgp_num, on the way out it's called pg_placement_num var = 'pg_placement_num' if pool[var] != val: pool[var] = val self._objects['osd_map']['epoch'] += 1
def main(): parser = argparse.ArgumentParser(description='Start simulated salt minions.') parser.add_argument('--count', dest='count', type=int, default=3, help='Number of simulated minions') parser.add_argument('--osds-per-host', dest='osds_per_host', type=int, default=4, help='Number of OSDs on each simulated server') args = parser.parse_args() config_path = os.getcwd() handler = logging.StreamHandler() handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(name)s %(message)s")) log.addHandler(handler) handler = logging.FileHandler("minion_sim.log") handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(name)s %(message)s")) log.addHandler(handler) sim = MinionSim(config_path, args.count, args.osds_per_host) log.debug("Starting simulator...") sim.start() try: log.debug("Waiting for simulator...") while True: time.sleep(0.5) except KeyboardInterrupt: log.debug("Terminating simulator...") sim.stop() sim.join() log.debug("Complete.")
def _create_pgs(self, pool_id, new_ids): pool = [p for p in self._objects['osd_map']['pools'] if p['pool'] == pool_id][0] for i in new_ids: pg_id = "%s.%s" % (pool['pool'], i) log.debug("_create_pgs created pg %s" % pg_id) osds = pseudorandom_subset(range(0, len(self._objects['osd_map']['osds'])), pool['size'], pg_id) self._objects['pg_brief'].append({ 'pgid': pg_id, 'state': 'creating', 'up': osds, 'acting': osds }) self._objects['pg_map']['version'] += 1
def _update_health(self): """ Update the 'health' object based on the cluster maps """ old_health = self._objects['health']['overall_status'] if any([pg['state'] != 'active+clean' for pg in self._objects['pg_brief']]): health = "HEALTH_WARN" self._objects['health']['summary'] = [{ 'severity': "HEALTH_WARN", 'summary': "Unclean PGs" }] else: health = "HEALTH_OK" if old_health != health: self._objects['health']['overall_status'] = health log.debug("update_health: %s->%s" % (old_health, health))
def _pg_monitor(self, recovery_credits=0, creation_credits=0): """ Crude facimile of the PG monitor. For each PG, based on its current state and the state of its OSDs, update it: usually do nothing, maybe mark it stale, maybe remap it. """ osds = dict([(osd['osd'], osd) for osd in self._objects['osd_map']['osds']]) changes = False for pg in self._objects['pg_brief']: states = set(pg['state'].split('+')) primary_osd_id = pg['acting'][0] # Call a PG is stale if its primary OSD is down if osds[primary_osd_id]['in'] == 1 and osds[primary_osd_id]['up'] == 0: states.add('stale') else: states.discard('stale') # Call a PG active if any of its OSDs are in if any([osds[i]['in'] == 1 for i in pg['acting']]): states.add('active') else: states.discard('active') # Remap a PG if any of its OSDs are out if any([osds[i]['in'] == 0 for i in pg['acting']]): states.add('remapped') osd_ids = self._pg_id_to_osds(pg['pgid']) pg['up'] = osd_ids pg['acting'] = osd_ids # Call a PG clean if its not remapped and all its OSDs are in if all([osds[i]['in'] == 1 for i in pg['acting']]) and 'remapped' not in states: states.add('clean') else: states.discard('clean') if recovery_credits > 0 and 'remapped' in states: states.discard('remapped') recovery_credits -= 1 log.debug("Recovered PG %s" % pg['pgid']) if creation_credits > 0 and 'creating' in states: states.discard('creating') creation_credits -= 1 log.debug("Completed creation PG %s" % pg['pgid']) new_state = "+".join(sorted(list(states))) if pg['state'] != new_state: log.debug("New PG state %s: %s" % (pg['pgid'], new_state)) changes = True pg['state'] = new_state if changes: self._objects['pg_map']['version'] += 1 self._update_health()
def set_osd_state(self, osd_id, up=None, osd_in=None): log.debug("set_osd_state: '%s' %s %s %s" % (osd_id, osd_id.__class__, up, osd_in)) # Update OSD map dirty = False osd = [o for o in self._objects['osd_map']['osds'] if o['osd'] == osd_id][0] if up is not None and osd['up'] != up: log.debug("Mark OSD %s up=%s" % (osd_id, up)) osd['up'] = up dirty = True if osd_in is not None and osd['in'] != osd_in: log.debug("Mark OSD %s in=%s" % (osd_id, osd_in)) osd['in'] = osd_in dirty = True if not dirty: return log.debug("Advancing OSD map") self._objects['osd_map']['epoch'] += 1 self._pg_monitor() self._update_health()
def run(self): # A thread to generate some synthetic activity on the synthetic cluster load_gen = LoadGenerator(self.cluster) load_gen.start() self.start_minions() self._server_available.set() log.debug("Starting XMLRPC server...") self._server.serve_forever() self._server.server_close() log.debug("XMLRPC server terminated, stopping threads") log.debug("Stopping load gen") load_gen.stop() load_gen.join() log.debug("Stopping minions") self.halt_minions() log.debug("Saving state") self.cluster.save() log.debug("Complete.")