def create_pools(ctx, clients): """Create replicated or erasure coded data pools for rgw.""" log.info('Creating data pools') for client in clients: log.debug("Obtaining remote for client {}".format(client)) (remote, ) = ctx.cluster.only(client).remotes.iterkeys() data_pool = 'default.rgw.buckets.data' cluster_name, daemon_type, client_id = teuthology.split_role(client) if ctx.rgw.ec_data_pool: create_ec_pool(remote, data_pool, client, ctx.rgw.data_pool_pg_size, ctx.rgw.erasure_code_profile, cluster_name, 'rgw') else: create_replicated_pool(remote, data_pool, ctx.rgw.data_pool_pg_size, cluster_name, 'rgw') index_pool = 'default.rgw.buckets.index' create_replicated_pool(remote, index_pool, ctx.rgw.index_pool_pg_size, cluster_name, 'rgw') if ctx.rgw.cache_pools: create_cache_pool(remote, data_pool, data_pool + '.cache', 64, 64 * 1024 * 1024, cluster_name) log.debug('Pools created') yield
def create_zone_pools(ctx, zone): """ Create the data_pool for each placement type """ gateway = zone.gateways[0] cluster = zone.cluster for pool_config in zone.data.get('placement_pools', []): pool_name = pool_config['val']['storage_classes']['STANDARD']['data_pool'] if ctx.rgw.ec_data_pool: create_ec_pool(gateway.remote, pool_name, zone.name, 64, ctx.rgw.erasure_code_profile, cluster.name, 'rgw') else: create_replicated_pool(gateway.remote, pool_name, 64, cluster.name, 'rgw')
def create_zone_pools(ctx, zone): """ Create the data_pool for each placement type """ gateway = zone.gateways[0] cluster = zone.cluster for pool_config in zone.data.get('placement_pools', []): pool_name = pool_config['val']['data_pool'] if ctx.rgw.ec_data_pool: create_ec_pool(gateway.remote, pool_name, zone.name, 64, ctx.rgw.erasure_code_profile, cluster.name, 'rgw') else: create_replicated_pool(gateway.remote, pool_name, 64, cluster.name, 'rgw')
def create_nonregion_pools(ctx, config, regions): """Create replicated or erasure coded data pools for rgw.""" if regions: yield return log.info('creating data pools') for client in config.keys(): (remote,) = ctx.cluster.only(client).remotes.iterkeys() data_pool = '.rgw.buckets' if ctx.rgw.ec_data_pool: create_ec_pool(remote, data_pool, client, 64, ctx.rgw.erasure_code_profile) else: create_replicated_pool(remote, data_pool, 64) if ctx.rgw.cache_pools: create_cache_pool(remote, data_pool, data_pool + '.cache', 64, 64*1024*1024) yield
def create_nonregion_pools(ctx, config, regions): """Create replicated or erasure coded data pools for rgw.""" if regions: yield return log.info('creating data pools') for client in config.keys(): (remote,) = ctx.cluster.only(client).remotes.iterkeys() data_pool = '.rgw.buckets' if ctx.rgw.ec_data_pool: create_ec_pool(remote, data_pool, client, 64, ctx.rgw.erasure_code_profile) else: create_replicated_pool(remote, data_pool, 64) if ctx.rgw.cache_pools: create_cache_pool(remote, data_pool, data_pool + '.cache', 64, 64*1024*1024) yield
def create_pools(ctx, config): """Create replicated or erasure coded data pools for rgw.""" log.info('Creating data pools') for client in config.keys(): (remote,) = ctx.cluster.only(client).remotes.iterkeys() data_pool = '.rgw.buckets' cluster_name, daemon_type, client_id = teuthology.split_role(client) if ctx.rgw.ec_data_pool: create_ec_pool(remote, data_pool, client, 64, ctx.rgw.erasure_code_profile, cluster_name) else: create_replicated_pool(remote, data_pool, 64, cluster_name) if ctx.rgw.cache_pools: create_cache_pool(remote, data_pool, data_pool + '.cache', 64, 64*1024*1024, cluster_name) log.debug('Pools created') yield
def create_pools(ctx, clients): """Create replicated or erasure coded data pools for rgw.""" log.info('Creating data pools') for client in clients: log.debug("Obtaining remote for client {}".format(client)) (remote,) = ctx.cluster.only(client).remotes.iterkeys() data_pool = 'default.rgw.buckets.data' cluster_name, daemon_type, client_id = teuthology.split_role(client) if ctx.rgw.ec_data_pool: create_ec_pool(remote, data_pool, client, ctx.rgw.data_pool_pg_size, ctx.rgw.erasure_code_profile, cluster_name, 'rgw') else: create_replicated_pool(remote, data_pool, ctx.rgw.data_pool_pg_size, cluster_name, 'rgw') index_pool = 'default.rgw.buckets.index' create_replicated_pool(remote, index_pool, ctx.rgw.index_pool_pg_size, cluster_name, 'rgw') if ctx.rgw.cache_pools: create_cache_pool(remote, data_pool, data_pool + '.cache', 64, 64*1024*1024, cluster_name) log.debug('Pools created') yield
def task(ctx, config): """ Run ceph_objectstore_tool test The config should be as follows:: ceph_objectstore_tool: objects: 20 # <number of objects> pgnum: 12 """ if config is None: config = {} assert isinstance(config, dict), \ 'ceph_objectstore_tool task only accepts a dict for configuration' log.info('Beginning ceph_objectstore_tool...') log.debug(config) log.debug(ctx) clients = ctx.cluster.only(teuthology.is_type('client')) assert len(clients.remotes) > 0, 'Must specify at least 1 client' (cli_remote, _) = clients.remotes.popitem() log.debug(cli_remote) # clients = dict(teuthology.get_clients(ctx=ctx, roles=config.keys())) # client = clients.popitem() # log.info(client) osds = ctx.cluster.only(teuthology.is_type('osd')) log.info("OSDS") log.info(osds) log.info(osds.remotes) manager = ctx.managers['ceph'] while (len(manager.get_osd_status()['up']) != len( manager.get_osd_status()['raw'])): time.sleep(10) while (len(manager.get_osd_status()['in']) != len( manager.get_osd_status()['up'])): time.sleep(10) manager.raw_cluster_cmd('osd', 'set', 'noout') manager.raw_cluster_cmd('osd', 'set', 'nodown') PGNUM = config.get('pgnum', 12) log.info("pgnum: {num}".format(num=PGNUM)) ERRORS = 0 REP_POOL = "rep_pool" REP_NAME = "REPobject" create_replicated_pool(cli_remote, REP_POOL, PGNUM) ERRORS += test_objectstore(ctx, config, cli_remote, REP_POOL, REP_NAME) EC_POOL = "ec_pool" EC_NAME = "ECobject" create_ec_pool(cli_remote, EC_POOL, 'default', PGNUM) ERRORS += test_objectstore(ctx, config, cli_remote, EC_POOL, EC_NAME, ec=True) if ERRORS == 0: log.info("TEST PASSED") else: log.error("TEST FAILED WITH {errcount} ERRORS".format(errcount=ERRORS)) assert ERRORS == 0 try: yield finally: log.info('Ending ceph_objectstore_tool')
def task(ctx, config): """ Run ceph_objectstore_tool test The config should be as follows:: ceph_objectstore_tool: objects: 20 # <number of objects> pgnum: 12 """ if config is None: config = {} assert isinstance(config, dict), \ 'ceph_objectstore_tool task only accepts a dict for configuration' log.info('Beginning ceph_objectstore_tool...') log.debug(config) log.debug(ctx) clients = ctx.cluster.only(teuthology.is_type('client')) assert len(clients.remotes) > 0, 'Must specify at least 1 client' (cli_remote, _) = clients.remotes.popitem() log.debug(cli_remote) # clients = dict(teuthology.get_clients(ctx=ctx, roles=config.keys())) # client = clients.popitem() # log.info(client) osds = ctx.cluster.only(teuthology.is_type('osd')) log.info("OSDS") log.info(osds) log.info(osds.remotes) manager = ctx.managers['ceph'] while (len(manager.get_osd_status()['up']) != len(manager.get_osd_status()['raw'])): time.sleep(10) while (len(manager.get_osd_status()['in']) != len(manager.get_osd_status()['up'])): time.sleep(10) manager.raw_cluster_cmd('osd', 'set', 'noout') manager.raw_cluster_cmd('osd', 'set', 'nodown') PGNUM = config.get('pgnum', 12) log.info("pgnum: {num}".format(num=PGNUM)) ERRORS = 0 REP_POOL = "rep_pool" REP_NAME = "REPobject" create_replicated_pool(cli_remote, REP_POOL, PGNUM) ERRORS += test_objectstore(ctx, config, cli_remote, REP_POOL, REP_NAME) EC_POOL = "ec_pool" EC_NAME = "ECobject" create_ec_pool(cli_remote, EC_POOL, 'default', PGNUM) ERRORS += test_objectstore(ctx, config, cli_remote, EC_POOL, EC_NAME, ec=True) if ERRORS == 0: log.info("TEST PASSED") else: log.error("TEST FAILED WITH {errcount} ERRORS".format(errcount=ERRORS)) assert ERRORS == 0 try: yield finally: log.info('Ending ceph_objectstore_tool')
def configure_regions_and_zones(ctx, config, regions, role_endpoints): """ Configure regions and zones from rados and rgw. """ if not regions: log.debug( 'In rgw.configure_regions_and_zones() and regions is None. ' 'Bailing') yield return log.info('Configuring regions and zones...') log.debug('config is %r', config) log.debug('regions are %r', regions) log.debug('role_endpoints = %r', role_endpoints) # extract the zone info role_zones = dict([(client, extract_zone_info(ctx, client, c_config)) for client, c_config in config.iteritems()]) log.debug('roles_zones = %r', role_zones) # extract the user info and append it to the payload tuple for the given # client for client, c_config in config.iteritems(): if not c_config: user_info = None else: user_info = extract_user_info(c_config) (region, zone, zone_info) = role_zones[client] role_zones[client] = (region, zone, zone_info, user_info) region_info = dict([ (region_name, extract_region_info(region_name, r_config)) for region_name, r_config in regions.iteritems()]) fill_in_endpoints(region_info, role_zones, role_endpoints) # clear out the old defaults first_mon = teuthology.get_first_mon(ctx, config) (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys() # removing these objects from .rgw.root and the per-zone root pools # may or may not matter rados(ctx, mon, cmd=['-p', '.rgw.root', 'rm', 'region_info.default']) rados(ctx, mon, cmd=['-p', '.rgw.root', 'rm', 'zone_info.default']) for client in config.iterkeys(): for role, (_, zone, zone_info, user_info) in role_zones.iteritems(): rados(ctx, mon, cmd=['-p', zone_info['domain_root'], 'rm', 'region_info.default']) rados(ctx, mon, cmd=['-p', zone_info['domain_root'], 'rm', 'zone_info.default']) (remote,) = ctx.cluster.only(role).remotes.keys() for pool_info in zone_info['placement_pools']: remote.run(args=['ceph', 'osd', 'pool', 'create', pool_info['val']['index_pool'], '64', '64']) if ctx.rgw.ec_data_pool: create_ec_pool(remote, pool_info['val']['data_pool'], zone, 64, ctx.rgw.erasure_code_profile) else: create_replicated_pool( remote, pool_info['val']['data_pool'], 64) rgwadmin(ctx, client, cmd=['-n', client, 'zone', 'set', '--rgw-zone', zone], stdin=StringIO(json.dumps(dict( zone_info.items() + user_info.items()))), check_status=True) for region, info in region_info.iteritems(): region_json = json.dumps(info) log.debug('region info is: %s', region_json) rgwadmin(ctx, client, cmd=['-n', client, 'region', 'set'], stdin=StringIO(region_json), check_status=True) if info['is_master']: rgwadmin(ctx, client, cmd=['-n', client, 'region', 'default', '--rgw-region', region], check_status=True) rgwadmin(ctx, client, cmd=['-n', client, 'regionmap', 'update']) yield
def configure_regions_and_zones(ctx, config, regions, role_endpoints, realm): """ Configure regions and zones from rados and rgw. """ if not regions: log.debug( 'In rgw.configure_regions_and_zones() and regions is None. ' 'Bailing') yield return if not realm: log.debug( 'In rgw.configure_regions_and_zones() and realm is None. ' 'Bailing') yield return log.info('Configuring regions and zones...') log.debug('config is %r', config) log.debug('regions are %r', regions) log.debug('role_endpoints = %r', role_endpoints) log.debug('realm is %r', realm) # extract the zone info role_zones = dict([(client, extract_zone_info(ctx, client, c_config)) for client, c_config in config.iteritems()]) log.debug('roles_zones = %r', role_zones) # extract the user info and append it to the payload tuple for the given # client for client, c_config in config.iteritems(): if not c_config: user_info = None else: user_info = extract_user_info(c_config) (region, zone, zone_info) = role_zones[client] role_zones[client] = (region, zone, zone_info, user_info) region_info = dict([ (region_name, extract_region_info(region_name, r_config)) for region_name, r_config in regions.iteritems()]) fill_in_endpoints(region_info, role_zones, role_endpoints) # clear out the old defaults first_mon = teuthology.get_first_mon(ctx, config) (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys() # removing these objects from .rgw.root and the per-zone root pools # may or may not matter rados(ctx, mon, cmd=['-p', '.rgw.root', 'rm', 'region_info.default']) rados(ctx, mon, cmd=['-p', '.rgw.root', 'rm', 'zone_info.default']) # read master zonegroup and master_zone for zonegroup, zg_info in region_info.iteritems(): if zg_info['is_master']: master_zonegroup = zonegroup master_zone = zg_info['master_zone'] break for client in config.iterkeys(): (zonegroup, zone, zone_info, user_info) = role_zones[client] if zonegroup == master_zonegroup and zone == master_zone: master_client = client break log.debug('master zonegroup =%r', master_zonegroup) log.debug('master zone = %r', master_zone) log.debug('master client = %r', master_client) log.debug('config %r ', config) (ret, out)=rgwadmin(ctx, master_client, cmd=['realm', 'create', '--rgw-realm', realm, '--default']) log.debug('realm create ret %r exists %r', -ret, errno.EEXIST) assert ret == 0 or ret != -errno.EEXIST if ret is -errno.EEXIST: log.debug('realm %r exists', realm) for client in config.iterkeys(): for role, (zonegroup, zone, zone_info, user_info) in role_zones.iteritems(): rados(ctx, mon, cmd=['-p', zone_info['domain_root'], 'rm', 'region_info.default']) rados(ctx, mon, cmd=['-p', zone_info['domain_root'], 'rm', 'zone_info.default']) (remote,) = ctx.cluster.only(role).remotes.keys() for pool_info in zone_info['placement_pools']: remote.run(args=['sudo', 'ceph', 'osd', 'pool', 'create', pool_info['val']['index_pool'], '64', '64']) if ctx.rgw.ec_data_pool: create_ec_pool(remote, pool_info['val']['data_pool'], zone, 64, ctx.rgw.erasure_code_profile) else: create_replicated_pool( remote, pool_info['val']['data_pool'], 64) zone_json = json.dumps(dict(zone_info.items() + user_info.items())) log.debug('zone info is: %r', zone_json) rgwadmin(ctx, client, cmd=['zone', 'set', '--rgw-zonegroup', zonegroup, '--rgw-zone', zone], stdin=StringIO(zone_json), check_status=True) for region, info in region_info.iteritems(): region_json = json.dumps(info) log.debug('region info is: %s', region_json) rgwadmin(ctx, client, cmd=['zonegroup', 'set'], stdin=StringIO(region_json), check_status=True) if info['is_master']: rgwadmin(ctx, client, cmd=['zonegroup', 'default', '--rgw-zonegroup', master_zonegroup], check_status=True) (zonegroup, zone, zone_info, user_info) = role_zones[client] rgwadmin(ctx, client, cmd=['zone', 'default', zone], check_status=True) rgwadmin(ctx, master_client, cmd=['-n', master_client, 'period', 'update', '--commit'], check_status=True) yield
def configure_multisite_regions_and_zones(ctx, config, regions, role_endpoints, realm, master_client): """ Configure multisite regions and zones from rados and rgw. """ if not regions: log.debug( 'In rgw.configure_multisite_regions_and_zones() and regions is None. ' 'Bailing') yield return if not realm: log.debug( 'In rgw.configure_multisite_regions_and_zones() and realm is None. ' 'Bailing') yield return log.info('Configuring multisite regions and zones...') log.debug('config is %r', config) log.debug('regions are %r', regions) log.debug('role_endpoints = %r', role_endpoints) log.debug('realm is %r', realm) # extract the zone info role_zones = dict([(client, extract_zone_info(ctx, client, c_config)) for client, c_config in config.iteritems()]) log.debug('role_zones = %r', role_zones) # extract the user info and append it to the payload tuple for the given # client for client, c_config in config.iteritems(): if not c_config: user_info = None else: user_info = extract_user_info(c_config) (region, zone, zone_info) = role_zones[client] role_zones[client] = (region, zone, zone_info, user_info) region_info = dict([ (region_name, extract_region_info(region_name, r_config)) for region_name, r_config in regions.iteritems()]) fill_in_endpoints(region_info, role_zones, role_endpoints) # clear out the old defaults first_mon = teuthology.get_first_mon(ctx, config) (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys() # read master zonegroup and master_zone for zonegroup, zg_info in region_info.iteritems(): if zg_info['is_master']: master_zonegroup = zonegroup master_zone = zg_info['master_zone'] break log.debug('master zonegroup =%r', master_zonegroup) log.debug('master zone = %r', master_zone) log.debug('master client = %r', master_client) rgwadmin(ctx, master_client, cmd=['realm', 'create', '--rgw-realm', realm, '--default'], check_status=True) for region, info in region_info.iteritems(): region_json = json.dumps(info) log.debug('region info is: %s', region_json) rgwadmin(ctx, master_client, cmd=['zonegroup', 'set'], stdin=StringIO(region_json), check_status=True) rgwadmin(ctx, master_client, cmd=['zonegroup', 'default', '--rgw-zonegroup', master_zonegroup], check_status=True) for role, (zonegroup, zone, zone_info, user_info) in role_zones.iteritems(): (remote,) = ctx.cluster.only(role).remotes.keys() for pool_info in zone_info['placement_pools']: remote.run(args=['sudo', 'ceph', 'osd', 'pool', 'create', pool_info['val']['index_pool'], '64', '64']) if ctx.rgw.ec_data_pool: create_ec_pool(remote, pool_info['val']['data_pool'], zone, 64, ctx.rgw.erasure_code_profile) else: create_replicated_pool(remote, pool_info['val']['data_pool'], 64) (zonegroup, zone, zone_info, user_info) = role_zones[master_client] zone_json = json.dumps(dict(zone_info.items() + user_info.items())) log.debug("zone info is: %r", zone_json) rgwadmin(ctx, master_client, cmd=['zone', 'set', '--rgw-zonegroup', zonegroup, '--rgw-zone', zone], stdin=StringIO(zone_json), check_status=True) rgwadmin(ctx, master_client, cmd=['-n', master_client, 'zone', 'default', zone], check_status=True) rgwadmin(ctx, master_client, cmd=['-n', master_client, 'period', 'update', '--commit'], check_status=True) yield
def task(ctx, config): """ Run ceph_objectstore_tool test The config should be as follows:: ceph_objectstore_tool: objects: <number of objects> """ if config is None: config = {} assert isinstance(config, dict), \ 'ceph_objectstore_tool task only accepts a dict for configuration' TEUTHDIR = teuthology.get_testdir(ctx) # clients = config['clients'] # assert len(clients) > 0, # 'ceph_objectstore_tool task needs at least 1 client' REP_POOL = "rep_pool" REP_NAME = "REPobject" # EC_POOL = "ec_pool" # EC_NAME = "ECobject" NUM_OBJECTS = config.get('objects', 10) ERRORS = 0 DATADIR = os.path.join(TEUTHDIR, "data") # Put a test dir below the data dir # TESTDIR = os.path.join(DATADIR, "test") DATALINECOUNT = 10000 # PROFNAME = "testecprofile" log.info('Beginning ceph_objectstore_tool...') log.info("objects: {num}".format(num=NUM_OBJECTS)) log.debug(config) log.debug(ctx) clients = ctx.cluster.only(teuthology.is_type('client')) assert len(clients.remotes) > 0, 'Must specify at least 1 client' (cli_remote, _) = clients.remotes.popitem() log.debug(cli_remote) # clients = dict(teuthology.get_clients(ctx=ctx, roles=config.keys())) # client = clients.popitem() # log.info(client) osds = ctx.cluster.only(teuthology.is_type('osd')) log.info("OSDS") log.info(osds) log.info(osds.remotes) first_mon = teuthology.get_first_mon(ctx, config) (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys() manager = ceph_manager.CephManager( mon, ctx=ctx, config=config, logger=log.getChild('ceph_manager'), ) ctx.manager = manager # ctx.manager.raw_cluster_cmd('osd', 'pool', 'create', REP_POOL, '12', '12', 'replicated') create_replicated_pool(cli_remote, REP_POOL, 12) REPID = get_pool_id(ctx, REP_POOL) log.debug("repid={num}".format(num=REPID)) while len(manager.get_osd_status()['up']) != len(manager.get_osd_status()['raw']): time.sleep(10) while len(manager.get_osd_status()['in']) != len(manager.get_osd_status()['up']): time.sleep(10) manager.raw_cluster_cmd('osd', 'set', 'noout') manager.raw_cluster_cmd('osd', 'set', 'nodown') db = {} LOCALDIR = tempfile.mkdtemp("cod") cod_setup_local_data(log, ctx, NUM_OBJECTS, LOCALDIR, REP_NAME, DATALINECOUNT) allremote = [] allremote.append(cli_remote) allremote += osds.remotes.keys() allremote = list(set(allremote)) for remote in allremote: cod_setup_remote_data(log, ctx, remote, NUM_OBJECTS, DATADIR, REP_NAME, DATALINECOUNT) ERRORS += cod_setup(log, ctx, cli_remote, NUM_OBJECTS, DATADIR, REP_NAME, DATALINECOUNT, REP_POOL, db) pgs = {} jsontext = manager.raw_cluster_cmd('pg', 'dump_json') pgdump = json.loads(jsontext) PGS = [str(p["pgid"]) for p in pgdump["pg_stats"] if p["pgid"].find(str(REPID) + ".") == 0] for stats in pgdump["pg_stats"]: if stats["pgid"] in PGS: for osd in stats["acting"]: if not pgs.has_key(osd): pgs[osd] = [] pgs[osd].append(stats["pgid"]) log.info(pgs) log.info(db) for osd in manager.get_osd_status()['up']: manager.kill_osd(osd) time.sleep(5) pgswithobjects = set() objsinpg = {} # Test --op list and generate json for all objects log.info("Test --op list by generating json for all objects") prefix = "sudo ceph_objectstore_tool --data-path {fpath} --journal-path {jpath} ".format(fpath=FSPATH, jpath=JPATH) for remote in osds.remotes.iterkeys(): log.debug(remote) log.debug(osds.remotes[remote]) for role in osds.remotes[remote]: if string.find(role, "osd.") != 0: continue osdid = int(role.split('.')[1]) log.info("process osd.{id} on {remote}".format(id=osdid, remote=remote)) for pg in pgs[osdid]: cmd = (prefix + "--op list --pgid {pg}").format(id=osdid, pg=pg) proc = remote.run(args=cmd.split(), check_status=False, stdout=StringIO()) # proc.wait() if proc.exitstatus != 0: log.error("Bad exit status {ret} from --op list request".format(ret=proc.exitstatus)) ERRORS += 1 else: data = proc.stdout.getvalue() if len(data): # This pg has some objects in it pgswithobjects.add(pg) pglines = data.split('\n') # All copies of a pg are the same so we can overwrite objsinpg[pg] = [] while(len(pglines)): # Drop any blank lines if (len(pglines[-1]) == 0): pglines.pop() continue objjson = pglines.pop() name = json.loads(objjson)['oid'] objsinpg[pg].append(name) db[name]["pgid"] = pg db[name]["json"] = objjson log.info(db) log.info(pgswithobjects) log.info(objsinpg) # Test get-bytes log.info("Test get-bytes and set-bytes") for basename in db.keys(): file = os.path.join(DATADIR, basename) JSON = db[basename]["json"] GETNAME = os.path.join(DATADIR, "get") SETNAME = os.path.join(DATADIR, "set") for remote in osds.remotes.iterkeys(): for role in osds.remotes[remote]: if string.find(role, "osd.") != 0: continue osdid = int(role.split('.')[1]) pg = db[basename]['pgid'] if pg in pgs[osdid]: cmd = (prefix + "--pgid {pg}").format(id=osdid, pg=pg).split() cmd.append(run.Raw("'{json}'".format(json=JSON))) cmd += "get-bytes {fname}".format(fname=GETNAME).split() proc = remote.run(args=cmd, check_status=False) if proc.exitstatus != 0: remote.run(args="rm -f {getfile}".format(getfile=GETNAME).split()) log.error("Bad exit status {ret}".format(ret=proc.exitstatus)) ERRORS += 1 continue cmd = "diff -q {file} {getfile}".format(file=file, getfile=GETNAME) proc = remote.run(args=cmd.split()) if proc.exitstatus != 0: log.error("Data from get-bytes differ") # log.debug("Got:") # cat_file(logging.DEBUG, GETNAME) # log.debug("Expected:") # cat_file(logging.DEBUG, file) ERRORS += 1 remote.run(args="rm -f {getfile}".format(getfile=GETNAME).split()) data = "put-bytes going into {file}\n".format(file=file) teuthology.write_file(remote, SETNAME, data) cmd = (prefix + "--pgid {pg}").format(id=osdid, pg=pg).split() cmd.append(run.Raw("'{json}'".format(json=JSON))) cmd += "set-bytes {fname}".format(fname=SETNAME).split() proc = remote.run(args=cmd, check_status=False) proc.wait() if proc.exitstatus != 0: log.info("set-bytes failed for object {obj} in pg {pg} osd.{id} ret={ret}".format(obj=basename, pg=pg, id=osdid, ret=proc.exitstatus)) ERRORS += 1 cmd = (prefix + "--pgid {pg}").format(id=osdid, pg=pg).split() cmd.append(run.Raw("'{json}'".format(json=JSON))) cmd += "get-bytes -".split() proc = remote.run(args=cmd, check_status=False, stdout=StringIO()) proc.wait() if proc.exitstatus != 0: log.error("get-bytes after set-bytes ret={ret}".format(ret=proc.exitstatus)) ERRORS += 1 else: if data != proc.stdout.getvalue(): log.error("Data inconsistent after set-bytes, got:") log.error(proc.stdout.getvalue()) ERRORS += 1 cmd = (prefix + "--pgid {pg}").format(id=osdid, pg=pg).split() cmd.append(run.Raw("'{json}'".format(json=JSON))) cmd += "set-bytes {fname}".format(fname=file).split() proc = remote.run(args=cmd, check_status=False) proc.wait() if proc.exitstatus != 0: log.info("set-bytes failed for object {obj} in pg {pg} osd.{id} ret={ret}".format(obj=basename, pg=pg, id=osdid, ret=proc.exitstatus)) ERRORS += 1 log.info("Test list-attrs get-attr") for basename in db.keys(): file = os.path.join(DATADIR, basename) JSON = db[basename]["json"] GETNAME = os.path.join(DATADIR, "get") SETNAME = os.path.join(DATADIR, "set") for remote in osds.remotes.iterkeys(): for role in osds.remotes[remote]: if string.find(role, "osd.") != 0: continue osdid = int(role.split('.')[1]) pg = db[basename]['pgid'] if pg in pgs[osdid]: cmd = (prefix + "--pgid {pg}").format(id=osdid, pg=pg).split() cmd.append(run.Raw("'{json}'".format(json=JSON))) cmd += ["list-attrs"] proc = remote.run(args=cmd, check_status=False, stdout=StringIO(), stderr=StringIO()) proc.wait() if proc.exitstatus != 0: log.error("Bad exit status {ret}".format(ret=proc.exitstatus)) ERRORS += 1 continue keys = proc.stdout.getvalue().split() values = dict(db[basename]["xattr"]) for key in keys: if key == "_" or key == "snapset": continue key = key.strip("_") if key not in values: log.error("The key {key} should be present".format(key=key)) ERRORS += 1 continue exp = values.pop(key) cmd = (prefix + "--pgid {pg}").format(id=osdid, pg=pg).split() cmd.append(run.Raw("'{json}'".format(json=JSON))) cmd += "get-attr {key}".format(key="_" + key).split() proc = remote.run(args=cmd, check_status=False, stdout=StringIO()) proc.wait() if proc.exitstatus != 0: log.error("get-attr failed with {ret}".format(ret=proc.exitstatus)) ERRORS += 1 continue val = proc.stdout.getvalue() if exp != val: log.error("For key {key} got value {got} instead of {expected}".format(key=key, got=val, expected=exp)) ERRORS += 1 if len(values) != 0: log.error("Not all keys found, remaining keys:") log.error(values) log.info("Test pg info") for remote in osds.remotes.iterkeys(): for role in osds.remotes[remote]: if string.find(role, "osd.") != 0: continue osdid = int(role.split('.')[1]) for pg in pgs[osdid]: cmd = (prefix + "--op info --pgid {pg}").format(id=osdid, pg=pg).split() proc = remote.run(args=cmd, check_status=False, stdout=StringIO()) proc.wait() if proc.exitstatus != 0: log.error("Failure of --op info command with {ret}".format(proc.exitstatus)) ERRORS += 1 continue info = proc.stdout.getvalue() if not str(pg) in info: log.error("Bad data from info: {info}".format(info=info)) ERRORS += 1 log.info("Test pg logging") for remote in osds.remotes.iterkeys(): for role in osds.remotes[remote]: if string.find(role, "osd.") != 0: continue osdid = int(role.split('.')[1]) for pg in pgs[osdid]: cmd = (prefix + "--op log --pgid {pg}").format(id=osdid, pg=pg).split() proc = remote.run(args=cmd, check_status=False, stdout=StringIO()) proc.wait() if proc.exitstatus != 0: log.error("Getting log failed for pg {pg} from osd.{id} with {ret}".format(pg=pg, id=osdid, ret=proc.exitstatus)) ERRORS += 1 continue HASOBJ = pg in pgswithobjects MODOBJ = "modify" in proc.stdout.getvalue() if HASOBJ != MODOBJ: log.error("Bad log for pg {pg} from osd.{id}".format(pg=pg, id=osdid)) MSG = (HASOBJ and [""] or ["NOT "])[0] log.error("Log should {msg}have a modify entry".format(msg=MSG)) ERRORS += 1 log.info("Test pg export") EXP_ERRORS = 0 for remote in osds.remotes.iterkeys(): for role in osds.remotes[remote]: if string.find(role, "osd.") != 0: continue osdid = int(role.split('.')[1]) for pg in pgs[osdid]: fpath = os.path.join(DATADIR, "osd{id}.{pg}".format(id=osdid, pg=pg)) cmd = (prefix + "--op export --pgid {pg} --file {file}").format(id=osdid, pg=pg, file=fpath) proc = remote.run(args=cmd, check_status=False, stdout=StringIO()) proc.wait() if proc.exitstatus != 0: log.error("Exporting failed for pg {pg} on osd.{id} with {ret}".format(pg=pg, id=osdid, ret=proc.exitstatus)) EXP_ERRORS += 1 ERRORS += EXP_ERRORS log.info("Test pg removal") RM_ERRORS = 0 for remote in osds.remotes.iterkeys(): for role in osds.remotes[remote]: if string.find(role, "osd.") != 0: continue osdid = int(role.split('.')[1]) for pg in pgs[osdid]: cmd = (prefix + "--op remove --pgid {pg}").format(pg=pg, id=osdid) proc = remote.run(args=cmd, check_status=False, stdout=StringIO()) proc.wait() if proc.exitstatus != 0: log.error("Removing failed for pg {pg} on osd.{id} with {ret}".format(pg=pg, id=osdid, ret=proc.exitstatus)) RM_ERRORS += 1 ERRORS += RM_ERRORS IMP_ERRORS = 0 if EXP_ERRORS == 0 and RM_ERRORS == 0: log.info("Test pg import") for remote in osds.remotes.iterkeys(): for role in osds.remotes[remote]: if string.find(role, "osd.") != 0: continue osdid = int(role.split('.')[1]) for pg in pgs[osdid]: fpath = os.path.join(DATADIR, "osd{id}.{pg}".format(id=osdid, pg=pg)) cmd = (prefix + "--op import --file {file}").format(id=osdid, file=fpath) proc = remote.run(args=cmd, check_status=False, stdout=StringIO()) proc.wait() if proc.exitstatus != 0: log.error("Import failed from {file} with {ret}".format(file=fpath, ret=proc.exitstatus)) IMP_ERRORS += 1 else: log.warning("SKIPPING IMPORT TESTS DUE TO PREVIOUS FAILURES") ERRORS += IMP_ERRORS if EXP_ERRORS == 0 and RM_ERRORS == 0 and IMP_ERRORS == 0: log.info("Restarting OSDs....") # They are still look to be up because of setting nodown for osd in manager.get_osd_status()['up']: manager.revive_osd(osd) # Wait for health? time.sleep(5) # Let scrub after test runs verify consistency of all copies log.info("Verify replicated import data") objects = range(1, NUM_OBJECTS + 1) for i in objects: NAME = REP_NAME + "{num}".format(num=i) TESTNAME = os.path.join(DATADIR, "gettest") REFNAME = os.path.join(DATADIR, NAME) proc = rados(ctx, cli_remote, ['-p', REP_POOL, 'get', NAME, TESTNAME], wait=False) ret = proc.wait() if ret != 0: log.errors("After import, rados get failed with {ret}".format(ret=r[0].exitstatus)) ERRORS += 1 continue cmd = "diff -q {gettest} {ref}".format(gettest=TESTNAME, ref=REFNAME) proc = cli_remote.run(args=cmd, check_status=False) proc.wait() if proc.exitstatus != 0: log.error("Data comparison failed for {obj}".format(obj=NAME)) ERRORS += 1 if ERRORS == 0: log.info("TEST PASSED") else: log.error("TEST FAILED WITH {errcount} ERRORS".format(errcount=ERRORS)) try: yield finally: log.info('Ending ceph_objectstore_tool')
def configure_multisite_regions_and_zones(ctx, config, regions, role_endpoints, realm, master_client): """ Configure multisite regions and zones from rados and rgw. """ if not regions: log.debug( 'In rgw.configure_multisite_regions_and_zones() and regions is None. ' 'Bailing') yield return if not realm: log.debug( 'In rgw.configure_multisite_regions_and_zones() and realm is None. ' 'Bailing') yield return log.info('Configuring multisite regions and zones...') log.debug('config is %r', config) log.debug('regions are %r', regions) log.debug('role_endpoints = %r', role_endpoints) log.debug('realm is %r', realm) # extract the zone info role_zones = dict([(client, extract_zone_info(ctx, client, c_config)) for client, c_config in config.iteritems()]) log.debug('role_zones = %r', role_zones) # extract the user info and append it to the payload tuple for the given # client for client, c_config in config.iteritems(): if not c_config: user_info = None else: user_info = extract_user_info(c_config) (region, zone, zone_info) = role_zones[client] role_zones[client] = (region, zone, zone_info, user_info) region_info = dict([(region_name, extract_region_info(region_name, r_config)) for region_name, r_config in regions.iteritems()]) fill_in_endpoints(region_info, role_zones, role_endpoints) # clear out the old defaults cluster_name, daemon_type, client_id = teuthology.split_role(master_client) first_mon = teuthology.get_first_mon(ctx, config, cluster_name) (mon, ) = ctx.cluster.only(first_mon).remotes.iterkeys() # read master zonegroup and master_zone for zonegroup, zg_info in region_info.iteritems(): if zg_info['is_master']: master_zonegroup = zonegroup master_zone = zg_info['master_zone'] break log.debug('master zonegroup =%r', master_zonegroup) log.debug('master zone = %r', master_zone) log.debug('master client = %r', master_client) rgwadmin(ctx, master_client, cmd=['realm', 'create', '--rgw-realm', realm, '--default'], check_status=True) for region, info in region_info.iteritems(): region_json = json.dumps(info) log.debug('region info is: %s', region_json) rgwadmin(ctx, master_client, cmd=['zonegroup', 'set'], stdin=StringIO(region_json), check_status=True) rgwadmin(ctx, master_client, cmd=['zonegroup', 'default', '--rgw-zonegroup', master_zonegroup], check_status=True) for role, (zonegroup, zone, zone_info, user_info) in role_zones.iteritems(): (remote, ) = ctx.cluster.only(role).remotes.keys() for pool_info in zone_info['placement_pools']: remote.run(args=[ 'sudo', 'ceph', 'osd', 'pool', 'create', pool_info['val'] ['index_pool'], '64', '64', '--cluster', cluster_name ]) if ctx.rgw.ec_data_pool: create_ec_pool(remote, pool_info['val']['data_pool'], zone, 64, ctx.rgw.erasure_code_profile, cluster_name) else: create_replicated_pool(remote, pool_info['val']['data_pool'], 64, cluster_name) (zonegroup, zone, zone_info, user_info) = role_zones[master_client] zone_json = json.dumps(dict(zone_info.items() + user_info.items())) log.debug("zone info is: %r", zone_json) rgwadmin( ctx, master_client, cmd=['zone', 'set', '--rgw-zonegroup', zonegroup, '--rgw-zone', zone], stdin=StringIO(zone_json), check_status=True) rgwadmin(ctx, master_client, cmd=['zone', 'default', '--rgw-zone', zone], check_status=True) rgwadmin(ctx, master_client, cmd=['period', 'update', '--commit'], check_status=True) yield
def configure_regions_and_zones(ctx, config, regions, role_endpoints, realm): """ Configure regions and zones from rados and rgw. """ if not regions: log.debug( 'In rgw.configure_regions_and_zones() and regions is None. ' 'Bailing') configure_compression_in_default_zone(ctx, config) yield return if not realm: log.debug( 'In rgw.configure_regions_and_zones() and realm is None. ' 'Bailing') configure_compression_in_default_zone(ctx, config) yield return log.info('Configuring regions and zones...') log.debug('config is %r', config) log.debug('regions are %r', regions) log.debug('role_endpoints = %r', role_endpoints) log.debug('realm is %r', realm) # extract the zone info role_zones = dict([(client, extract_zone_info(ctx, client, c_config)) for client, c_config in config.iteritems()]) log.debug('roles_zones = %r', role_zones) # extract the user info and append it to the payload tuple for the given # client for client, c_config in config.iteritems(): if not c_config: user_info = None else: user_info = extract_user_info(c_config) (region, zone, zone_info) = role_zones[client] role_zones[client] = (region, zone, zone_info, user_info) region_info = dict([ (region_name, extract_region_info(region_name, r_config)) for region_name, r_config in regions.iteritems()]) fill_in_endpoints(region_info, role_zones, role_endpoints) # clear out the old defaults cluster_name, daemon_type, client_id = teuthology.split_role(client) first_mon = teuthology.get_first_mon(ctx, config, cluster_name) (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys() # removing these objects from .rgw.root and the per-zone root pools # may or may not matter rados(ctx, mon, cmd=['-p', '.rgw.root', 'rm', 'region_info.default', '--cluster', cluster_name]) rados(ctx, mon, cmd=['-p', '.rgw.root', 'rm', 'zone_info.default', '--cluster', cluster_name]) # read master zonegroup and master_zone for zonegroup, zg_info in region_info.iteritems(): if zg_info['is_master']: master_zonegroup = zonegroup master_zone = zg_info['master_zone'] break for client in config.iterkeys(): (zonegroup, zone, zone_info, user_info) = role_zones[client] if zonegroup == master_zonegroup and zone == master_zone: master_client = client break log.debug('master zonegroup =%r', master_zonegroup) log.debug('master zone = %r', master_zone) log.debug('master client = %r', master_client) log.debug('config %r ', config) (ret, out)=rgwadmin(ctx, client, cmd=['realm', 'create', '--rgw-realm', realm, '--default']) log.debug('realm create ret %r exists %r', -ret, errno.EEXIST) assert ret == 0 or ret != -errno.EEXIST if ret is -errno.EEXIST: log.debug('realm %r exists', realm) for client in config.iterkeys(): for role, (zonegroup, zone, zone_info, user_info) in role_zones.iteritems(): rados(ctx, mon, cmd=['-p', zone_info['domain_root'], 'rm', 'region_info.default', '--cluster', cluster_name]) rados(ctx, mon, cmd=['-p', zone_info['domain_root'], 'rm', 'zone_info.default', '--cluster', cluster_name]) (remote,) = ctx.cluster.only(role).remotes.keys() for pool_info in zone_info['placement_pools']: remote.run(args=['sudo', 'ceph', 'osd', 'pool', 'create', pool_info['val']['index_pool'], '64', '64', '--cluster', cluster_name]) if ctx.rgw.ec_data_pool: create_ec_pool(remote, pool_info['val']['data_pool'], zone, 64, ctx.rgw.erasure_code_profile, cluster_name) else: create_replicated_pool( remote, pool_info['val']['data_pool'], 64, cluster_name) zone_json = json.dumps(dict(zone_info.items() + user_info.items())) log.debug('zone info is: %r', zone_json) rgwadmin(ctx, client, cmd=['zone', 'set', '--rgw-zonegroup', zonegroup, '--rgw-zone', zone], stdin=StringIO(zone_json), check_status=True) for region, info in region_info.iteritems(): region_json = json.dumps(info) log.debug('region info is: %s', region_json) rgwadmin(ctx, client, cmd=['zonegroup', 'set'], stdin=StringIO(region_json), check_status=True) if info['is_master']: rgwadmin(ctx, client, cmd=['zonegroup', 'default', '--rgw-zonegroup', master_zonegroup], check_status=True) (zonegroup, zone, zone_info, user_info) = role_zones[client] rgwadmin(ctx, client, cmd=['zone', 'default', '--rgw-zone', zone], check_status=True) #this used to take master_client, need to edit that accordingly rgwadmin(ctx, client, cmd=['period', 'update', '--commit'], check_status=True) yield