def process(self):
"""Process all keystone accounts to sync."""
orig_auth_url = get_config('auth', 'keystone_origin')
orig_admin_tenant, orig_admin_user, orig_admin_password = (get_config(
'auth', 'keystone_origin_admin_credentials').split(':'))
oa_st_url, orig_admin_token = self.get_swift_auth(
orig_auth_url, orig_admin_tenant, orig_admin_user,
orig_admin_password)
dest_auth_url = get_config('auth', 'keystone_dest')
# we assume orig and dest passwd are the same obv synchronized.
dst_st_url, dest_admin_token = self.get_swift_auth(
dest_auth_url, orig_admin_tenant, orig_admin_user,
orig_admin_password)
bare_oa_st_url = oa_st_url[:oa_st_url.find('AUTH_')] + "AUTH_"
bare_dst_st_url = dst_st_url[:dst_st_url.find('AUTH_')] + "AUTH_"
self.keystone_cnx = self.get_ks_auth_orig()
# if user has defined target tenants, limit the migration
# to them
_targets_filters = self.get_target_tenant_filter()
if _targets_filters is not None:
_targets = (tenant for tenant in self.keystone_cnx.tenants.list()
if tenant.name in _targets_filters)
else:
_targets = self.keystone_cnx.tenants.list()
for tenant in _targets:
user_orig_st_url = bare_oa_st_url + tenant.id
user_dst_st_url = bare_dst_st_url + tenant.id
self.sync_account(user_orig_st_url, orig_admin_token,
user_dst_st_url, dest_admin_token)
def swift_cnx(acc, user):
ks_url = get_config('auth', 'keystone_origin')
cnx = sclient.Connection(ks_url,
user=user,
key=get_config('filler', 'default_user_password'),
tenant_name=acc[0],
auth_version=2)
return cnx
def swift_cnx(acc, user):
ks_url = get_config('auth', 'keystone_origin')
cnx = sclient.Connection(ks_url,
user=user,
key=get_config('filler', 'default_user_password'),
tenant_name=acc[0],
auth_version=2)
return cnx
def get_ks_auth_orig(self):
"""Get keystone cnx from config."""
orig_auth_url = get_config('auth', 'keystone_origin')
cfg = get_config('auth', 'keystone_origin_admin_credentials')
(tenant_name, username, password) = cfg.split(':')
return keystoneclient.v2_0.client.Client(auth_url=orig_auth_url,
username=username,
password=password,
tenant_name=tenant_name)
def get_ks_auth_orig(self):
"""Get keystone cnx from config."""
orig_auth_url = get_config('auth', 'keystone_origin')
cfg = get_config('auth', 'keystone_origin_admin_credentials')
(tenant_name, username, password) = cfg.split(':')
return keystoneclient.v2_0.client.Client(auth_url=orig_auth_url,
username=username,
password=password,
tenant_name=tenant_name)
def test_fill_swift(self):
self.cont_cnt = 0
self.obj_cnt = 0
return_dict_ref = {}
def create_objects(*args, **kwargs):
self.obj_cnt += 1
def create_containers(*args, **kwargs):
self.cont_cnt += 1
def swift_cnx(*args, **kargs):
return self.get_connection()
self.stubs.Set(filler, 'swift_cnx', swift_cnx)
self.stubs.Set(filler, 'create_objects', create_objects)
self.stubs.Set(filler, 'create_containers', create_containers)
concurrency = int(
utils.get_config('concurrency', 'filler_swift_client_concurrency'))
pool = eventlet.GreenPool(concurrency)
created = {
('account1', 'account1_id'): ['test', 'test_id', 'role_id'],
('account2', 'account2_id'): ['test', 'test_id', 'role_id']
}
filler.fill_swift(pool, created, 1, 1, 2048, return_dict_ref)
self.assertEqual(self.cont_cnt, 2)
self.assertEqual(self.obj_cnt, 2)
def test_create_swift_account_fail(self):
self.ret_index = {}
self.pa = 0
def create_tenant(*args):
if self.pa == 0:
self.pa += 1
raise KSClientException('Fake msg')
else:
self.pa += 1
return FakeKSTenant('foo1')
def create_swift_user(*args):
pass
client = FakeKSClient()
self.stubs.Set(client.tenants, 'create', create_tenant)
self.stubs.Set(filler, 'create_swift_user', create_swift_user)
concurrency = int(
utils.get_config('concurrency',
'filler_keystone_client_concurrency'))
pile = eventlet.GreenPile(concurrency)
filler.create_swift_account(client, pile, 3, 1, self.ret_index)
self.assertEqual(len(self.ret_index.keys()), 2)
def test_fill_swift(self):
self.cont_cnt = 0
self.obj_cnt = 0
return_dict_ref = {}
def create_objects(*args, **kwargs):
self.obj_cnt += 1
def create_containers(*args, **kwargs):
self.cont_cnt += 1
def swift_cnx(*args, **kargs):
return self.get_connection()
self.stubs.Set(filler, 'swift_cnx', swift_cnx)
self.stubs.Set(filler, 'create_objects', create_objects)
self.stubs.Set(filler, 'create_containers', create_containers)
concurrency = int(utils.get_config('concurrency',
'filler_swift_client_concurrency'))
pool = eventlet.GreenPool(concurrency)
created = {('account1', 'account1_id'): ['test', 'test_id', 'role_id'],
('account2', 'account2_id'): ['test', 'test_id', 'role_id']}
filler.fill_swift(pool, created, 1, 1, 2048, return_dict_ref)
self.assertEqual(self.cont_cnt, 2)
self.assertEqual(self.obj_cnt, 2)
def test_create_swift_account_fail(self):
self.ret_index = {}
self.pa = 0
def create_tenant(*args):
if self.pa == 0:
self.pa += 1
raise KSClientException('Fake msg')
else:
self.pa += 1
return FakeKSTenant('foo1')
def create_swift_user(*args):
pass
client = FakeKSClient()
self.stubs.Set(client.tenants, 'create', create_tenant)
self.stubs.Set(filler, 'create_swift_user', create_swift_user)
concurrency = int(utils.get_config('concurrency',
'filler_keystone_client_concurrency'))
pile = eventlet.GreenPile(concurrency)
filler.create_swift_account(client, pile, 3, 1, self.ret_index)
self.assertEqual(len(self.ret_index.keys()), 2)
def _create_user(account_name, account_id):
user = get_rand_str(mode='user_')
# Create a user in that tenant
uid = client.users.create(user,
get_config('filler',
'default_user_password'),
get_config('filler', 'default_user_email'),
account_id)
# Get swift_operator_role id
roleid = [role.id for role in client.roles.list()
if role.name == get_config('filler', 'swift_operator_role')]
if not roleid:
logging.error('Could not find swift_operator_role %s in keystone' %
get_config('filler', 'swift_operator_role'))
sys.exit(1)
roleid = roleid[0]
# Add tenant/user in swift operator role/group
client.roles.add_user_role(uid.id, roleid, account_id)
return (user, uid.id, roleid)
def _create_user(account_name, account_id):
user = get_rand_str(mode='user_')
# Create a user in that tenant
uid = client.users.create(
user, get_config('filler', 'default_user_password'),
get_config('filler', 'default_user_email'), account_id)
# Get swift_operator_role id
roleid = [
role.id for role in client.roles.list()
if role.name == get_config('filler', 'swift_operator_role')
]
if not roleid:
logging.error('Could not find swift_operator_role %s in keystone' %
get_config('filler', 'swift_operator_role'))
sys.exit(1)
roleid = roleid[0]
# Add tenant/user in swift operator role/group
client.roles.add_user_role(uid.id, roleid, account_id)
return (user, uid.id, roleid)
def load_containers_index():
index_containers_path = get_config('filler', 'index_containers_path')
if os.path.isfile(index_containers_path):
try:
index = pickle.load(file(index_containers_path))
logging.info("Load previous index for %s" % index_containers_path)
except Exception:
index = {}
else:
index = {}
return index
def load_containers_index():
index_containers_path = get_config('filler', 'index_containers_path')
if os.path.isfile(index_containers_path):
try:
index = pickle.load(file(index_containers_path))
logging.info("Load previous index for %s" % index_containers_path)
except Exception:
index = {}
else:
index = {}
return index
def process(self):
"""Process all keystone accounts to sync."""
orig_auth_url = get_config('auth', 'keystone_origin')
orig_admin_tenant, orig_admin_user, orig_admin_password = (
get_config('auth', 'keystone_origin_admin_credentials').split(':'))
oa_st_url, orig_admin_token = self.get_swift_auth(
orig_auth_url, orig_admin_tenant,
orig_admin_user, orig_admin_password)
dest_auth_url = get_config('auth', 'keystone_dest')
# we assume orig and dest passwd are the same obv synchronized.
dst_st_url, dest_admin_token = self.get_swift_auth(
dest_auth_url, orig_admin_tenant,
orig_admin_user, orig_admin_password)
bare_oa_st_url = oa_st_url[:oa_st_url.find('AUTH_')] + "AUTH_"
bare_dst_st_url = dst_st_url[:dst_st_url.find('AUTH_')] + "AUTH_"
self.keystone_cnx = self.get_ks_auth_orig()
# if user has defined target tenants, limit the migration
# to them
_targets_filters = self.get_target_tenant_filter()
if _targets_filters is not None:
_targets = (tenant for tenant in self.keystone_cnx.tenants.list()
if tenant.name in _targets_filters)
else:
_targets = self.keystone_cnx.tenants.list()
for tenant in _targets:
user_orig_st_url = bare_oa_st_url + tenant.id
user_dst_st_url = bare_dst_st_url + tenant.id
self.sync_account(user_orig_st_url,
orig_admin_token,
user_dst_st_url,
dest_admin_token)
def get_target_tenant_filter(self):
"""Returns a set of target tenants from the tenant_list_file.
tenant_list_file is defined in the config file or given as a command
line argument.
If tenant_list_file is not defined, returns None (an empty filter).
"""
try:
tenant_filter_filename = get_config('sync', 'tenant_filter_file')
with open(tenant_filter_filename) as tenantsfile:
return {name.strip() for name in tenantsfile.readlines()}
except ConfigurationError:
return None
def get_target_tenant_filter(self):
"""Returns a set of target tenants from the tenant_list_file.
tenant_list_file is defined in the config file or given as a command
line argument.
If tenant_list_file is not defined, returns None (an empty filter).
"""
try:
tenant_filter_filename = get_config('sync', 'tenant_filter_file')
with open(tenant_filter_filename) as tenantsfile:
return {name.strip() for name in tenantsfile.readlines()}
except ConfigurationError:
return None
def test_create_swift_account(self):
self.ret_index = {}
self.user_cnt = 0
def create_swift_user(*args):
self.user_cnt += 1
self.stubs.Set(filler, 'create_swift_user', create_swift_user)
concurrency = int(utils.get_config('concurrency',
'filler_keystone_client_concurrency'))
pile = eventlet.GreenPile(concurrency)
client = FakeKSClient()
filler.create_swift_account(client, pile, 1, 1, self.ret_index)
self.assertEqual(self.user_cnt, 1)
self.assertEqual(len(self.ret_index.keys()), 1)
def test_create_swift_account(self):
self.ret_index = {}
self.user_cnt = 0
def create_swift_user(*args):
self.user_cnt += 1
self.stubs.Set(filler, 'create_swift_user', create_swift_user)
concurrency = int(
utils.get_config('concurrency',
'filler_keystone_client_concurrency'))
pile = eventlet.GreenPile(concurrency)
client = FakeKSClient()
filler.create_swift_account(client, pile, 1, 1, self.ret_index)
self.assertEqual(self.user_cnt, 1)
self.assertEqual(len(self.ret_index.keys()), 1)
def setUp(self):
self.o_st = get_config('auth', 'keystone_origin')
self.d_st = get_config('auth', 'keystone_dest')
self.default_user_password = get_config('filler',
'default_user_password')
# Retreive configuration for filler
self.o_admin_tenant, self.o_admin_user, self.o_admin_password = (
get_config('auth', 'keystone_origin_admin_credentials').split(':'))
self.sw_c_concu = int(get_config('concurrency',
'filler_swift_client_concurrency'))
self.ks_c_concu = int(get_config('concurrency',
'filler_keystone_client_concurrency'))
self.filter_filename = get_config('sync', 'tenant_filter_file')
self.pile = eventlet.GreenPile(self.sw_c_concu)
self.pool = eventlet.GreenPool(self.ks_c_concu)
# Set a keystone connection to origin server
self.o_ks_client = ksclient.Client(
auth_url=self.o_st,
username=self.o_admin_user,
password=self.o_admin_password,
tenant_name=self.o_admin_tenant)
# Set a keystone connection to destination server
self.d_ks_client = ksclient.Client(
auth_url=self.d_st,
username=self.o_admin_user,
password=self.o_admin_password,
tenant_name=self.o_admin_tenant)
# Retreive admin (ResellerAdmin) token
(self.o_admin_auth_url, self.o_admin_token) = (
sclient.Connection(self.o_st,
"%s:%s" % (self.o_admin_tenant,
self.o_admin_user),
self.o_admin_password,
auth_version=2).get_auth())
# Retreive admin (ResellerAdmin) token
(self.d_admin_auth_url, self.d_admin_token) = (
sclient.Connection(self.d_st,
"%s:%s" % (self.o_admin_tenant,
self.o_admin_user),
self.o_admin_password,
auth_version=2).get_auth())
# Instanciate syncer
self.swsync = accounts.Accounts()
def test_create_swift_user(self):
self.create_cnt = 0
self.role_cnt = 0
def create(*args, **kargs):
self.create_cnt += 1
return FakeKSUser()
def add_user_role(*args, **kargs):
self.role_cnt += 1
co = utils.get_config('auth',
'keystone_origin_admin_credentials').split(':')
tenant_name, username, password = co
client = FakeKSClient()
client.roles.add_user_role = add_user_role
client.users.create = create
filler.create_swift_user(client, 'account1', 'account1_id', 1)
self.assertEqual(self.create_cnt, 1)
self.assertEqual(self.role_cnt, 1)
def test_create_swift_user(self):
self.create_cnt = 0
self.role_cnt = 0
def create(*args, **kargs):
self.create_cnt += 1
return FakeKSUser()
def add_user_role(*args, **kargs):
self.role_cnt += 1
co = utils.get_config('auth',
'keystone_origin_admin_credentials').split(':')
tenant_name, username, password = co
client = FakeKSClient()
client.roles.add_user_role = add_user_role
client.users.create = create
filler.create_swift_user(client, 'account1', 'account1_id', 1)
self.assertEqual(self.create_cnt, 1)
self.assertEqual(self.role_cnt, 1)
def test_create_swift_user_fail(self):
self.pa = 0
def create(*args, **kargs):
if self.pa == 0:
self.pa += 1
raise KSClientException('Fake msg')
else:
self.pa += 1
return FakeKSUser()
def add_user_role(*args, **kargs):
pass
co = utils.get_config('auth',
'keystone_origin_admin_credentials').split(':')
tenant_name, username, password = co
client = FakeKSClient()
client.roles.add_user_role = add_user_role
client.users.create = create
users = filler.create_swift_user(client, 'account1', 'account1_id', 3)
self.assertEqual(len(users), 2)
def test_create_swift_user_fail(self):
self.pa = 0
def create(*args, **kargs):
if self.pa == 0:
self.pa += 1
raise KSClientException('Fake msg')
else:
self.pa += 1
return FakeKSUser()
def add_user_role(*args, **kargs):
pass
co = utils.get_config('auth',
'keystone_origin_admin_credentials').split(':')
tenant_name, username, password = co
client = FakeKSClient()
client.roles.add_user_role = add_user_role
client.users.create = create
users = filler.create_swift_user(client, 'account1', 'account1_id', 3)
self.assertEqual(len(users), 2)
def setUp(self):
self.o_st = get_config('auth', 'keystone_origin')
self.d_st = get_config('auth', 'keystone_dest')
self.default_user_password = get_config('filler',
'default_user_password')
# Retreive configuration for filler
self.o_admin_tenant, self.o_admin_user, self.o_admin_password = (
get_config('auth', 'keystone_origin_admin_credentials').split(':'))
self.sw_c_concu = int(
get_config('concurrency', 'filler_swift_client_concurrency'))
self.ks_c_concu = int(
get_config('concurrency', 'filler_keystone_client_concurrency'))
self.filter_filename = get_config('sync', 'tenant_filter_file')
self.pile = eventlet.GreenPile(self.sw_c_concu)
self.pool = eventlet.GreenPool(self.ks_c_concu)
# Set a keystone connection to origin server
self.o_ks_client = ksclient.Client(auth_url=self.o_st,
username=self.o_admin_user,
password=self.o_admin_password,
tenant_name=self.o_admin_tenant)
# Set a keystone connection to destination server
self.d_ks_client = ksclient.Client(auth_url=self.d_st,
username=self.o_admin_user,
password=self.o_admin_password,
tenant_name=self.o_admin_tenant)
# Retreive admin (ResellerAdmin) token
(self.o_admin_auth_url, self.o_admin_token) = (sclient.Connection(
self.o_st,
"%s:%s" % (self.o_admin_tenant, self.o_admin_user),
self.o_admin_password,
auth_version=2).get_auth())
# Retreive admin (ResellerAdmin) token
(self.d_admin_auth_url, self.d_admin_token) = (sclient.Connection(
self.d_st,
"%s:%s" % (self.o_admin_tenant, self.o_admin_user),
self.o_admin_password,
auth_version=2).get_auth())
# Instanciate syncer
self.swsync = accounts.Accounts()
def get_connection(self, *args):
return swiftclient.client.Connection(utils.get_config(
'auth', 'keystone_origin'),
'test',
'password',
tenant_name='test')
def get_connection(self, *args):
return swiftclient.client.Connection(utils.get_config(
'auth', 'keystone_origin'),
'test', 'password',
tenant_name='test')
