def extend_account_capacity(project_name, capacity, domain):
'''
系统扩容，这里通过系统编码查询对应的account然后对account进行扩容
:param project_name:
:param capacity:
:param domain:
:return:
'''
status = ''
message = ''
resp = {"status": status, "message": message}
try:
ainfo = db.session.query(SfoAccountManager).filter(
and_(SfoAccountManager.project_name == project_name,
SfoAccountManager.account_stat == '1',
SfoAccountManager.domain == domain)).first()
admin_user = SfoCofigureMethod.query_value_from_con_key('KEYSTONE_USERNAME')
admin_password = SfoCofigureMethod.query_value_from_con_key('KEYSTONE_PASSWORD')
if ainfo:
# 设置account容量
stg_url, stg_token = stg_authenticate(account=project_name,
user=admin_user.config_value,
passwd=admin_password.config_value,
domain=domain,
admin=True)
if str(capacity).isdigit() and int(capacity) > int(ainfo.system_capacity):
rec = requests.post(stg_url,
headers={'X-Account-Meta-Quota-Bytes': bytes(capacity), "X-Auth-Token": stg_token})
if rec.status_code == 204:
ainfo.system_capacity = capacity
ainfo.add_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
db.session.add(ainfo)
db.session.commit()
status = 200
message = 'OK'
else:
status = 503
message = 'extend capacity failed'
else:
status = 304
message = 'No expansion is needed'
else:
status = 404
message = 'No system code was found!'
except Exception as ex:
status = 501
message = str(ex)
finally:
resp.update({"status": status, "message": message})
return resp, status
def create_container_to_account(container_name, project_name, domain):
'''
校验account和manager，然后向指定的account下创建传入名称的container
:param container_name:
:param account:
:param manager:
:return:
'''
status = ''
message = ''
data = {}
resp = {"status": status, "data": data, "message": message}
try:
ainfo = db.session.query(SfoAccountManager).filter(
and_(SfoAccountManager.account_stat == '1',
SfoAccountManager.project_name == project_name,
SfoAccountManager.domain == domain)).first()
admin_user = SfoCofigureMethod.query_value_from_con_key('KEYSTONE_USERNAME')
admin_password = SfoCofigureMethod.query_value_from_con_key('KEYSTONE_PASSWORD')
if ainfo:
stg_url, stg_token = stg_authenticate(account=project_name,
user=admin_user.config_value,
passwd=admin_password.config_value,
domain=domain,
admin=True)
account_stat, container_list = swclient.get_account(stg_url, stg_token)
if container_name:
if len(container_list) > 0:
has_ctn_list = map(lambda x: x['name'], container_list)
if container_name in has_ctn_list:
raise ValueError('Container %s exists ' % container_name )
requrl = stg_url + '/' + container_name
crec = requests.put(requrl, headers={"X-Auth-Token": stg_token})
if crec.status_code <= 202:
status = 200
message = 'OK'
data['url'] = requrl
else:
status = 503
message = 'Container create failed'
else:
status = 404
message = "Not Found Record"
except Exception as ex:
status = 501
message = str(ex)
finally:
resp.update({"status": status, "data": data, "message": message})
return resp, status
def create_config(config_json):
status = ''
message = ''
data = []
resp = {"status": status, "message": message, "data": data}
config_group = config_json.get('config_group', '')
config_key = config_json.get('config_key', '')
if SfoCofigureMethod.query_filter_by_group_key(config_group,config_key):
status = 301
message = '%s exists in %s'%(config_key, config_group)
resp.update({"status": status, "message": message, "data": data})
return resp, status
if config_group and config_key:
sfo_config = SfoCofigure()
for config, value in config_json.items():
if hasattr(sfo_config, config):
setattr(sfo_config, config, value)
sfo_config.guid = str(uuid.uuid4())
sfo_config.add_time = timestamp_format(time.time())
db.session.add(sfo_config)
db.session.commit()
status = 201
message = 'Create Success'
else:
status = 400
message = 'config group or key parameters is required'
resp.update({"status": status, "message": message, "data": data})
return resp, status
def disable_account(project_name, domain):
'''
禁用account，实际并不会真正禁用account，只是将account上除系统管理员以外的所有授权清除
:param project_name:
:param domain:
:return:
'''
status = ''
message = ''
resp = {"status": status, "message": message}
try:
ainfo = db.session.query(SfoAccountManager).filter(
and_(SfoAccountManager.project_name == project_name,
SfoAccountManager.account_stat == '1',
SfoAccountManager.domain == domain)).first()
admin_user = SfoCofigureMethod.query_value_from_con_key('KEYSTONE_USERNAME')
admin_password = SfoCofigureMethod.query_value_from_con_key('KEYSTONE_PASSWORD')
keystonecli = init_keystonecli(user=admin_user.config_value,
password=admin_password.config_value)
if ainfo:
remove_result = revoke_roles_project(keystonecli=keystonecli, projectid=ainfo.account_id)
if remove_result:
ainfo.account_stat = '0'
ainfo.add_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
db.session.add(ainfo)
db.session.commit()
status = 200
message = 'OK'
else:
status = 503
message = 'revoke roles failed'
else:
status = 404
message = 'No system code was found!'
except Exception as ex:
status = 501
message = str(ex)
finally:
resp.update({"status": status, "message": message})
return resp, status
def get_config(page, query, limit):
page = int(page) if page else 1
limit = int(limit) if limit else 10
status = ''
message = ''
data = {"config_total": '', "configs": []}
resp = {"status": status, "message": message, "data": data}
sfo_configs = SfoCofigureMethod.query_all_config(page, query, limit)
if sfo_configs:
data['config_total'] = sfo_configs.total
data['configs'].extend(sfo_configs.items)
status = 200
message = 'OK'
else:
status = 404
message = 'Not Found Record'
resp.update({"status": status, "message": message, "data": data})
return resp, status
def update_config(config_json):
status = ''
message = ''
data = []
resp = {"status": status, "message": message, "data": data}
config_group = config_json.get('config_group', '')
config_key = config_json.get('config_key', '')
if config_group and config_key:
sfo_config = SfoCofigureMethod.query_filter_by_group_key(config_group, config_key)
for config, value in config_json.items():
if hasattr(sfo_config, config):
setattr(sfo_config, config, value)
db.session.add(sfo_config)
db.session.commit()
status = 201
message = 'Update Success'
else:
status = 400
message = 'config group or key parameters is required'
resp.update({"status": status, "message": message, "data": data})
return resp, status
def delete_config(config_json):
status = ''
message = ''
data = []
resp = {"status": status, "message": message, "data": data}
config_group = config_json.get('config_group', '')
config_key = config_json.get('config_key', '')
if config_group and config_key:
sfo_configs = SfoCofigureMethod.query_filter_by_group_key(config_group, config_key)
if sfo_configs:
db.session.delete(sfo_configs)
db.session.commit()
status = 204
message = 'Delete Success'
else:
status = 404
message = 'Not Record Found'
else:
status = 400
message = 'config group or key parameters is required'
resp.update({"status": status, "message": message, "data": data})
return resp, status
def cluster_account_info(apply_account, system_env):
status = ''
message = ''
resp = {"status": status, "message": message}
sfo_system_env_con = SfoCofigureMethod.query_value_from_con_key(system_env)
if not sfo_system_env_con:
raise ValueError('system_env %s is invalid' % system_env)
sfo_acc = SfoAccountManager.query.filter_by(system_code=apply_account,
system_env=system_env).first()
if sfo_acc:
status = 304
message = '%s Account existed' % apply_account
data = sfo_acc.system_capacity
resp.update({
"data": {
"systemCapacity": capacity_translate(data),
"ossSystemAccount": apply_account
}
})
else:
status = 200
message = '%s Account available ' % apply_account
resp.update({"status": status, "message": message})
return resp
def create_cluster(
cluster_name,
taskid,
ser_ip,
account_ring_json,
con_ring_json,
obj_ring_json,
proxy_json,
):
from sfo_utils.socket_utils import LocalProcessSocketClient
sfo_nodes = SfoClusterNodesMethod.query_not_used_hosts()
auth_switch = SfoCofigureMethod.query_value_from_con_key('AUTH_SWITCH')
admin_user = SfoCofigureMethod.query_value_from_con_key(
'KEYSTONE_USERNAME')
admin_password = SfoCofigureMethod.query_value_from_con_key(
'KEYSTONE_PASSWORD')
for sfo_node in sfo_nodes:
enable_cluster = False
node_role = {
"Proxy-Server": "NO",
"Container-Server": "NO",
"Account-Server": "NO",
"Object-Server": "NO"
}
for proxy_node in proxy_json['nodes']:
if sfo_node.node_host_name == proxy_node['host_name']:
node_role.update({"Proxy-Server": "YES"})
sfo_node.node_inet_ip = proxy_node['ip']
enable_cluster = True
for account_node in account_ring_json['nodes']:
if sfo_node.node_host_name == account_node['host_name']:
node_role.update({"Account-Server": "YES"})
sfo_node.node_inet_ip = account_node['ip']
sfo_node.node_replicate_ip = account_node["replication_ip"]
enable_cluster = True
for con_node in con_ring_json['nodes']:
if sfo_node.node_host_name == con_node['host_name']:
node_role.update({"Container-Server": "YES"})
sfo_node.node_inet_ip = con_node['ip']
sfo_node.node_replicate_ip = con_node["replication_ip"]
enable_cluster = True
for obj_node in obj_ring_json['nodes']:
if sfo_node.node_host_name == obj_node['host_name']:
node_role.update({"Object-Server": "YES"})
sfo_node.node_inet_ip = obj_node['ip']
sfo_node.node_replicate_ip = obj_node["replication_ip"]
enable_cluster = True
if enable_cluster:
sfo_node.node_role = json.dumps(node_role, encoding='utf-8')
sfo_node.cluster_name = cluster_name
db.session.commit()
def install_relation_service():
sfo_nodes = SfoClusterNodesMethod.query_host_list_by_cluster_name(
cluster_name)
if sfo_nodes:
for idy, sfo_node in enumerate(sfo_nodes):
if sfo_node.cluster_name == cluster_name:
node_role = json.loads(sfo_node.node_role)
node_services = map(
lambda x: x.split('-')[0].lower(),
filter(lambda x: node_role[x] == 'YES', node_role))
for idx, srv in enumerate(node_services):
add_service(sfo_node, srv, taskid)
if srv == 'proxy':
add_service(sfo_node, 'memcached', taskid)
def standard_ga_template():
standard_system(cluster_name, taskid)
sfo_nodes = SfoClusterNodesMethod.query_host_list_by_cluster_name(
cluster_name)
if sfo_nodes:
sfo_proxy_nodes = filter(
lambda x: json.loads(x.node_role)['Proxy-Server'] == 'YES',
sfo_nodes)
sfo_storage_nodes = filter(
lambda x: json.loads(x.node_role)['Account-Server'] == 'YES' or
json.loads(x.node_role)['Container-Server'] == 'YES' or json.
loads(x.node_role)['Object-Server'] == 'YES', sfo_nodes)
for node in sfo_proxy_nodes:
ser_op = ServiceOperation(node.node_inet_ip)
ser_op.excute_cmd(
"sed -i 's/\"64\"/\"1024\"/g' /etc/sysconfig/memcached")
ser_op.excute_cmd(
'sed -i \'$c OPTIONS="-l 0.0.0.0 -U 11211 -t 12 >> /var/log/memcached.log 2>&1"\' /etc/sysconfig/memcached'
)
for node in sfo_storage_nodes:
systemd_zip_path = os.path.join(config.sfo_server_temp_file,
'standard')
give_away_file_to_host(os.path.join(systemd_zip_path,
'SwiftDiskMount.sh'),
node.node_host_name,
'/usr/bin',
taskid=taskid)
ser_op = ServiceOperation(node.node_inet_ip)
ser_op.excute_cmd('chmod +x /usr/bin/SwiftDiskMount.sh')
ser_op.excute_cmd(
'systemctl enable openstack-swift-disk-mount.service')
sfo_memcached_proxy = map(lambda x: x.node_inet_ip + ':11211',
sfo_proxy_nodes)
for idy, sfo_node in enumerate(sfo_nodes):
base_dir = config.sfo_server_temp_file
abs_dir = os.path.join(base_dir, sfo_node.node_inet_ip)
if not os.path.exists(abs_dir):
os.mkdir(abs_dir)
filenames = create_new_config_file(
abs_dir, sfo_node, **{
"private_ip":
sfo_node.node_replicate_ip,
"public_ip":
sfo_node.node_inet_ip,
"ser_ip":
ser_ip,
"account_server_+hostname":
"account_server_%s" % sfo_node.node_host_name,
"object_server_+hostname":
"object_server_%s" % sfo_node.node_host_name,
"container_server_+hostname":
"container_server_%s" % sfo_node.node_host_name,
"proxy_server_+hostname":
"proxy_server_%s" % sfo_node.node_host_name,
"memcachehost1:11211,memcachehost2:11211,memcachehost3:11211":
','.join(sfo_memcached_proxy)
})
for idx, filename in enumerate(filenames):
target_file = os.path.join(abs_dir, filename)
if filename.startswith('account'):
cp_file_path = '/etc/swift/account-server'
elif filename.startswith('container'):
cp_file_path = '/etc/swift/container-server'
elif filename.startswith('object'):
cp_file_path = '/etc/swift/object-server'
elif filename.startswith('rsync'):
cp_file_path = '/etc'
elif filename == 'openstack-swift.conf':
cp_file_path = '/etc/rsyslog.d'
else:
cp_file_path = '/etc/swift'
give_away_file_to_host(target_file,
sfo_node.node_host_name,
cp_file_path, taskid)
def mount_all_node_disks():
sfo_nodes = SfoClusterNodesMethod.query_host_list_by_cluster_name(
cluster_name)
if sfo_nodes:
sfo_nodes = filter(
lambda x: json.loads(x.node_role)['Account-Server'] == 'YES' or
json.loads(x.node_role)['Container-Server'] == 'YES' or json.
loads(x.node_role)['Object-Server'] == 'YES', sfo_nodes)
for node in sfo_nodes:
add_disk(node.node_inet_ip, node.node_replicate_ip,
node.node_host_name, '', taskid)
def create_rings():
rings = ['account', 'container', 'object']
for idy, i in enumerate(rings):
if i == 'account':
create_ring(cluster_name, 'account',
account_ring_json['part_power'],
account_ring_json['replicas'],
account_ring_json['min_part_hours'], '', taskid)
elif i == 'container':
create_ring(cluster_name, 'container',
con_ring_json['part_power'],
con_ring_json['replicas'],
con_ring_json['min_part_hours'], '', taskid)
else:
create_ring(cluster_name, 'object',
obj_ring_json['part_power'],
obj_ring_json['replicas'],
obj_ring_json['min_part_hours'], '', taskid)
def add_all_disk_to_rings():
for host_dict in account_ring_json['nodes']:
ip = host_dict['ip']
port = host_dict['port']
zone = host_dict['zone']
region = host_dict['region']
host_name = host_dict['host_name']
replication_ip = host_dict['replication_ip']
replication_port = host_dict['replication_port']
do = DiskOperation(ip, replication_ip)
_, mount_disks = do.mounted_disks()
for idx, disk in enumerate(mount_disks):
_dist_mt_info = disk.strip()
disk_mt_info_list = _dist_mt_info.split(' ')
if len(disk_mt_info_list) >= 5:
disk_name = disk_mt_info_list[0]
label = disk_mt_info_list[4].replace('[',
'').replace(']', '')
sfo_disk_per = SfoHostInfoMethod.query_host_info_by_host_name(
host_name)
disk_total = json.loads(
sfo_disk_per.disk_useful_size)[disk_name]
disk_total_bytes = reverse_unit(disk_total)
weight = weight_con(disk_total_bytes)
add_disk_2_ring(taskid=taskid,
ring_name='account',
region=region,
zone=zone,
ip=ip,
port=port,
disk_device=label,
weight=weight,
replication_ip=replication_ip,
replication_port=replication_port,
cluster_name=cluster_name)
for host_dict in con_ring_json['nodes']:
ip = host_dict['ip']
port = host_dict['port']
zone = host_dict['zone']
region = host_dict['region']
host_name = host_dict['host_name']
replication_ip = host_dict['replication_ip']
replication_port = host_dict['replication_port']
do = DiskOperation(ip, replication_ip)
_, mount_disks = do.mounted_disks()
for idx, disk in enumerate(mount_disks):
_dist_mt_info = disk.strip()
disk_mt_info_list = _dist_mt_info.split(' ')
if len(disk_mt_info_list) >= 5:
disk_name = disk_mt_info_list[0]
label = disk_mt_info_list[4].replace('[',
'').replace(']', '')
sfo_disk_per = SfoHostInfoMethod.query_host_info_by_host_name(
host_name)
disk_total = json.loads(
sfo_disk_per.disk_useful_size)[disk_name]
disk_total_bytes = reverse_unit(disk_total)
weight = weight_con(disk_total_bytes)
add_disk_2_ring(taskid=taskid,
ring_name='container',
region=region,
zone=zone,
ip=ip,
port=port,
disk_device=label,
weight=weight,
replication_ip=replication_ip,
replication_port=replication_port,
cluster_name=cluster_name)
for host_dict in obj_ring_json['nodes']:
ip = host_dict['ip']
port = host_dict['port']
zone = host_dict['zone']
region = host_dict['region']
host_name = host_dict['host_name']
replication_ip = host_dict['replication_ip']
replication_port = host_dict['replication_port']
do = DiskOperation(ip, replication_ip)
_, mount_disks = do.mounted_disks()
for idx, disk in enumerate(mount_disks):
_dist_mt_info = disk.strip()
disk_mt_info_list = _dist_mt_info.split(' ')
if len(disk_mt_info_list) >= 5:
disk_name = disk_mt_info_list[0]
label = disk_mt_info_list[4].replace('[',
'').replace(']', '')
sfo_disk_per = SfoHostInfoMethod.query_host_info_by_host_name(
host_name)
disk_total = json.loads(
sfo_disk_per.disk_useful_size)[disk_name]
disk_total_bytes = reverse_unit(disk_total)
weight = weight_con(disk_total_bytes)
add_disk_2_ring(taskid=taskid,
ring_name='object',
region=region,
zone=zone,
ip=ip,
port=port,
disk_device=label,
weight=weight,
replication_ip=replication_ip,
replication_port=replication_port,
cluster_name=cluster_name)
def rebalance_ring():
# # 监听创建环后执行添加磁盘
# # # 添加磁盘到环
rebalance('account', '', cluster_name, taskid)
rebalance('container', '', cluster_name, taskid)
rebalance('object', '', cluster_name, taskid)
def ga_ring():
give_away_ring('account.ring.gz', cluster_name, taskid)
give_away_ring('container.ring.gz', cluster_name, taskid)
give_away_ring('object.ring.gz', cluster_name, taskid)
install_function_list = [
install_relation_service, standard_ga_template, mount_all_node_disks,
create_rings, add_all_disk_to_rings, rebalance_ring, ga_ring
]
local_soc_client = LocalProcessSocketClient(host='127.0.0.1', port=54444)
for step, func in enumerate(install_function_list):
try:
step += 1
func()
except AssertionError:
local_soc_client.send(
json.dumps({
"taskid":
taskid,
"float_percent":
round(float(step) / float(len(install_function_list)), 2),
"status":
500
}))
break
else:
local_soc_client.send(
json.dumps({
"taskid":
taskid,
"float_percent":
round(float(step) / float(len(install_function_list)), 2),
"status":
200
}))
import socket
import logging.config
from sfo_common.config_parser import Config
from sfo_server.models import SfoCofigureMethod
sys.path.append(
os.path.join(os.path.dirname(os.path.abspath(__file__)), os.path.pardir))
import socket
from sfo_server.models import SfoClusterNodesMethod, SfoCofigureMethod
def getLogger(name=''):
"""
返回日志对象 Logger
:param name: 配置内的key
:return: Logger
"""
config = Config()
logging.config.fileConfig(config.logging_conf)
logger = logging.getLogger(name) if name else logging
return logger
access_logger = getLogger('access')
# access_logger = logging
script_path = SfoCofigureMethod.query_value_from_con_key('script_path')
SCRIPT_PATH = script_path.config_value if script_path else '/app/sfo-agent/scripts'
SCRIPT_PATH = SCRIPT_PATH.rstrip('/')
PROXY_HOST_IP = SfoCofigureMethod.query_value_from_con_key('proxy_host_ip')
PROXY_HOST_IP = PROXY_HOST_IP.config_value if PROXY_HOST_IP else socket.gethostbyname(
socket.gethostname())
def update_swift_user_info(guid, role='', account_id=''):
status = ''
message = ''
data = {}
resp = {"status": status, "message": message, "data": data}
sfo_swift_user = SfoSwiftUserMethod.query_user_by_guid(guid)
if role:
sfo_role = SfoSwiftRoleMethod.query_role(role)
if not sfo_role:
raise ValueError('Not Found Role %s ' % role)
if account_id:
sfo_account = SfoAccountManagerMethod.query_system_by_accountid(
account_id)
if not sfo_account:
raise ValueError('Not Found Account ID %s ' % account_id)
if sfo_swift_user and (not sfo_swift_user.account_id
or not sfo_swift_user.role_name):
_sfo_swift_user = SfoSwiftUserMethod.query_user_by_unique_constraint(
account_id, role, sfo_swift_user.system_user)
if _sfo_swift_user:
raise ValueError('Account ID:%s Role:%s System User:%s Exists' %
(account_id, role, sfo_swift_user.system_user))
else:
if role:
sfo_swift_user.role_name = role
if account_id:
sfo_swift_user.account_id = account_id
if sfo_swift_user.account_id and sfo_swift_user.role_name:
admin_user = SfoCofigureMethod.query_value_from_con_key(
'KEYSTONE_USERNAME')
admin_password = SfoCofigureMethod.query_value_from_con_key(
'KEYSTONE_PASSWORD')
sfo_account = SfoAccountManagerMethod.query_system_by_accountid(
account_id)
sfo_role = SfoSwiftRoleMethod.query_role(role)
_stg_url, stg_token = stg_authenticate(
account=sfo_account.project_name,
user=admin_user.config_value,
passwd=admin_password.config_value,
domain=sfo_account.domain,
admin=True)
headers = {'X-Auth-Token': stg_token}
swift_role_header = requests.head(_stg_url, headers=headers)
access_logger.info('HEAD_URL:%s, Status Code:%s, Headers:%s' %
(_stg_url, swift_role_header.status_code,
swift_role_header.headers))
if swift_role_header.status_code <= 299:
x_account_control = swift_role_header.headers.get(
'X-Account-Access-Control')
if x_account_control:
x_account_control = json.loads(x_account_control)
role_meta_header = x_account_control.get(
sfo_role.role_meta)
if role_meta_header:
sub_role_meta_set = set(
x_account_control[sfo_role.role_meta])
sub_role_meta_set.add(
"%s:%s" %
(account_id, sfo_swift_user.system_user))
x_account_control[sfo_role.role_meta] = list(
sub_role_meta_set)
else:
sub_role_meta_set = set()
sub_role_meta_set.add(
"%s:%s" %
(account_id, sfo_swift_user.system_user))
x_account_control[sfo_role.role_meta] = list(
sub_role_meta_set)
headers.update({
'X-Account-Access-Control':
json.dumps(x_account_control)
})
else:
role_meta_set = set()
role_meta_set.add(
"%s:%s" % (account_id, sfo_swift_user.system_user))
headers['X-Account-Access-Control'] = json.dumps(
{sfo_role.role_meta: list(role_meta_set)})
access_logger.info('HEAD_URL:%s, Update Headers:%s' %
(_stg_url, headers))
swift_role_control = requests.post(_stg_url,
headers=headers,
timeout=5)
access_logger.info(
'HEAD_URL:%s, Status Code:%s, Headers:%s' %
(_stg_url, swift_role_control.status_code,
swift_role_control.headers))
if swift_role_control.status_code <= 299:
status = 200
message = 'Account Control OK'
sfo_swift_user_passwd = SfoSwiftUserMethod.query_passwd_exists_by_account_id_system_user(
account_id=account_id,
system_user=sfo_swift_user.system_user)
if sfo_swift_user_passwd:
_password = json.loads(
sfo_swift_user_passwd.extend)
password = _password['password']
else:
password = create_random_password()
if not sfo_swift_user.extend:
sfo_swift_user.extend = json.dumps(
{"password": password})
else:
extend = json.loads(sfo_swift_user.extend)
extend['password'] = password
sfo_swift_user.extend = json.dumps(extend)
after_update_header_ensure = requests.head(
_stg_url,
headers={'X-Auth-Token': stg_token},
timeout=5)
if after_update_header_ensure.status_code <= 299:
ensure_headers(after_update_header_ensure.headers,
account_id,
sfo_swift_user.system_user,
sfo_role.role_meta)
# 添加角色
data['password'] = password
data['account'] = account_id
data['username'] = sfo_swift_user.system_user
db.session.add(sfo_swift_user)
db.session.commit()
else:
status = swift_role_control.status_code
message = 'Account Control Fail , Status Code %s' % swift_role_control.status_code
else:
message = 'Head Storage Url %s Fail , Status Code %s' % (
_stg_url, swift_role_header.status_code)
else:
db.session.add(sfo_swift_user)
db.session.commit()
status = 201
message = 'Update OK'
elif sfo_swift_user and sfo_swift_user.role_name and sfo_swift_user.account_id and sfo_swift_user.role_name != role:
_sfo_swift_user = SfoSwiftUserMethod.query_user_by_unique_constraint(
account_id, role, sfo_swift_user.system_user)
if _sfo_swift_user:
raise ValueError('Account ID:%s Role:%s System User:%s Exists' %
(account_id, role, sfo_swift_user.system_user))
else:
if not role:
raise ValueError('Role not allow null')
if sfo_swift_user.account_id and sfo_swift_user.role_name:
admin_user = SfoCofigureMethod.query_value_from_con_key(
'KEYSTONE_USERNAME')
admin_password = SfoCofigureMethod.query_value_from_con_key(
'KEYSTONE_PASSWORD')
sfo_account = SfoAccountManagerMethod.query_system_by_accountid(
account_id)
pre_sfo_role = SfoSwiftRoleMethod.query_role(
sfo_swift_user.role_name)
cur_sfo_role = SfoSwiftRoleMethod.query_role(role)
_stg_url, stg_token = stg_authenticate(
account=sfo_account.project_name,
user=admin_user.config_value,
passwd=admin_password.config_value,
domain=sfo_account.domain,
admin=True)
headers = {'X-Auth-Token': stg_token}
swift_role_header = requests.head(_stg_url, headers=headers)
access_logger.info('HEAD_URL:%s, Status Code:%s, Headers:%s' %
(_stg_url, swift_role_header.status_code,
swift_role_header.headers))
if swift_role_header.status_code <= 299:
_x_account_control = swift_role_header.headers.get(
'X-Account-Access-Control', '')
if _x_account_control:
x_account_control = json.loads(_x_account_control)
try:
idx = x_account_control[
pre_sfo_role.role_meta].index(
'%s:%s' % (sfo_swift_user.account_id,
sfo_swift_user.system_user))
x_account_control[pre_sfo_role.role_meta].pop(idx)
except ValueError, error:
access_logger.warning(str(error))
role_meta_header = x_account_control.get(
cur_sfo_role.role_meta, '')
if role_meta_header:
sub_role_meta_set = set(
x_account_control[cur_sfo_role.role_meta])
sub_role_meta_set.add(
"%s:%s" %
(account_id, sfo_swift_user.system_user))
x_account_control[cur_sfo_role.role_meta] = list(
sub_role_meta_set)
else:
sub_role_meta_set = set()
sub_role_meta_set.add(
"%s:%s" %
(account_id, sfo_swift_user.system_user))
x_account_control[cur_sfo_role.role_meta] = list(
sub_role_meta_set)
else:
role_meta_set = set()
role_meta_set.add(
"%s:%s" % (account_id, sfo_swift_user.system_user))
x_account_control = {
cur_sfo_role.role_meta: list(role_meta_set)
}
headers.update({
'X-Account-Access-Control':
json.dumps(x_account_control)
})
access_logger.info('HEAD_URL:%s, Update Headers:%s' %
(_stg_url, headers))
password = json.loads(sfo_swift_user.extend)['password']
swift_role_control = requests.post(_stg_url,
headers=headers,
timeout=5)
access_logger.info(
'HEAD_URL:%s, Status Code:%s, Headers:%s' %
(_stg_url, swift_role_control.status_code,
swift_role_control.headers))
if swift_role_control.status_code <= 299:
status = 200
message = 'Account Control OK'
after_update_header_ensure = requests.head(
_stg_url,
headers={'X-Auth-Token': stg_token},
timeout=5)
if after_update_header_ensure.status_code <= 299:
ensure_headers(after_update_header_ensure.headers,
account_id,
sfo_swift_user.system_user,
cur_sfo_role.role_meta)
# 删除之前的角色,添加当前的角色
access_logger.info(
'Add Scheduler Job give_away_file_to_proxy')
data['password'] = password
data['account'] = account_id
data['username'] = sfo_swift_user.system_user
data['storage_url'] = sfo_account.storage_url
data['auth_url'] = sfo_account.auth_url
sfo_swift_user.role_name = role
db.session.add(sfo_swift_user)
db.session.commit()
else:
status = swift_role_control.status_code
message = 'Account Control Fail , Status Code %s' % swift_role_control.status_code
else:
message = 'Head Storage Url %s Fail , Status Code %s' % (
_stg_url, swift_role_header.status_code)
def create_swift_user(cluster_name, swift_username, account_id='', role=''):
status = ''
message = ''
data = {}
resp = {"status": status, "message": message, "data": data}
if role:
sfo_role = SfoSwiftRoleMethod.query_role(role)
if not sfo_role:
raise ValueError('Not Found Role %s ' % role)
if account_id:
sfo_account = SfoAccountManagerMethod.query_system_by_accountid(
account_id)
if not sfo_account:
raise ValueError('Not Found Account ID %s ' % account_id)
sfo_swift_user = SfoSwiftUserMethod.create_user(cluster_name,
swift_username, account_id,
role)
if sfo_swift_user and account_id and role and sfo_swift_user.system_user:
admin_user = SfoCofigureMethod.query_value_from_con_key(
'KEYSTONE_USERNAME')
admin_password = SfoCofigureMethod.query_value_from_con_key(
'KEYSTONE_PASSWORD')
sfo_account = SfoAccountManagerMethod.query_system_by_accountid(
account_id)
sfo_role = SfoSwiftRoleMethod.query_role(role)
_stg_url, stg_token = stg_authenticate(
account=sfo_account.project_name,
user=admin_user.config_value,
passwd=admin_password.config_value,
domain=sfo_account.domain,
admin=True)
headers = {'X-Auth-Token': stg_token}
swift_role_header = requests.head(_stg_url, headers=headers, timeout=5)
access_logger.info('HEAD_URL:%s, Status Code:%s, Headers:%s' %
(_stg_url, swift_role_header.status_code,
swift_role_header.headers))
if swift_role_header.status_code <= 299:
x_account_control = swift_role_header.headers.get(
'X-Account-Access-Control')
if x_account_control:
x_account_control = json.loads(x_account_control)
role_meta_header = x_account_control.get(sfo_role.role_meta)
if role_meta_header:
sub_role_meta_set = set(
x_account_control[sfo_role.role_meta])
sub_role_meta_set.add("%s:%s" %
(account_id, swift_username))
x_account_control[sfo_role.role_meta] = list(
sub_role_meta_set)
else:
sub_role_meta_set = set()
sub_role_meta_set.add("%s:%s" %
(account_id, swift_username))
x_account_control[sfo_role.role_meta] = list(
sub_role_meta_set)
headers.update({
'X-Account-Access-Control':
json.dumps(x_account_control)
})
else:
role_meta_set = set()
role_meta_set.add("%s:%s" %
(account_id, sfo_swift_user.system_user))
headers['X-Account-Access-Control'] = json.dumps(
{sfo_role.role_meta: list(role_meta_set)})
access_logger.info('HEAD_URL:%s, Update Headers:%s' %
(_stg_url, headers))
swift_role_control = requests.post(_stg_url,
headers=headers,
timeout=5)
access_logger.info('HEAD_URL:%s, Status Code:%s, Headers:%s' %
(_stg_url, swift_role_control.status_code,
swift_role_control.headers))
if swift_role_control.status_code <= 299:
status = 200
message = 'Account Control OK'
sfo_swift_user_passwd = SfoSwiftUserMethod.query_passwd_exists_by_account_id_system_user(
account_id=account_id, system_user=swift_username)
if sfo_swift_user_passwd:
_password = json.loads(sfo_swift_user_passwd.extend)
password = _password['password']
else:
password = create_random_password()
if not sfo_swift_user.extend:
sfo_swift_user.extend = json.dumps({"password": password})
else:
extend = json.loads(sfo_swift_user.extend)
extend['password'] = password
sfo_swift_user.extend = json.dumps(extend)
after_update_header_ensure = requests.head(
_stg_url, headers={'X-Auth-Token': stg_token}, timeout=5)
if after_update_header_ensure.status_code <= 299:
ensure_headers(after_update_header_ensure.headers,
account_id, swift_username,
sfo_role.role_meta)
# 创建项目-用户-角色
access_logger.info(
'Add Account:%s User:%s role:%s' %
(sfo_account.account_id, sfo_swift_user.system_user,
sfo_swift_user.role_name))
data['password'] = password
data['account'] = account_id
data['username'] = swift_username
db.session.add(sfo_swift_user)
db.session.commit()
else:
status = swift_role_control.status_code
message = 'Account Control Fail , Status Code %s' % swift_role_control.status_code
else:
message = 'Head Storage Url %s Fail , Status Code %s' % (
_stg_url, swift_role_header.status_code)
elif sfo_swift_user:
db.session.add(sfo_swift_user)
db.session.commit()
status = 201
message = 'Create OK'
else:
status = 202
message = 'Account ID:%s Role:%s System User:%s Exists' % (
account_id, role, swift_username)
resp.update({"status": status, "message": message, "data": data})
return resp, status
def add_new_account(project_name, jsonstr):
'''
在swift中添加account，并对账号进行授权和限制容量
:param project_name: 项目名
:param sysname: 项目描述
:param capacity: 申请容量
:return:
'''
status = ''
message = ''
data = {}
resp = {"status": status, "message": message, "data": data}
par = jsonstr
try:
sysinfo = db.session.query(SfoAccountManager).filter(
and_(SfoAccountManager.system_code == project_name,
SfoAccountManager.account_stat == '1',
SfoAccountManager.domain == par['domain'])).first()
admin_user = SfoCofigureMethod.query_value_from_con_key('KEYSTONE_USERNAME')
admin_password = SfoCofigureMethod.query_value_from_con_key('KEYSTONE_PASSWORD')
admin_user_id = SfoCofigureMethod.query_value_from_con_key('KEYSTONE_USER_ID')
keystonecli = init_keystonecli(user=admin_user.config_value,
password=admin_password.config_value) #初始化admin keystone用户
# 如果账号已经存在，不再创建，直接返回相关信息
if sysinfo:
access_logger.info('Exists account %s extend capacity entrance'%(sysinfo.account_id))
access_logger.info('Exists account input parameter %s'%jsonstr)
if int(par['capacity']) > int(sysinfo.system_capacity):
ex_capa_rec, status = extend_account_capacity(project_name, par['capacity'], sysinfo.domain)
if status == 200:
sysinfo.system_capacity = par['capacity']
access_logger.info('account %s extend capacity %s'%(sysinfo.account_id, sysinfo.system_capacity))
data['user'] = sysinfo.system_user
data['account'] = sysinfo.account_id
data['system_code'] = sysinfo.project_name
data['capacity'] = capacity_translate(sysinfo.system_capacity)
sysinfo.add_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
db.session.add(sysinfo)
db.session.commit()
status = 200
message = 'OK'
else:
sysin = SfoAccountManager()
access_logger.info('Create Account Parameters : %s' % str(jsonstr))
sysin.project_name = project_name
sysin.cluster_name = par['clusterName']
access_logger.info('system_code:%s system_user:%s '%(sysin.system_code, sysin.system_user))
sysin.description = par['description']
sysin.system_capacity = par['capacity']
try:
random_password = create_random_password()
username = par.get('username', project_name)
user = keystone_user_create(keystonecli=keystonecli,
username=username,
description=sysin.system_name,
random_password=random_password,
domain=par['domain'])
sysin.system_passwd = random_password
sysin.system_user = username
sysin.keystone_user_id = user.id
data['passwd'] = random_password
# 创建keystone中的project用于swift中的account
# API
proj = keystone_project_create(keystonecli=keystonecli,
project=sysin.project_name,
description=sysin.description,
domain=par['domain'])
if proj:
sysin.account_id = proj.id
# 添加超级管理权限
grec = grant_role_user_project(keystonecli=keystonecli,
rolename='admin',
userid=admin_user_id.config_value,
projectid=proj.id)
grec = grec and grant_role_user_project(keystonecli=keystonecli,
rolename='ResellerAdmin',
userid=admin_user_id.config_value,
projectid=proj.id)
# 授权用户访问
grec = grec and grant_role_user_project(keystonecli=keystonecli,
rolename='swiftuser',
userid=sysin.keystone_user_id,
projectid=proj.id)
stg_url, stg_token = stg_authenticate(account=sysin.project_name,
user=admin_user.config_value,
passwd=admin_password.config_value,
domain=par['domain'],
admin=True)
else:
rollback(keystonecli, sysin.account_id, sysin.keystone_user_id)
raise Exception('create project from keystone fail')
access_logger.info('Create Account auth %s ' % stg_url)
sfo_swift_user = SfoSwiftUserMethod.create_user(cluster_name=par['clusterName'],
systemc_user=sysin.system_user,
account_id=sysin.account_id,
role_name='admin')
if sfo_swift_user:
sfo_swift_user.extend = json.dumps({"password": sysin.system_passwd})
db.session.add(sfo_swift_user)
access_logger.info(
'clusterName:%s, role: admin, user:%s account_id:%s Create Swift User Success, ' % (
par['clusterName'], sysin.system_user, sysin.account_id))
else:
access_logger.info(
'clusterName:%s, role: admin, user:%s account_id:%s Create Swift User Fail, Reason: Duplicate records ' % (
par['clusterName'], sysin.system_user, sysin.account_id))
capacity = par.get('capacity')
headers = {'X-Account-Meta-Quota-Bytes': bytes(capacity),
"X-Auth-Token": stg_token}
rrec = requests.post(stg_url,
headers=headers,
timeout=5)
access_logger.info('Create Account auth %s Return_code:%s' % (stg_url, rrec.status_code))
if rrec.status_code == 204:
data['capacity'] = capacity_translate(par['capacity'])
sysin.account_stat = '1'
sysin.guid = str(uuid.uuid1())
sysin.expire_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time() + (86400 * 365)))
sysin.add_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
db.session.add(sysin)
container_name = par.get('containerName')
if container_name:
requrl = stg_url + '/' + container_name
crec = requests.put(requrl,
headers={"X-Auth-Token": stg_token},
timeout=5)
access_logger.info('Create Container auth %s Return_code:%s ' % (requrl, crec.status_code))
if crec.status_code <= 202:
status = 200
message = 'OK'
data['container'] = container_name
else:
data = {}
status = 506
message = 'create container for account failed,error code is %s' % (
str(crec.status_code))
rollback(keystonecli, sysin.account_id, sysin.keystone_user_id)
else:
status = 200
message = 'OK'
db.session.commit()
else:
data = {}
status = 505
message = 'grant role failed or set capacity failed'
rollback(keystonecli, sysin.account_id, sysin.keystone_user_id)
except requests.ConnectTimeout, ex:
status = 501
message = str(ex)
rollback(keystonecli, sysin.account_id, sysin.keystone_user_id)
except Exception, ex:
status = 501
message = str(ex)
rollback(keystonecli, sysin.account_id, sysin.keystone_user_id)
def get_srv_list(cluster_name, **keyword):
"""
获取服务列表
:param cluster_name:
:return:
"""
data = []
sfo_srvs = []
apply_result_list = []
status = ''
message = ''
software_packs = SfoCofigureMethod.query_value_from_con_group(
'software_package')
resp = {
"status": status,
"data": data,
"message": message,
"software_packages": software_packs
}
software_packages_map = map(
lambda x: 'openstack-swift-%s*' % x.config_value
if x.config_value in ['account', 'container', 'object', 'proxy'] else
'%sd?' % x.config_value, software_packs)
software_reg = '|'.join(software_packages_map)
sfo_nodes = SfoClusterNodesMethod.query_host_list_by_cluster_name(
cluster_name)
if not sfo_nodes:
raise ValueError('Nodes is null')
pool = Pool(25)
for node in sfo_nodes:
so = ServiceOperation(node.node_inet_ip)
apply_result = pool.apply_async(func=so.srvs, args=(software_reg, ))
apply_dict = {
"host_name": node.node_host_name,
"ip": node.node_inet_ip,
"apply_result": apply_result
}
apply_result_list.append(apply_dict)
pool.close()
pool.join()
for apply_dict in apply_result_list:
ip = apply_dict['ip']
host_name = apply_dict['host_name']
apply_result = apply_dict['apply_result']
apply_result_data = apply_result.get(timeout=1)
if apply_result_data:
for srv in apply_result_data:
for node in sfo_nodes:
if host_name == node.node_host_name:
node_role = node.node_role
try:
node_role_js = json.loads(node_role)
if isinstance(node_role_js, dict):
roles = filter(
lambda x: node_role_js[x] == 'YES',
node_role_js.keys())
for role in roles:
role, _ = role.split('-')
role = role.lower()
role_com = re.compile(role)
search_result = role_com.search(
srv['service'])
if search_result:
hostsrv = HostSrv(
srv['service'], host_name, ip,
srv['stat'], srv['message'])
sfo_srvs.append(hostsrv)
except ValueError:
continue
if sfo_srvs:
if not keyword:
for srv in sfo_srvs:
if srv.service_name not in map(lambda x: x.service_name, data):
data.append(srv)
status = 200
message = 'OK'
else:
for query_key, keyword_val in keyword.items():
for srv in sfo_srvs:
if hasattr(srv, query_key):
if getattr(srv, query_key) == keyword_val:
data.append(srv)
status = 200
message = 'OK'
else:
raise AttributeError('%s has no %s attribute ' %
(srv, query_key))
else:
status = 404
message = 'Not Found Record'
resp.update({"status": status, "data": data, "message": message})
return resp, status