def getParam(self, param):
try:
mac = param['macaddress']
cluster_id = str(param['cluster_id'])
if mac == '' and cluster_id == '':
return ''
factoryDB = FactoryDBConnect()
condition = {}
if mac != '':
condition['macaddress'] = mac
param = factoryDB.queryTable(table_name.HPC_SERVER,condition)
elif cluster_id != '':
if len(cluster_id) > 10:
condition['task_uuid'] = cluster_id
param = factoryDB.queryTable(table_name.HPC_TASK,condition)
else:
condition['cluster_id'] = cluster_id
param = factoryDB.queryTable(table_name.HPC_SERVER,condition)
return param
except Exception, e:
print_except_line()
print e
def signal_handler(cls, signum, frame):
DEBUG and debug_print('In custom signal handler for SIGTERM')
if signum == signal.SIGTERM:
factoryDB = FactoryDBConnect()
tasks, task_num = factoryDB.queryTable(table_name.HPC_TASK, '')
for task in tasks:
if task['status'] != task_status.SUCCESS:
cls.update_task_status(task['task_uuid'], task_status.FAILED)
def task_process(cls):
while True:
while len(cls.task_list) != 0:
task_info = cls.task_list.pop(0)
cf_task = {}
key_list = ['task_uuid', 'idc_id', 'cluster_id',
'cluster_type', 'cluster_name', 'task_type']
for k in key_list:
if k in task_info.keys():
cf_task[k] = str(task_info[k])
cf_task['status'] = task_status.INITIAL
try:
factoryDB = FactoryDBConnect()
print '\n'
#print cf_task
result = factoryDB.insert_record(cf_task, table_name.CF_TASK)
if not result[0]:
print ADD_TASK_ERR
continue
condition = {}
condition['idc_id'] = task_info['idc_id']
idcs, idc_num = factoryDB.queryTable(table_name.CF_AGENT, '')
if idc_num < 0:
print 'No registration info for IDC: %s' % task_info['idc_id']
continue
xmlrpcObj = xmlrpclib.ServerProxy(idcs[0]['req_url'], allow_none=True)
result = xmlrpcObj.request(task_info)
if result['result'] != '0':
print result['message']
cls.update_task_status(task_info['task_uuid'], task_status.REQ_ERROR)
else:
cls.update_task_status(task_info['task_uuid'], task_status.PROGRESS)
except Exception, e:
print_except_line()
print e
def register(self, args_dict):
retMsg = {}
retMsg['return'] = ''
retMsg['message'] = ''
pattern_dict = {
'idc_id' : '',
'req_url' : '',
'admin_email' : ''
}
chk_ret = self._args_check_ok(pattern_dict, args_dict)
if chk_ret[0] is False:
return chk_ret[1]
try:
factoryDB = FactoryDBConnect()
condition = {}
condition['idc_id'] = args_dict['idc_id']
idcs, idc_num = factoryDB.queryTable(table_name.CF_AGENT, condition)
if idc_num > 0:
retMsg['result'] = enum_ret.OK
retMsg['message'] = ret_msg.REGISTRATION_OK
return retMsg
agent_info = {}
for k in args_dict.keys():
agent_info[k] = args_dict[k]
result = factoryDB.insert_record(agent_info, table_name.CF_AGENT)
if result[0]:
retMsg['result'] = enum_ret.OK
retMsg['message'] = ret_msg.REGISTRATION_OK
else:
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = ADD_AGENT_TO_DB_ERR
return retMsg
except Exception, e:
print_except_line()
print e
def server_powerstatus(self, args_dict):
retMsg = {}
retMsg['return'] = ''
retMsg['message'] = ''
chk_ret = self._args_check_ok(pattern_dict.POWERSTATUS, args_dict)
if chk_ret[0] is False:
return chk_ret[1]
server_info = {}
server_info['macaddress'] = args_dict["macaddress"]
try:
factoryDB = FactoryDBConnect()
servers, server_num = factoryDB.queryTable(table_name.HPC_SERVER, server_info)
if server_num <= 0:
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = err_msg.SERVER_NOT_EXISTED
return retMsg
ctlCmd = './remote_powerstatus.sh '+ servers[0]['idrac_ip'] + ' ' + servers[0]['idrac_user'] + ' ' + servers[0]['idrac_password']
ctlProcess = subprocess.Popen(ctlCmd, shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for line in iter(ctlProcess.stdout.readline, b''):
if 'ON' in line:
retMsg['result'] = enum_ret.OK
retMsg['return'] = 'ON'
elif 'OFF' in line:
retMsg['return'] = 'OFF'
retMsg['result'] = enum_ret.OK
else:
retMsg['result'] = enum_ret.ERROR
ctlProcess.communicate()
return retMsg
except Exception, e:
print_except_line()
print e
send_except_msg_to_server(e.__str__())
def update_server_status(cls, mac, status, comment=''):
update_info = {}
update_info['status'] = status
update_info['comment'] = comment
condition = {}
condition['macaddress'] = mac
try:
factoryDB = FactoryDBConnect()
factoryDB.update_record(update_info, condition, table_name.HPC_SERVER)
except Exception, e:
print_except_line()
print e
send_except_msg_to_server(e.__str__())
def update_task_status(cls, task_uuid, status, ret_dict={}, comment=''):
DEBUG and debug_print('\nTask: %s, status: %s\n' % (task_uuid, status, ) )
retMsg = {}
retMsg['return'] = ''
retMsg['message'] = ''
update_info = {}
update_info['status'] = status
update_info['comment'] = comment
condition = {}
condition['task_uuid'] = task_uuid
try:
factoryDB = FactoryDBConnect()
factoryDB.update_record(update_info, condition, table_name.CF_TASK)
condition = {}
condition['task_uuid'] = task_uuid
tasks, task_num = factoryDB.queryTable(table_name.CF_TASK, condition)
if task_num > 0:
if tasks[0]['task_type'] != task_type.DELETE:
if status == task_status.SUCCESS or status == task_status.FAILED or status == task_status.REQ_ERROR:
s=xmlrpclib.ServerProxy(hpc_mgr.REQ_URL)
DEBUG and debug_print('Send cluster info to HPC Manager:')
DEBUG and debug_print(ret_dict)
time.sleep(60)
s.afterCreateCloudFactory(ret_dict)
# cbThread = Thread(target=call_hpcmgr_back, kwargs=ret_dict)
# cbThread.daemon = True
# cbThread.start()
retMsg['result'] = enum_ret.OK
return retMsg
except Exception, e:
print_except_line()
retMsg['result'] = enum_ret.ERROR
return retMsg
def except_msg_from_agent(self, agent_id, msg):
retMsg = {}
retMsg['return'] = ''
retMsg['message'] = ''
try:
factoryDB = FactoryDBConnect()
condition = {}
condition['idc_id'] = agent_id
agents, agent_num = factoryDB.queryTable(table_name.CF_AGENT, condition)
admin_email = agents[0]['admin_email']
smtpObj = smtplib.SMTP(smtp_info.HOST, smtp_info.PORT)
msg = smtp_info.SUBJECT + agent_id + "\n\n" + msg
smtpObj.sendmail(smtp_info.SENDER, admin_email, msg)
DEBUG and debug_print("Sent email successfully")
alert_info = {}
alert_info['idc_id'] = agent_id
alert_info['message'] = msg
#factoryDB.insert_record(alert_info, table_name.CF_ALERT)
retMsg['result'] = enum_ret.OK
return retMsg
except Exception, e:
print_except_line()
print e
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = e.__str__()
return retMsg
def config_dhcp_for_server(cls, ip_dict):
must_keys=["macaddress", "action"]
for key in must_keys:
if not key in ip_dict.keys():
DEBUG and debug_print('No value set for %s' % key)
return False
if ip_dict["action"] == dhcp_config.ADD:
must_keys=["server_name", "bootfile"]
for key in must_keys:
if not key in ip_dict.keys():
DEBUG and debug_print('No value set for %s' % key)
return False
index = cls.get_free_index(ip_dict['macaddress'])
if index == 0:
DEBUG and debug_print('No ip address available!')
return False
fixed_ip = install_network_config.get("prefix") + str(index)
ctlCmd = './dhcp_config.sh '+ip_dict['action']+' '+ip_dict['macaddress']+' '+ip_dict['server_name']+' '+fixed_ip+' '+ip_dict['bootfile']
try:
factoryDB = FactoryDBConnect()
condition = {}
condition['macaddress'] = ip_dict['macaddress']
update_info = {}
update_info['install_ip'] = fixed_ip
factoryDB.update_record(update_info, condition, table_name.HPC_SERVER)
except Exception, e:
print_except_line()
print e
send_except_msg_to_server(e.__str__())
def update_task_status(cls, task_uuid, status, comment=''):
DEBUG and debug_print('\nTask: %s, status: %s\n' % (task_uuid, status, ))
update_info = {}
update_info['status'] = status
update_info['comment'] = comment
condition = {}
condition['task_uuid'] = task_uuid
xmlrpcObj = xmlrpclib.ServerProxy(cf_server.REQ_URL, allow_none=True)
try:
factoryDB = FactoryDBConnect()
factoryDB.update_record(update_info, condition, table_name.HPC_TASK)
except Exception, e:
print_except_line()
cls.unconfig_dhcp(task_uuid)
xmlrpcObj.update_task_status(task_uuid, status)
send_except_msg_to_server(e.__str__())
def process_pending_task(cls):
try:
factoryDB = FactoryDBConnect()
task_info = {}
task_info['status'] = task_status.PROGRESS
tasks, task_num = factoryDB.queryTable(table_name.HPC_TASK, task_info)
if task_num <= 0:
return
for task in tasks:
time_create = int(time.mktime(task['create_time'].timetuple()))
time_now = int(time.time())
if time_now - time_create < TASK_TIMEOUT:
continue
device_info = {}
device_info['task_uuid'] = task['task_uuid']
devices, device_num = factoryDB.queryTable(table_name.HPC_SERVER, device_info)
comment = 'Server: '
for device in devices:
if device['status'] != server_status.INSTALLED:
comment = comment + device['name'] + ', '
comment = comment + ' install time out'
cls.update_task_status(task['task_uuid'], task_status.FAILED, comment)
except Exception, e:
print_except_line()
print e
send_except_msg_to_server(e.__str__())
def statusNotify(cls, mac, status, comment=''):
if mac == '' or status == '':
return None
status = base64.b64decode(status)
cls.update_server_status(mac, status)
try:
condition = {}
condition['macaddress'] = mac
factoryDB = FactoryDBConnect()
devices, device_num = factoryDB.queryTable(table_name.HPC_SERVER, condition)
if device_num > 0:
DEBUG and debug_print('Status notify from (' + mac + ')' + devices[0]['name'] + ': ' + status)
condition = {}
condition['task_uuid'] = devices[0]['task_uuid']
devices, device_num = factoryDB.queryTable(table_name.HPC_SERVER, condition)
installed_num = 0
for device in devices:
if device['status'] == server_status.FAILED:
cls.update_task_status(device['task_uuid'],task_status.FAILED,device['name']+': '+comment)
break
elif device['status'] == server_status.INSTALLED:
installed_num = installed_num + 1
if installed_num == device_num:
cls.update_task_status(devices[0]['task_uuid'], task_status.SUCCESS)
except Exception, e:
print_except_line()
print e
send_except_msg_to_server(e.__str__())
def unconfig_dhcp(cls, task_uuid):
try:
factoryDB = FactoryDBConnect()
devices_info = {}
devices_info['task_uuid'] = task_uuid
devices, device_num = factoryDB.queryTable(table_name.HPC_SERVER, devices_info)
for device in devices:
if device['status'] != server_status.INITIAL:
ipconfig = {}
ipconfig['action'] = dhcp_config.DELETE
ipconfig['macaddress'] = device['macaddress']
if not cls.config_dhcp_for_server(ipconfig):
DEBUG and debug_print("Undo dhcp config for server: %s failed!" % (device['name'], ))
for k in cls.dhcp_ip_dict.keys():
if cls.dhcp_ip_dict[k] == device['macaddress']:
cls.dhcp_ip_dict[k] = 0
#condition = {}
#condition['macaddress'] = device['macaddress']
#factoryDB.delete_record(condition, table_name.HPC_SERVER)
#For testing
condition = {}
condition['task_uuid'] = task_uuid
factoryDB.delete_record(condition, table_name.HPC_TASK)
except Exception, e:
print_except_line()
print e
send_except_msg_to_server(e.__str__())
def deleteServers(self, args_dict):
retMsg = {}
retMsg['return'] = ''
retMsg['message'] = ''
chk_ret = self._args_check_ok(pattern_dict.DELETE, args_dict)
if chk_ret[0] is False:
return chk_ret[1]
server_info = {}
server_info['idc_id'] = args_dict["idc_id"]
server_info['cluster_id'] = str(args_dict["cluster_id"])
try:
factoryDB = FactoryDBConnect()
servers, server_num = factoryDB.queryTable(table_name.HPC_SERVER, server_info)
if server_num <= 0:
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = err_msg.CLUSTER_NOT_EXIST
return retMsg
else:
if len(args_dict['server_list']) == 0:
update_info = {}
update_info['status'] = server_status.DELETED
condition = {}
condition['idc_id'] = args_dict['idc_id']
condition['cluster_id'] = str(args_dict['cluster_id'])
factoryDB.update_record(update_info, condition, table_name.HPC_SERVER)
else:
for server in args_dict['server_list']:
update_info = {}
update_info['status'] = server_status.DELETED
condition = {}
condition['macaddress'] = server['macaddress']
factoryDB.update_record(update_info, condition, table_name.HPC_SERVER)
task_info = {}
task_info['task_uuid'] = args_dict['task_uuid']
task_info['idc_id'] = args_dict['idc_id']
task_info['cluster_id'] = str(args_dict['cluster_id'])
task_info['status'] = task_status.INITIAL
task_info['task_type'] = task_type.DELETE
result = factoryDB.insert_record(task_info, table_name.HPC_TASK)
if result[0]:
retMsg['result'] = enum_ret.OK
else:
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = err_msg.ADD_TASK_ERR
return retMsg
except Exception, e:
print_except_line()
print e
send_except_msg_to_server(e.__str__())
def request(self, stoken, args_dict):
DEBUG and debug_print('task arguments:')
DEBUG and debug_print(args_dict)
retMsg = {}
retMsg['return'] = ''
retMsg['message'] = ''
#For testing
#args_dict['cluster_type'] = cluster_type.XENSERVER
#Check security token
must_keys = {'access_uuid':'', 'security_hash':'', 'timestamp':''}
chk_ret = self._args_check_ok(must_keys, stoken)
if chk_ret[0] is False:
DEBUG and debug_print(chk_ret[1])
return chk_ret[1]
timestamp = int(time.time())
DEBUG and debug_print('client: ' + str(stoken['timestamp']))
DEBUG and debug_print('server: ' + str(timestamp))
#*************Check security token************************
if (timestamp - int(stoken['timestamp'])) > TOKEN_EXPIRE_TIME:
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = ret_msg.INVALID_TIME_STAMP
return retMsg
if access_uuid != stoken['access_uuid']:
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = ret_msg.INVALID_ACCESS_UUID
return retMsg
security_string = "<"+str(stoken['timestamp'])+"><"+ "request" +">"
security_hash = base64.encodestring(hmac.new(access_key.decode('utf8').encode('cp850'), security_string, sha).hexdigest()).strip()
if security_hash != stoken['security_hash']:
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = ret_msg.INVALID_SECURITY_HASH
return retMsg
#*************Check security token End************************
#For testing
#retMsg['result'] = enum_ret.OK
#retMsg['message'] = 'token check ok!'
#return retMsg
#End
#Check task arguments
if args_dict['task_type'] == task_type.POWERON or args_dict['task_type'] == task_type.POWERSTATUS or args_dict['task_type'] == task_type.POWEROFF:
must_keys = {'idc_id':'', 'task_type':''}
else:
must_keys = {'task_uuid':'', 'idc_id':'', 'task_type':''}
chk_ret = self._args_check_ok(must_keys, args_dict)
if chk_ret[0] is False:
DEBUG and debug_print(chk_ret[1])
return chk_ret[1]
#Create cluster
if args_dict['task_type'] == task_type.CREATE:
chk_ret = self._args_check_ok(pattern_dict.CREATE, args_dict)
if chk_ret[0] is False:
DEBUG and debug_print(chk_ret[1])
return chk_ret[1]
if not args_dict['cluster_type'] in [cluster_type.VMWARE, cluster_type.XENSERVER]:
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = ret_msg.INVALID_CLUSTER_TYPE
DEBUG and debug_print(retMsg)
return retMsg
#Add server
elif args_dict['task_type'] == task_type.ADD:
chk_ret = self._args_check_ok(pattern_dict.UPDATE, args_dict)
if chk_ret[0] is False:
DEBUG and debug_print(chk_ret[1])
return chk_ret[1]
#Delete cluster
elif args_dict['task_type'] == task_type.DELETE:
chk_ret = self._args_check_ok(pattern_dict.DELETE, args_dict)
if chk_ret[0] is False:
DEBUG and debug_print(chk_ret[1])
return chk_ret[1]
#Power up server
elif args_dict['task_type'] == task_type.POWERON:
chk_ret = self._args_check_ok(pattern_dict.POWERON, args_dict)
if chk_ret[0] is False:
DEBUG and debug_print(chk_ret[1])
return chk_ret[1]
#Get power status
elif args_dict['task_type'] == task_type.POWERSTATUS:
chk_ret = self._args_check_ok(pattern_dict.POWERSTATUS, args_dict)
if chk_ret[0] is False:
DEBUG and debug_print(chk_ret[1])
return chk_ret[1]
#Power down server
elif args_dict['task_type'] == task_type.POWEROFF:
chk_ret = self._args_check_ok(pattern_dict.POWEROFF, args_dict)
if chk_ret[0] is False:
DEBUG and debug_print(chk_ret[1])
return chk_ret[1]
else:
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = ret_msg.INVALID_TASK_TYPE+': ' + args_dict['task_type']
DEBUG and debug_print(retMsg)
return retMsg
#Synchronously calling for power status management
if args_dict['task_type'] == task_type.POWERON or args_dict['task_type'] == task_type.POWERSTATUS or args_dict['task_type'] == task_type.POWEROFF:
print 'power status-----------------'
condition = {}
condition['idc_id'] = args_dict['idc_id']
factoryDB = FactoryDBConnect()
idcs, idc_num = factoryDB.queryTable(table_name.CF_AGENT, '')
if idc_num < 0:
print 'No registration info for IDC: %s' % args_dict['idc_id']
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = ret_msg.INVALID_IDC_INFO
return retMsg
xmlrpcObj = xmlrpclib.ServerProxy(idcs[0]['req_url'], allow_none=True)
result = xmlrpcObj.request(args_dict)
retMsg['result'] = enum_ret.OK
retMsg['return'] = result['return']
return retMsg
#Asynchronously processing tasks
else:
self.task_list.append(args_dict)
retMsg['result'] = enum_ret.OK
retMsg['message'] = 'Request success!'
return retMsg
def createCluster(self, args_dict):
retMsg = {}
retMsg['return'] = ''
retMsg['message'] = ''
chk_ret = self._args_check_ok(pattern_dict.CREATE, args_dict)
if chk_ret[0] is False:
return chk_ret[1]
task_info = {}
task_info['task_uuid'] = args_dict['task_uuid']
try:
factoryDB = FactoryDBConnect()
tasks, task_num = factoryDB.queryTable(table_name.HPC_TASK, task_info)
if task_num > 0:
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = err_msg.INVALID_TASK_ID
return retMsg
#Allocate IP and store server info into db
ip_parts = args_dict['mgr_vlan']['range'][0].split('.')
ip_subnet= ip_parts[0] + '.' + ip_parts[1] + '.' + ip_parts[2] + '.'
ip_host = int(ip_parts[3])
for server in args_dict['server_list']:
server_info = {}
for k in server.keys():
if server[k] is not None:
server_info[k] = server[k]
server_info['mgr_ip'] = ip_subnet + str(ip_host)
server_info['idc_id'] = args_dict['idc_id']
server_info['cluster_id'] = str(args_dict['cluster_id'])
server_info['task_uuid'] = args_dict['task_uuid']
server_info['netmask'] = args_dict['mgr_vlan']['netmask']
server_info['gateway'] = args_dict['mgr_vlan']['gateway']
server_info['ssh_user'] = '******'
server_info['ssh_password'] = '******' #generatePwd()
server_info['status'] = server_status.INITIAL
# Delete existing server with the same mac address
condition = {}
condition['macaddress'] = server['macaddress']
factoryDB.delete_record(condition, table_name.HPC_SERVER)
result = factoryDB.insert_record(server_info, table_name.HPC_SERVER)
ip_host = ip_host + 1
DEBUG and debug_print('Add server:%s to cluster: %s\n' % (server_info['name'], args_dict['cluster_name']))
if result[0]:
retMsg['result'] = enum_ret.OK
else:
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = ADD_SERVER_TO_DB_ERR
return retMsg
#Store vlan info into db
vlan_info = {}
for k in args_dict['mgr_vlan'].keys():
if k != 'range':
vlan_info[k] = str(args_dict['mgr_vlan'][k])
result = factoryDB.insert_record(vlan_info, table_name.HPC_VLAN)
row_id = result[1]
if result[0]:
retMsg['result'] = enum_ret.OK
else:
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = ADD_SERVER_TO_DB_ERR
return retMsg
#Task should be added until cluster info have been fully recorded
task_info = {}
task_info['task_uuid'] = args_dict['task_uuid']
task_info['idc_id'] = args_dict['idc_id']
task_info['cluster_id'] = str(args_dict['cluster_id'])
task_info['cluster_type'] = args_dict['cluster_type']
task_info['cluster_name'] = args_dict['cluster_name']
task_info['dns'] = args_dict['dns']
task_info['san_ip'] = args_dict['storageinfo']['san_ip']
task_info['san_target'] = args_dict['storageinfo']['san_target']
task_info['san_user'] = args_dict['storageinfo']['san_user']
task_info['san_password'] = args_dict['storageinfo']['san_password']
task_info['mgr_vlan_id'] = str(row_id)
task_info['status'] = task_status.INITIAL
task_info['task_type'] = task_type.CREATE
#Determine cluster ip
if task_info['cluster_type'] == cluster_type.XENSERVER:
condition = {}
condition['server_level'] = server_level.XENSERVER_MASTER
condition['task_uuid'] = task_info['task_uuid']
devices, devices_num = factoryDB.queryTable(table_name.HPC_SERVER, condition)
if devices_num > 0:
task_info['cluster_ip'] = devices[0]['mgr_ip']
task_info['netmask'] = devices[0]['netmask']
task_info['gateway'] = devices[0]['gateway']
task_info['user'] = devices[0]['ssh_user']
task_info['password'] = devices[0]['ssh_password']
elif task_info['cluster_type'] == cluster_type.VSPHERE:
task_info['cluster_ip'] = ip_subnet + str(ip_host)
condition = {}
condition['server_level'] = server_level.VSPHERE_MASTER
condition['task_uuid'] = task_info['task_uuid']
devices, devices_num = factoryDB.queryTable(table_name.HPC_SERVER, condition)
if devices_num > 0:
task_info['netmask'] = devices[0]['netmask']
task_info['gateway'] = devices[0]['gateway']
task_info['user'] = devices[0]['ssh_user']
#task_info['password'] = devices[0]['ssh_password']
task_info['password'] = '******'
DEBUG and debug_print('New task:\n')
DEBUG and debug_print(task_info)
result = factoryDB.insert_record(task_info, table_name.HPC_TASK)
if result[0]:
retMsg['result'] = enum_ret.OK
DEBUG and debug_print('New task added: \n')
else:
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = err_msg.ADD_TASK_ERR
return retMsg
return retMsg
except Exception, e:
print_except_line()
print e
send_except_msg_to_server(e.__str__())
def addServers(self, args_dict):
retMsg = {}
retMsg['return'] = ''
retMsg['message'] = ''
chk_ret = self._args_check_ok(pattern_dict.ADD, args_dict)
if chk_ret[0] is False:
return chk_ret[1]
server_info = {}
server_info['idc_id'] = args_dict["idc_id"]
server_info['cluster_id'] = str(args_dict["cluster_id"])
try:
factoryDB = FactoryDBConnect()
servers, server_num = factoryDB.queryTable(table_name.HPC_SERVER, server_info)
if server_num <= 0:
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = err_msg.CLUSTER_NOT_EXIST
return retMsg
#Update Cluster
for server in args_dict['server_list']:
server_info = {}
for k in server.keys():
server_info[k] = server[k]
server_info['ssh_user'] = '******'
server_info['ssh_password'] = '******' #generatePwd()
server_info['status'] = server_status.INITIAL
result = factoryDB.insert_record(server_info, table_name.HPC_SERVER)
if result[0]:
retMsg['result'] = enum_ret.OK
DEBUG and debug_print('Update, add server:%s to cluster-----\n' % (server_info['name'], ))
else:
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = err_msg.ADD_SERVER_TO_CLUSTER_FAILED
return retMsg
#Add task to update cluster
task_info = {}
task_info['task_uuid'] = args_dict["task_uuid"]
task_info['idc_id'] = args_dict["idc_id"]
task_info['cluster_id'] = str(args_dict["cluster_id"])
task_info['status'] = task_status.INITIAL
task_info['task_type'] = task_type.ADD
result = factoryDB.insert_record(task_info, table_name.HPC_TASK)
if result[0]:
retMsg['result'] = enum_ret.OK
else:
retMsg['result'] = enum_ret.ERROR
retMsg['message'] = err_msg.ADD_TASK_ERR
return retMsg
return retMsg
except Exception, e:
print_except_line()
print e
send_except_msg_to_server(e.__str__())
def process_new_task(cls):
try:
factoryDB = FactoryDBConnect()
task_info = {}
task_info['status'] = task_status.INITIAL
tasks, task_num = factoryDB.queryTable(table_name.HPC_TASK, task_info)
if task_num <= 0:
return
for task in tasks:
device_info = {}
device_info['cluster_id'] = str(task['cluster_id'])
device_info['idc_id'] = str(task['idc_id'])
devices, device_num = factoryDB.queryTable(table_name.HPC_SERVER, device_info)
if device_num <= 0:
DEBUG and debug_print('No server in cluster: ' + str(task['cluster_id']) + '\n')
continue
#****************************Create cluster**************************#
if task['task_type'] == task_type.CREATE :
DEBUG and debug_print('*****************Create cluster*****************\n')
#****For testing purpose****
#cls.update_task_status(task['task_uuid'], task_status.SUCCESS)
#return
#**************************
#change task status to 'progress'
cls.update_task_status(task['task_uuid'], task_status.PROGRESS)
for device in devices:
ipconfig = {}
ipconfig['action'] = dhcp_config.ADD
ipconfig['server_name'] = device['uuid']
ipconfig['nameserver'] = task['dns']
ipconfig['macaddress'] = device['macaddress']
ipconfig['bootfile'] = task['cluster_type'] + '/pxelinux.0'
if task['cluster_type'] == cluster_type.CCP:
ipconfig['bootfile'] = device['server_level'] + '/pxelinux.0'
if not cls.config_dhcp_for_server(ipconfig):
DEBUG and debug_print('Failed to config dhcp for server: %s' % (device['name'], ))
#change task status to 'failed'
cls.update_task_status(task['task_uuid'], task_status.FAILED)
break
DEBUG and debug_print('Configure /etc/dhcpd.conf for %s\n' % (device['name'], ))
cls.update_server_status(device['macaddress'], server_status.PROGRESS)
#Boot server from PXE immediately
if not cls.power_control(device['idrac_ip'],
device['idrac_user'],
device['idrac_password'],
power_state.START):
#change task status to 'failed'
cls.update_task_status(task['task_uuid'], task_status.FAILED)
DEBUG and debug_print('Could no start server "' + device['name'] + '" remotely\n')
break
DEBUG and debug_print('Start server "' + device['name'] + '" remotely-----\n')
#****************************Destroy cluster**************************#
elif task['task_type'] == task_type.DELETE:
DEBUG and debug_print('*********************Delete cluster************************\n')
for device in devices:
if device['status'] == server_status.DELETED:
ipconfig = {}
ipconfig['action'] = dhcp_config.DELETE
ipconfig['macaddress'] = device['macaddress']
#if not cls.config_dhcp_for_server(ipconfig):
# print 'Failed to remove dhcp info for server: %s' % (device['name'], )
if not cls.power_control(device['idrac_ip'],
device['idrac_user'],
device['idrac_password'],
power_state.SHUTDOWN):
DEBUG and debug_print('Failed to shutdown server: %s' % (device['name'], ))
else:
DEBUG and debug_print('Shutdown server: %s' % (device['name'], ))
condition = {}
condition['macaddress'] = device['macaddress']
factoryDB.delete_record(condition, table_name.HPC_SERVER)
#change task status to 'success'
cls.update_task_status(task['task_uuid'], task_status.SUCCESS)
#****************************Update cluster**************************#
elif task['task_type'] == task_type.ADD:
DEBUG and debug_print('*********************Update cluster************************\n')
cls.update_task_status(task['task_uuid'], task_status.PROGRESS)
for device in devices:
if device['status'] == server_status.INITIAL:
#Add IP info to /etc/dhcpd.conf
ipconfig = {}
ipconfig['action'] = dhcp_config.ADD
ipconfig['macaddress'] = device['macaddress']
ipconfig['server_name'] = device['uuid']
ipconfig['nameserver'] = task['dns']
ipconfig['bootfile'] = task['cluster_type'] + '/pxelinux.0'
if task['cluster_type'] == cluster_type.CCP:
ipconfig['bootfile'] = device['server_level'] + '/pxelinux.0'
if not cls.config_dhcp_for_server(ipconfig):
#change task status to 'failed'
cls.update_task_status(task['task_uuid'], task_status.FAILED)
DEBUG and debug_print('Failed to config dhcp for server: %s' % (device['name'], ))
break
DEBUG and debug_print('Config in /etc/dhcpd.conf for %s\n' % (device['name'], ))
cls.update_server_status(device['macaddress'], server_status.PROGRESS)
#Boot server from PXE immediately
if not cls.power_control(device['idrac_ip'],
device['idrac_user'],
device['idrac_password'],
power_state.START):
#change task status to 'failed'
cls.update_task_status(task['task_uuid'], task_status.FAILED)
DEBUG and debug_print('Could no start server "' + device['name'] + '" remotely\n')
break
DEBUG and debug_print('Start server "' + device['name'] + '" remotely-----\n')
except Exception, e:
print_except_line()
print e
send_except_msg_to_server(e.__str__())