class CheckOpTimeoutsThread(threading.Thread):
"""Thread class for checking operations timeouts"""
def __init__(self, active_operations):
self._active_operations = active_operations
self._is_started = True
self._dbconn = DatabaseConnection()
# Initialize the thread
threading.Thread.__init__(self)
def _finish_operation(self, session_id, operation):
try:
self._dbconn.modify("UPDATE NM_OPERATION_INSTANCE SET status=%s, end_datetime=%s WHERE id=%s",
(ORS_TIMEOUTED, datetime.now(), session_id))
self._dbconn.modify("UPDATE NM_OPERATION_PROGRESS SET ret_code=%s, ret_message=%s \
WHERE instance_id=%s AND progress<>100 AND (ret_code=0 OR ret_code=NULL)",
(12, 'Operation is timeouted!', session_id))
self._active_operations.delete(session_id)
operation.callbackFunction(operation.operation_name, session_id, ORS_TIMEOUTED, operation.ret_params_map)
except Exception, err:
logger.error('CheckOpTimeoutsThread._finish_operation: %s'%err)
def writer_loop():
""" The main loop.
$HOST_FROM,$FACILITY,$PRIORITY,$LEVEL,$TAG,$PROGRAM,$ISODATE,$MSG
Please see http://www.balabit.com/sites/default/files/documents/syslog-ng-admin-guide_en.html/reference_macros.html
for a description of the macros used above.
"""
dbconn = DatabaseConnection()
NODES = {}
rows = dbconn.select("SELECT hostname, id FROM nm_node")
for row in rows:
NODES[row[0]] = row[1]
while True:
in_line = sys.stdin.readline()
if not in_line: #EOF occured
break
host, facility, priority, level, tag, program, isodate, msg = in_line.split('[-]')
host = host.strip()
node_id = NODES.get(host, None)
if node_id is None:
rows = dbconn.select("SELECT id FROM nm_node WHERE hostname=%s",(host,))
if rows:
NODES[host] = rows[0][0]
node_id = rows[0][0]
dbconn.modify("INSERT INTO logs (node_id, host, facility, priority, level, tag, program, log_timestamp, msg) \
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)", (node_id, host, facility.strip(), priority.strip(),
level.strip(), tag.strip(), program.strip(), isodate.strip(), msg.strip()))
def _update_pending_operations(self):
try:
dbconn = DatabaseConnection()
dbconn.modify("UPDATE NM_OPERATION_INSTANCE SET status=%s, end_datetime=%s WHERE status=%s",
(ORS_TIMEOUTED, datetime.now(), ORS_INPROGRESS,))
dbconn.modify("UPDATE NM_OPERATION_PROGRESS SET ret_code=%s, ret_message=%s \
WHERE progress<>100 AND (ret_code=0 OR ret_code=NULL)",
(12, 'Operation is timeouted!'))
del dbconn
except Exception, err:
logger.error('CheckOpTimeoutsThread._update_pending_operations: %s'%err)
def __init__(self, active_operations):
self._active_operations = active_operations
self._is_started = True
self._dbconn = DatabaseConnection()
# Initialize the thread
threading.Thread.__init__(self)
def test_02_simulate_call(self):
dbconn = DatabaseConnection()
caller = friBase.FriCaller()
packet = { 'uuid': '32542523esdf23r32r3fr',
'hostname': 'test_node_01',
'ip_address': '193.13.12.22',
'mac_address': '34:34:34:34:34:34',
'login': '******',
'password': '******',
'processor': 'Intel i7 3.4Ghz',
'memory': '4023'
}
code,msg = caller.call('127.0.0.1', packet, 1986)
self.assertEqual(code, 0, msg)
rows = dbconn.select("SELECT hostname FROM nm_node WHERE node_uuid='32542523esdf23r32r3fr'")
self.assertEqual(len(rows), 1)
self.assertEqual(rows[0][0], 'test_node_01')
#change params and call
packet['ip_address'] = '123.23.123.33'
code,msg = caller.call('127.0.0.1', packet, 1986)
self.assertEqual(code, 0, msg)
rows = dbconn.select("SELECT ip_address, login FROM nm_node WHERE node_uuid='32542523esdf23r32r3fr'")
self.assertEqual(len(rows), 1)
self.assertEqual(rows[0][0], '123.23.123.33')
self.assertEqual(rows[0][1], 'fabregas')
#ip_address is optional
del packet['ip_address']
code,msg = caller.call('127.0.0.1', packet, 1986)
self.assertEqual(code, 0, msg)
#ip_address is optional
del packet['login']
code,msg = caller.call('127.0.0.1', packet, 1986)
self.assertNotEqual(code, 0, msg)
def __init__(self):
self.__check_op_timeouts_thread = None
self._fri_client = None
self._dbconn = DatabaseConnection()
self._active_operations = OperationsMap()
#start thread for checking timeouted operations
self.__check_op_timeouts_thread = CheckOpTimeoutsThread(self._active_operations)
self.__check_op_timeouts_thread.setName('operationsEngine.CheckOpTimeoutsThread')
self.__check_op_timeouts_thread.start()
self._fri_client = WrappedFriClient(self._active_operations, self._dbconn)
self._fri_client.start(Config.oper_callers_count, Config.oper_results_threads)
class NodesMonitor(threading.Thread):
def __init__(self):
self.__monitor_timeout = Config.nodes_monitor_timeout
self.__threads = []
self.__nodes_queue = Queue()
self.__dbconn = DatabaseConnection()
self.__stoped = False
for i in xrange(Config.monitor_workers_count):
thread = MonitorWorkerThread(self.__nodes_queue, Config.monitor_wait_response_timeout)
thread.setName('MonitorWorkerThread#%i'%i)
thread.start()
self.__threads.append(thread)
threading.Thread.__init__(self, name='NodesMonitor')
def stop(self):
if self.__stoped:
return
self.__stoped = True
for i in self.__threads:
self.__nodes_queue.put(FINISH_FLAG)
self.__nodes_queue.join()
def run(self):
logger.info('NodesMonitor started!')
while not self.__stoped:
try:
t0_point = datetime.now()
rows = self.__dbconn.select("SELECT N.hostname, N.current_state FROM nm_node N, nm_cluster C \
WHERE N.cluster_id=C.id AND N.admin_status=%s AND C.status=%s",
(ANS_ACTIVE, CS_ACTIVE))
logger.debug('NodesMonitor: Selected %i nodes for checking state'%len(rows))
for row in rows:
self.__nodes_queue.put((row[0], row[1]))
self.__nodes_queue.join()
except Exception, err:
logger.error('NodesMonitor failed: %s' % err)
finally:
def test_02_simulate_node(self):
db = DatabaseConnection()
state = db.modify("UPDATE nm_node SET current_state=0 WHERE hostname='127.0.0.1'") # shutdown node :)
node = NodeSimulator()
def start_server(node):
node.start()
thread.start_new_thread(start_server, (node,))
try:
time.sleep(1.5)
state = db.select("SELECT current_state FROM nm_node WHERE hostname='127.0.0.1'")[0][0]
self.assertEqual(state, 1) # ON
node.stop()
time.sleep(1.5)
state = db.select("SELECT current_state FROM nm_node WHERE hostname='127.0.0.1'")[0][0]
self.assertEqual(state, 0) # OFF
finally:
node.stop()
print "STOPED"
def __init__(self):
self.__monitor_timeout = Config.nodes_monitor_timeout
self.__threads = []
self.__nodes_queue = Queue()
self.__dbconn = DatabaseConnection()
self.__stoped = False
for i in xrange(Config.monitor_workers_count):
thread = MonitorWorkerThread(self.__nodes_queue, Config.monitor_wait_response_timeout)
thread.setName('MonitorWorkerThread#%i'%i)
thread.start()
self.__threads.append(thread)
threading.Thread.__init__(self, name='NodesMonitor')
def __init__(self):
self.__dbconn = DatabaseConnection()
FriServer.__init__(self, hostname='0.0.0.0', port=LISTENER_PORT, workers_count=1)
class BootEventListener(FriServer):
def __init__(self):
self.__dbconn = DatabaseConnection()
FriServer.__init__(self, hostname='0.0.0.0', port=LISTENER_PORT, workers_count=1)
def onDataReceive( self, json_object ):
#get hostname
hostname = self.__get_element(json_object, 'hostname')
#get ip_address (this field is optional)
ip_address = json_object.get('ip_address', None)
#get login
login = self.__get_element(json_object, 'login')
#get password
password = self.__get_element(json_object, 'password')
#get processor
processor = self.__get_element(json_object, 'processor')
#get memory
memory = self.__get_element(json_object, 'memory')
#get mac_address
mac_address = self.__get_element(json_object, 'mac_address')
#get uuid
uuid = self.__get_element(json_object, 'uuid')
#process boot event
self.__process_event(uuid, hostname, login, password, mac_address, ip_address, processor, memory)
def __get_element(self, json_object, name):
element = json_object.get(name, None)
if element is None:
raise Exception('Element <%s> is not found in boot event packet!'%name)
if type(element) == str:
element = element.strip()
return element
def __process_event(self, uuid, hostname, login, password, mac_address, ip_address, processor, memory):
hw_info = 'Processor: %s\nMemory: %s'%(processor, memory)
rows = self.__dbconn.select("SELECT hostname FROM nm_node WHERE node_uuid=%s", (uuid,))
if not rows:
#this is new node, create it in database in NEW status
self.__dbconn.modify("INSERT INTO nm_node (node_uuid, hostname, login, password, mac_address, ip_address, admin_status, current_state, hw_info)\
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)",
(uuid, hostname, login, password, mac_address, ip_address, NAS_NEW, NCS_UP, hw_info))
else:
#we already has this node in database, update it
self.__dbconn.modify("UPDATE nm_node SET hostname=%s, login=%s, password=%s, mac_address=%s, ip_address=%s, hw_info=%s, current_state=%s\
WHERE node_uuid=%s", (hostname, login, password, mac_address, ip_address, hw_info, NCS_UP, uuid))
caller = self._get_operation_caller()
if caller:
logger.debug('Synchronize %s node parameters'%hostname)
ret_code, ret_message = caller.call_nodes_operation(ADMIN, [hostname], SYNC_OPER, {})
logger.debug('call SYNC operation result: [%s] %s'%(ret_code, ret_message))
def _get_operation_caller(self):
#try import nodes manager caller
try:
from blik.nodesManager.dbusClient import DBUSInterfaceClient
client = DBUSInterfaceClient()
return client
except ImportError, err:
logger.warning('Boot manager require nodes manager for automatic changing hostname.')
return None
from blik.utils.config import Config
Config.db_name = 'blik_cloud_db_test'
import unittest
from blik.nodesManager.plugins import base_operations
from blik.nodesManager.operationsPluginManager import CallObject
from blik.utils.databaseConnection import DatabaseConnection
dbconn = DatabaseConnection()
base_operations.SynchronizeOperation.dbConn = dbconn
base_operations.RebootOperation.dbConn = dbconn
class BaseClusterOperationsTestCase(unittest.TestCase):
def test01_sync_operation(self):
sync_oper = base_operations.SynchronizeOperation()
call_object = CallObject(CallObject.CLUSTER, 'TEST_CLUSTER')
try:
sync_oper.beforeCall('SYNC', call_object, {})
except Exception, err:
print ('EXPECTED exception: %s'%err)
else:
raise Exception('should be exception in this case')
call_object = CallObject(CallObject.NODES, ['node-01','node-02'])
try:
sync_oper.beforeCall('SYNC', call_object, {})
except:
class OperationsEngine:
def __init__(self):
self.__check_op_timeouts_thread = None
self._fri_client = None
self._dbconn = DatabaseConnection()
self._active_operations = OperationsMap()
#start thread for checking timeouted operations
self.__check_op_timeouts_thread = CheckOpTimeoutsThread(self._active_operations)
self.__check_op_timeouts_thread.setName('operationsEngine.CheckOpTimeoutsThread')
self.__check_op_timeouts_thread.start()
self._fri_client = WrappedFriClient(self._active_operations, self._dbconn)
self._fri_client.start(Config.oper_callers_count, Config.oper_results_threads)
def __del__(self):
self.stop()
def stop(self):
if self.__check_op_timeouts_thread:
self.__check_op_timeouts_thread.stop()
if self._fri_client:
self._fri_client.stop()
def __insert_operation_into_db(self, operation_id, user_name, nodes):
self._dbconn.start_transaction()
try:
rows = self._dbconn.modify_fetch("INSERT INTO NM_OPERATION_INSTANCE (operation_id, start_datetime, status, initiator_id) \
VALUES (%s,%s,%s,(SELECT id FROM NM_USER WHERE name=%s)) RETURNING id",
(operation_id, datetime.now(), ORS_INPROGRESS, user_name))
session_id = rows[0][0]
for node in nodes:
self._dbconn.modify("INSERT INTO NM_OPERATION_PROGRESS (node_id, instance_id, progress)\
VALUES (%s, %s, %s)", (node.id, session_id, 0))
return session_id
finally:
self._dbconn.end_transaction()
def __delete_session(self, session_id):
self._dbconn.start_transaction()
try:
self._dbconn.modify("DELETE FROM NM_OPERATION_PROGRESS WHERE instance_id=%s", (session_id,))
self._dbconn.modify("DELETE FROM NM_OPERATION_INSTANCE WHERE id=%s", (session_id,))
except Exception, err:
logger.error('OperationsEngine.__delete_session: %s'%err)
finally: