def call_compliance_checker():
"""
Description:
This function handles the compliance-checker-request. Set the client compliance-check status to True
once the client to the compliance checking. And call the function
`streaming_event_compliance.services.compliance_checker.compliance_checker` to do the compliance
with the given event from the request. Return the response of the compliance checking.
:request param:
`args: uuid`: :client-id
`json`: :event
:return: :`json` {
'body': other information (Mandatory)
'case_id': case id (Option)
'source_node': source node of the current connection (Option)
'sink_node': sink node of the current connection (Option)
'cause': alert cause of the current connection
{M: missing, T: lower than threshold}(Option)
'expect': expect connection (Option)
}, `flask_api.status`
"""
func_name = sys._getframe().f_code.co_name
uuid = request.args.get('uuid')
response = json.dumps({'body': 'Error, something wrong!'
}), status.HTTP_409_CONFLICT
try:
with lock:
if uuid not in gVars.clients_status:
check_client_stauts(uuid)
elif gVars.get_client_status(uuid):
dbtools.delete_alert(uuid)
dbtools.update_client_status(uuid, False)
gVars.clients_status[uuid] = False
if uuid not in gVars.clients_cc_status:
gVars.clients_cc_status[uuid] = True
setup.init_compliance_checking(uuid)
event = request.json
event = json.loads(event)
ServerLogging().log_info(
func_name, "server", "Compliance checking for " +
event['case_id'] + " " + event['activity'])
response = compliance_checker.compliance_checker(
uuid, event), status.HTTP_200_OK
except KeyError:
console.error('Something wrong in parameter getting!' +
traceback.format_exc())
ServerLogging().log_error(func_name, "server",
'Something wrong in parameter getting!')
except ThreadException:
console.error('Something wrong in threading!' + traceback.format_exc())
ServerLogging().log_error(func_name, "server",
'Something wrong in threading!')
except Exception:
console.error('Something wrong by running!' + traceback.format_exc())
ServerLogging().log_error(func_name, "server", 'Something wrong!')
finally:
return response
def build_automata_pro():
"""
Description:
Reads the training event logger from database.config.TRAINING_EVENT_LOG_PATH and build automata.
It generates the probability between SourceNode and SinkNode with different prefix size
and stores corresponding information into the database.
"""
thread_pool = ThreadPoolManager(1000)
func_name = sys._getframe().f_code.co_name
try:
trace_log = xes_importer.import_log(TRAINING_EVENT_LOG_PATH)
event_log = transform.transform_trace_log_to_event_log(trace_log)
event_log.sort()
ServerLogging().log_info(func_name, "server", "Training file processed and sorted")
ti = time.clock()
number_event_process = 0
console.secure("[ Preprocessing eventlog time ]", ti)
console.secure("[ Number of event ]", len(event_log))
except Exception:
ServerLogging().log_error(func_name, "server", "Training file cannot be processed")
raise ReadFileException(TRAINING_EVENT_LOG_PATH, traceback.format_exc())
for one_event in event_log:
event = {}
number_event_process += 1
try:
for item in one_event.keys():
if item == 'concept:name':
event['activity'] = one_event.get(item)
elif item == 'case:concept:name':
event['case_id'] = one_event.get(item)
except AttributeError:
raise EventException(event)
else:
try:
if event['case_id'] in C.dictionary_cases:
C.dictionary_cases.get(event['case_id']).append(event['activity'])
thread_pool.add_job(case_thread.run_build, *(event['case_id'], ))
else:
ServerLogging().log_info(func_name, "server", event['case_id'], event['activity'],
"Creating dictionary_case memorizer")
C.dictionary_cases[event['case_id']] = ['*' for _ in range(0, MAXIMUN_WINDOW_SIZE)]
C.dictionary_cases[event['case_id']].append(event['activity'])
lock = threading.RLock()
C.lock_List[event['case_id']] = lock
thread_pool.add_job(case_thread.run_build, *(event['case_id'], ))
except Exception:
console.error('build_auto_pro:' + traceback.format_exc())
ServerLogging().log_error(func_name, "server", "Exception raised while creating dictionary_case")
raise ThreadException(traceback.format_exc())
for item in C.dictionary_cases:
end_event = {'activity': '~!@#$%', 'case_id': item}
C.dictionary_cases.get(end_event['case_id']).append(end_event['activity'])
thread_pool.add_job(case_thread.run_build, *(end_event['case_id'],))
thread_pool.wait_completion()
def call_login():
"""
Description:
This function handles the login-request. Return refuse, if the the client has already logined.
:request param:
`args: uuid`: :client-id
:return: :`string` {OK, Refuse}, `flask_api.status`
"""
func_name = sys._getframe().f_code.co_name
uuid = request.args.get('uuid')
ServerLogging().log_info(func_name, "server", uuid + " is logging.")
if uuid in gVars.clients_cc_status:
ServerLogging().log_info(func_name, "server", uuid + " is refused.")
return 'Refuse', status.HTTP_200_OK
ServerLogging().log_info(func_name, "server",
uuid + " is logged successfully.")
return str(check_client_stauts(uuid)), status.HTTP_200_OK
def show_deviation_pdf(uuid):
"""
Description:
This function checks if deviation_pdf exists. If the pdf exists, then return 1.
If not, init alert-information from database and use function `visualization_automata`
to create pdf. Return 0, if there are no information about this client in database.
Otherwise, return 1.
:param uuid: :`string`: client-id
:return: :int: {0: not created, 1: created successfully}
"""
func_name = sys._getframe().f_code.co_name
path = CLEINT_DATA_PATH + uuid + '_' + AUTOMATA_FILE + FILE_TYPE
ServerLogging().log_info(func_name, uuid, "Check, whether PDF Path exists")
if not os.path.exists(path):
ServerLogging().log_info(func_name, uuid,
"PDF exists not, init alertlog if not exist")
alogs, status = setup.init_client_alert_automata(uuid)
if status == 1:
ServerLogging().log_info(
func_name, uuid, "PDF exists not, create again using alertlog")
visualization_automata(gVars.autos, alogs, uuid)
ServerLogging().log_info(func_name, uuid,
"PDF is created successfully")
else:
ServerLogging().log_info(func_name, uuid,
"Alertlog exists not, can't create PDF")
return status
else:
ServerLogging().log_info(func_name, uuid,
"PDF exists, don't need to create again")
return 1
def run_build(case_id):
func_name = sys._getframe().f_code.co_name
ServerLogging().log_info(func_name, str(threading.current_thread()))
try:
if C.lock_List.get(case_id).acquire():
ServerLogging().log_info(func_name, "server", case_id,
"Acquiring lock")
windows_memory = C.dictionary_cases.get(
case_id)[0:MAXIMUN_WINDOW_SIZE + 1]
C.dictionary_cases.get(case_id).pop(0)
C.lock_List.get(case_id).release()
ServerLogging().log_info(func_name, "server", case_id,
"Released lock")
executing_order4test(case_id, windows_memory)
calculate_connection_for_different_prefix_automata(windows_memory)
ServerLogging().log_info(func_name, "server", case_id,
"Calculating connections")
except Exception:
ServerLogging().log_error(func_name, "server", case_id,
"Error with Caselock")
raise ThreadException(traceback.format_exc())
def test_log_info_2(self):
func_name = sys._getframe().f_code.co_name
test_data_in_log_file = ''
compare_test_data_in_log_file = "INFO Username:Unknown test_log_info_2 'Testing logging info with 2 arguments'"
ServerLogging().log_info(func_name,
"Testing logging info with 2 arguments")
with open(SERVER_LOG_PATH, 'r') as f:
lines = f.read().splitlines()
test_data_in_log_file = lines[-1]
test_data_in_log_file = ' '.join(
test_data_in_log_file.split(' ')[3:])
# print(test_data_in_log_file)
# print(compare_test_data_in_log_file)
self.assertEqual(test_data_in_log_file, compare_test_data_in_log_file)
def build_automata():
func_name = sys._getframe().f_code.co_name
console.info('---------------------Start: Training automata starts!--------------------------------------')
ServerLogging().log_info(func_name, "Training automata starts!")
try:
process_ = Process(target=build_automata_pro())
process_.start()
process_.join()
except ThreadException as ec:
raise ec
else:
autos = gVars.autos
ServerLogging().log_info(func_name, "Setting probabilities...")
for ws in WINDOW_SIZE:
autos[ws].set_probability()
dbtools.insert_node_and_connection(autos)
ServerLogging().log_info(func_name, "Inserting automata to database")
console.info('---------------------End: Everything for training automata is Done!---------------------------')
ServerLogging().log_info(func_name, "Training automata ends!")
finally:
gVars.auto_status = 1
setup.clear_build_automata_memorizer()
ServerLogging().log_info(func_name, "Cleared automata memorizer")
def run(self):
"""
Description:
This function acquires lock on caseMemorizer and releases lock when processing of the
event is done. The event, which has done the compliance checking, will be removed.
This function provide 2 ways to do the compliance checking, respectively `DELETE_M_EVENT` and
`KEEP_ALL_EVENTS`.
"""
func_name = sys._getframe().f_code.co_name
client_cases = self.CCM.dictionary_cases.get(self.client_uuid)
client_locks = self.CCM.lock_List.get(self.client_uuid)
try:
if client_locks.get(self.event['case_id']).acquire():
ServerLogging().log_info(func_name, self.client_uuid, self.index, self.event['case_id'],
self.event['activity'], "Acquiring lock for event case: " + self.event['case_id'])
windows_memory = client_cases.get(self.event['case_id'])[0: MAXIMUN_WINDOW_SIZE + 1]
response = create_source_sink_node(windows_memory, self.client_uuid, self.event, self.index)
ServerLogging().log_info(func_name, self.client_uuid, self.index, self.event['case_id'],
self.event['activity'],
"Calculating response")
flag = True
for ws, res in response.items():
if ws == min(WINDOW_SIZE) and res.get('body') == 'M' and CHECKING_TYPE == 'DELETE_M_EVENT':
flag = False
client_cases.get(self.event['case_id']).pop(MAXIMUN_WINDOW_SIZE)
if flag:
client_cases.get(self.event['case_id']).pop(0)
client_locks.get(self.event['case_id']).release()
ServerLogging().log_info(func_name, self.client_uuid, self.index, self.event['case_id'],
self.event['activity'], "Released lock for event case: " + self.event['case_id'])
self._message.put(response)
self._status_queue.put(None)
except Exception as ec:
ServerLogging().log_info(func_name, self.client_uuid, self.index, self.event['case_id'],
self.event['activity'],
"in this thread's _status_queue ec is putted in: " + traceback.format_exc())
self._status_queue.put(ec)
def test_log_error_3(self):
func_name = sys._getframe().f_code.co_name
uuid = 'test_user'
test_data_in_log_file = ''
compare_test_data_in_log_file = "ERROR test_user test_log_error_3 'Testing logging error with 3 arguments'"
ServerLogging().log_error(func_name, uuid,
"Testing logging error with 3 arguments")
with open(SERVER_LOG_PATH, 'r') as f:
lines = f.read().splitlines()
test_data_in_log_file = lines[-1]
test_data_in_log_file = ' '.join(
test_data_in_log_file.split(' ')[3:])
# print(test_data_in_log_file)
# print(compare_test_data_in_log_file)
self.assertEqual(test_data_in_log_file, compare_test_data_in_log_file)
def call_show_deviation_pdf():
"""
Description:
This function handles the request `show-deviation-pdf`, call the function
`streaming_event_compliance.services.visualization.visualization_deviation_automata.show_deviation_pdf`
to create pdf and render pdf to the client.
:request param:
`args: uuid`: :client-id
:return: :`deviation-pdf-file`, `flask_api.status`
"""
func_name = sys._getframe().f_code.co_name
client_uuid = request.args.get('uuid')
pdf_status = visualization_deviation_automata.show_deviation_pdf(
client_uuid)
if pdf_status == 1:
try:
automata_pdf = open(
CLEINT_DATA_PATH + client_uuid + '_' + AUTOMATA_FILE +
FILE_TYPE, 'rb')
ServerLogging().log_info(func_name, "server",
'Create deviation_pdf for ' + client_uuid)
return send_file(
automata_pdf,
attachment_filename='file.pdf'), status.HTTP_200_OK
except Exception:
console.error(
'Error! Something wrong in call_show_deviation_pdf!' +
traceback.format_exc())
ServerLogging().log_error(
func_name, "server",
'Error! Something wrong in call_show_deviation_pdf!')
return '', status.HTTP_404_NOT_FOUND
else:
return '', status.HTTP_200_OK
def test_log_info_5(self):
func_name = sys._getframe().f_code.co_name
uuid = 'test_user'
dic = {
'case_id': 'test1',
'activity': 'testing_log_info_5',
}
test_data_in_log_file = ''
compare_test_data_in_log_file = "INFO test_user test_log_info_5 Case_id:test1 Activity:testing_log_info_5 " \
"'Testing logging info with 5 arguments'"
ServerLogging().log_info(func_name, uuid, dic['case_id'],
dic['activity'],
"Testing logging info with 5 "
"arguments")
with open(SERVER_LOG_PATH, 'r') as f:
lines = f.read().splitlines()
test_data_in_log_file = lines[-1]
test_data_in_log_file = ' '.join(
test_data_in_log_file.split(' ')[3:])
# print(test_data_in_log_file)
# print(compare_test_data_in_log_file)
self.assertEqual(test_data_in_log_file, compare_test_data_in_log_file)
def test_log_error_6(self):
func_name = sys._getframe().f_code.co_name
uuid = 'test_user'
dic = {
'case_id': 'test1',
'activity': 'testing_log_info_6',
}
thread_id = 1
test_data_in_log_file = ''
compare_test_data_in_log_file = "ERROR test_user test_log_error_6 Thread:1 Case_id:test1 Activity:testing_" \
"log_info_6 " \
"'Testing logging error with 6 arguments'"
ServerLogging().log_error(func_name, uuid, thread_id, dic['case_id'],
dic['activity'],
"Testing logging error with 6 "
"arguments")
with open(SERVER_LOG_PATH, 'r') as f:
lines = f.read().splitlines()
test_data_in_log_file = lines[-1]
test_data_in_log_file = ' '.join(
test_data_in_log_file.split(' ')[3:])
# print(test_data_in_log_file)
# print(compare_test_data_in_log_file)
self.assertEqual(test_data_in_log_file, compare_test_data_in_log_file)
def compliance_checker(client_uuid, event):
"""
Description:
This function will do compliance checking for each event from the streaming data provided from client_uuid.
It will first check the global variable 'autos', to check if tt's status is true,
if it's false, that means the automata has not built, return this information into user;
if it's true, then it will get initialed CCM(Case memory for 'client_uuid') and CTM(Thread memory for
'client_uuid'), and check firstly if this particular client_uuid is already in them,
if yes, add the current event into CCM(corresponding key 'client_uuid'),
create a thread for this event,
add this thread into CTM(corresponding key 'client_uuid')
start this thread
if no, add this client_uuid into CCM and CTM, do the same thing as above.
if the current event is endEvent, then it will join all threads for the streaming event provided from
client_uuid, and insert this client_uuid and all alerts into database, after doing this, it will generate
the visualization of deviation automata, return the url getting this pdf into user.
:param client_uuid: :`string` client-id
:param event: :`dict`={'case_id': `string`, 'activity': `string`}
:return: :`json` {
'body': other information (Mandatory)
'case_id': case id (Option)
'source_node': source node of the current connection (Option)
'sink_node': sink node of the current connection (Option)
'cause': alert cause of the current connection
{M: missing, T: lower than threshold}(Option)
'expect': expect connection (Option)
}
"""
func_name = sys._getframe().f_code.co_name
global threads_index
if gVars.auto_status:
client_cases = CCM.dictionary_cases.get(client_uuid)
client_thread = CTM.dictionary_threads.get(client_uuid)
client_locks = CCM.lock_List.get(client_uuid)
if event['case_id'] != 'NONE' and event['activity'] != 'END':
if event['case_id'] in client_cases:
client_cases.get(event['case_id']).append(event['activity'])
thread = case_thread_cc.CaseThreadForCC(
event, threads_index, client_uuid)
try:
thread.start()
if len(client_thread) > 1000:
for th in client_thread:
try:
th.join_with_exception()
except ThreadException as ec:
console.error('compliance_checker' +
traceback.format_exc())
ServerLogging().log_error(
func_name, client_uuid,
"Exception raised while joining thread.")
raise ThreadException(str(ec))
except Exception:
ServerLogging().log_error(
func_name, client_uuid,
"Exception raised while joining thread.")
console.error(traceback.format_exc())
else:
client_thread.clear()
client_thread.append(thread)
threads_index = threads_index + 1
return json.dumps(thread.get_message().get())
except Exception:
console.error('compliance_checker' +
traceback.format_exc())
ServerLogging().log_error(
func_name, client_uuid,
"Exception raised while starting thread")
raise ThreadException(traceback.format_exc())
else:
client_cases[event['case_id']] = [
'*' for _ in range(0, MAXIMUN_WINDOW_SIZE)
]
client_cases[event['case_id']].append(event['activity'])
thread = case_thread_cc.CaseThreadForCC(
event, threads_index, client_uuid)
lock = threading.Lock()
client_locks[event['case_id']] = lock
try:
thread.start()
if len(client_thread) > 1000:
for th in client_thread:
try:
th.join_with_exception()
except ThreadException as ec:
console.error('compliance_checker' +
traceback.format_exc())
ServerLogging().log_error(
func_name, client_uuid,
"Exception raised while joining thread." +
traceback.format_exc())
raise ThreadException(str(ec))
except Exception:
ServerLogging().log_error(
func_name, client_uuid,
"Exception raised while joining thread." +
traceback.format_exc())
console.error(traceback.format_exc())
else:
client_thread.clear()
client_thread.append(thread)
threads_index = threads_index + 1
return json.dumps(thread.get_message().get())
except Exception:
console.error('compliance_checker' +
traceback.format_exc())
ServerLogging().log_error(
func_name, client_uuid,
"Exception raised while starting thread." +
traceback.format_exc())
raise ThreadException(traceback.format_exc())
elif event['case_id'] == 'NONE' and event['activity'] == 'END':
for th in client_thread:
try:
th.join_with_exception()
except ThreadException as ec:
console.error('compliance_checker' +
traceback.format_exc())
ServerLogging().log_error(
func_name, client_uuid,
"ThreadException: Exception raised while "
"joining thread." + traceback.format_exc())
raise ThreadException(str(ec))
except Exception:
ServerLogging().log_error(
func_name, client_uuid,
"Exception raised while joining thread." +
traceback.format_exc())
console.error(traceback.format_exc())
alert_log = gVars.get_client_alert_logs(client_uuid)
dbtools.create_client(client_uuid)
dbtools.insert_alert_log(alert_log)
path = CLEINT_DATA_PATH + client_uuid + '_' + AUTOMATA_FILE + FILE_TYPE
if os.path.exists(path):
os.remove(path)
ServerLogging().log_info(func_name, client_uuid, "showing pdf")
visualization_deviation_automata.show_deviation_pdf(client_uuid)
ServerLogging().log_info(
func_name, client_uuid,
"Setting client status to true as compliance checking "
"is completed for that client")
dbtools.update_client_status(client_uuid, True)
gVars.clients_status[client_uuid] = True
ServerLogging().log_info(
func_name, client_uuid,
"Clearing memorizers for client- " + client_uuid)
setup.clear_cc_memorizer(client_uuid)
ServerLogging().log_info(func_name, client_uuid,
"Compliance checking is finished.")
return json.dumps({
'End': {
'body':
'The compliance checking is over, you can get the deviation pdf!'
}
})
else:
ServerLogging().log_error(func_name, client_uuid,
"Automata is not built!")
return json.dumps({
'End': {
'body':
'Sorry, automata has not built, please wait for a while!'
}
})
def check_alert(window_size, source_node, sink_node, client_uuid, event, thread_id):
"""
Description:
This function takes sink_node, source_node and checks if the automata of 'windowsize'
has any source_node and sink_node that matches tbe source_node, sink_node. If
there is a match then checks for its probability ,if probability below than threshold (Alert Typ: T) or
no match found (Alert Typ: M) then it inserts data to Alertlog object and returns an alert messsage.
:param window_size: :int, the length of the automata node
:param source_node: :`string` source node combination in fixed size, with separator `,`
:param sink_node: :`string` sink node combination in fixed size, with separator `,`
:param client_uuid: :`string` client-id
:param event: :dict`={'case_id': `string`, 'activity': `string`}
:param thread_id: :int, the id of the thread that is running the compliance check
:return: :int: {0: OK, 1: Alert, lower threshold, 2: Alert, no such connection}
"""
lock_list = CAL.c_alerts_lock_list.get(client_uuid)
func_name = sys._getframe().f_code.co_name
ServerLogging().log_info(func_name, client_uuid, thread_id, event['case_id'],
event['activity'], "the lock is in the lock_list: " + source_node + " to " + sink_node)
alert_log = gVars.get_client_alert_logs(client_uuid)[window_size]
auto = gVars.autos[window_size]
conn = automata.Connection(source_node, sink_node)
if auto.contains_connection(conn):
if auto.get_connection_probability(conn) >= THRESHOLD:
return 0
else:
try:
if lock_list.get((source_node, sink_node)).acquire():
ServerLogging().log_info(func_name, client_uuid, thread_id, event['case_id'],
event['activity'],
"Acquiring lock for alert: " + "(" + source_node + " to " + sink_node + ")")
alert_log.update_alert_record(
alertlog.AlertRecord (client_uuid, source_node, sink_node, 1, 'T'))
ServerLogging().log_info(func_name, client_uuid, thread_id, event['case_id'],
event['activity'], "Alert raised as probability of "
"connection between " +
source_node + " and " + sink_node + " is lesser than threshold")
except Exception as ec:
ServerLogging().log_info(func_name, client_uuid, thread_id, event['case_id'], event['activity'],
"Exception by check_alert between " + source_node
+ " and " + sink_node)
console.error(traceback.format_exc())
raise ec
else:
lock_list.get((source_node, sink_node)).release()
ServerLogging().log_info(func_name, client_uuid, thread_id, event['case_id'],
event['activity'],
"Release lock for alert: " + "(" + source_node + " to " + sink_node + ")")
return 1
elif source_node == 'NONE' and auto.contains_source_node(sink_node):
return 0
else:
try:
if lock_list.get((source_node, sink_node)).acquire():
ServerLogging().log_info(func_name, client_uuid, thread_id, event['case_id'],
event['activity'],
"Acquiring lock for alert: " + "(" + source_node + " to " + sink_node + ")")
alert_log.update_alert_record(alertlog.AlertRecord(client_uuid, source_node, sink_node, 1, 'M'))
ServerLogging().log_info(func_name, client_uuid, thread_id, event['case_id'], event['activity'],
"Alert raised as there should not be connection between " + source_node
+ " and " + sink_node)
except Exception:
ServerLogging().log_info(func_name, client_uuid, thread_id, event['case_id'], event['activity'],
"Exception by check_alert between " + source_node
+ " and " + sink_node)
console.error(traceback.format_exc())
else:
lock_list.get((source_node, sink_node)).release()
ServerLogging().log_info(func_name, client_uuid, thread_id, event['case_id'],
event['activity'],
"Release lock for alert: " + "(" + source_node + " to " + sink_node + ")")
return 2
def create_source_sink_node(windows_memory, client_uuid, event, thread_id):
"""
Description:
This function creates sink node and source node based on prefix sizes and call function to
check compliance with automata. Let the source node be NONE and sink node be the node,
if it is the start node.
:param windows_memory: :`list` a list of activities from the same case_id of current event(another event) with
size `maximum_window_size`, and the current event is at the last position of the windowsMemory
:param client_uuid: :`string` client-id
:param event: :`dict`={'case_id': `string`, 'activity': `string`}
:param thread_id: :int, the id of the thread that is running the compliance check
:return: `json` {
'body': other information (Mandatory)
'case_id': case id (Option)
'source_node': source node of the current connection, with separator `,` (Option)
'sink_node': sink node of the current connection, with separator `,` (Option)
'cause': alert cause of the current connection
{M: missing, T: lower than threshold}(Option)
'expect': expect connection (Option)
}
"""
response = {}
func_name = sys._getframe().f_code.co_name
try:
for ws in WINDOW_SIZE:
source_node = ','.join(windows_memory[MAXIMUN_WINDOW_SIZE - ws: MAXIMUN_WINDOW_SIZE])
sink_node = ','.join(windows_memory[MAXIMUN_WINDOW_SIZE - ws + 1: MAXIMUN_WINDOW_SIZE + 1])
if source_node.find('*') != -1 and sink_node.find('*') != -1:
response[str(ws)] = {'body': 'OK'}
break
elif source_node.find('*') != -1:
source_node = 'NONE'
lock_list = CAL.c_alerts_lock_list.get(client_uuid)
if (source_node, sink_node) not in lock_list:
lock = threading.Lock()
lock_list[source_node, sink_node] = lock
ServerLogging().log_info(func_name, client_uuid, thread_id, event['case_id'],
event['activity'],
"the lock has been created: " + source_node + " to " + sink_node)
matches = check_alert(ws, source_node, sink_node, client_uuid, event, thread_id)
if matches == 2:
response[str(ws)] = {
'case_id': event['case_id'],
'source_node': source_node,
'sink_node': sink_node,
'expect': gVars.autos[ws].get_sink_nodes(source_node),
'body': 'M'
}
if ALERT_TYPE == 'RETURN_ONE':
return response
elif matches == 1:
response[str(ws)] = {
'case_id': event['case_id'],
'source_node': source_node,
'sink_node': sink_node,
'cause': gVars.autos[ws].get_connection_probability(automata.Connection(source_node, sink_node)),
'expect': THRESHOLD,
'body': 'T'
}
else:
response[str(ws)] = {'body': 'OK'}
return response
except Exception as ec:
console.error(traceback.format_exc())
raise ec
from streaming_event_compliance import app, db
import time, traceback
from streaming_event_compliance.objects.logging.server_logging import ServerLogging
from streaming_event_compliance.objects.exceptions.exception import ThreadException, ReadFileException
from console_logging.console import Console
import sys
# resource.setrlimit(resource.RLIMIT_NOFILE, (2000, -1))
console = Console()
console.setVerbosity(5)
if __name__ == '__main__':
func_name = sys._getframe().f_code.co_name
try:
ServerLogging().log_info(func_name, "Created all db tables")
db.create_all()
except Exception as ec:
console.error('Error: Database connection!' + str(ec.__class__) + traceback.format_exc())
ServerLogging().log_error(func_name, "Database connection error!")
exit(1)
from streaming_event_compliance.objects.variable.globalvar import gVars
from streaming_event_compliance.services import setup
from streaming_event_compliance.services.build_automata import build_automata
from streaming_event_compliance.database import dbtools
dbtools.empty_tables()
setup.init_automata()
if gVars.auto_status == 0:
start = time.clock()
console.secure("Start time: ", start)
try:
def visualization_automata(autos, alogs, uuid):
"""
Description:
This function takes the automata and alertlog information to create a corresponding deviation pdf
<client_uuid>_automata.pdf and save in p_automata file.
:param autos: :`dict`={int: class `streaming_event_compliance.object.automata.Automata`}: {window size: the automata
used to do compliance checking}
:param alogs: :`dict`={int: class `streaming_event_compliance.object.automata.Alertlog`}: {window size: the alertlog
used to stored the alert-information}
:param uuid: :`string` client-id
"""
func_name = sys._getframe().f_code.co_name
ServerLogging().log_info(func_name, uuid, "Start to render pdf....")
viz = Digraph(comment='probability_automata', format='pdf', engine='dot')
viz.format = 'pdf'
viz.attr('node', fixedsize='true', width='0.7')
viz.attr(rankdir='LR')
for i in range(len(WINDOW_SIZE) - 1, -1, -1):
auto = autos[WINDOW_SIZE[i]]
alog = alogs[WINDOW_SIZE[i]]
with viz.subgraph(name='cluster' + str(WINDOW_SIZE[i])) as sub:
sub.attr(color='black',
label='Probability Graph With Prefix Size ' +
str(WINDOW_SIZE[i]))
for node in auto.get_nodes().keys():
sub.node(node, node, color='black')
for conn in auto.get_connections():
if conn.source_node != 'NONE' and conn.count > 0 and conn.probability > THRESHOLD:
sub.edge(conn.source_node,
conn.sink_node,
color='black',
label=str(round(conn.probability * 100, 2)) + '%',
penwidth=str(conn.probability *
2)) # penwidth = '0.5'
max_count = alog.get_max_count()
for record in alog.get_alert_log():
if record.source_node not in auto.get_nodes().keys(
) and record.source_node != 'NONE':
sub.node(record.source_node,
record.source_node,
fillcolor='red',
style='filled')
elif record.sink_node not in auto.get_nodes().keys():
sub.node(record.sink_node,
record.sink_node,
fillcolor='red',
style='filled')
if record.alert_cause == 'M' and record.source_node != 'NONE':
sub.edge(record.source_node,
record.sink_node,
color='red',
label='count = ' + str(record.alert_count),
penwidth=str(record.alert_count / max_count * 3))
elif record.alert_cause == 'T':
sub.edge(record.source_node,
record.sink_node,
color='green',
label='count = ' + str(record.alert_count),
penwidth=str(record.alert_count / max_count * 3))
ServerLogging().log_info(func_name, uuid, "Start to render pdf legend....")
with viz.subgraph(name='cluster0') as sub:
legend(sub)
ServerLogging().log_info(func_name, uuid, "Finish rendering pdf")
viz.render(filename=uuid + '_' + AUTOMATA_FILE,
directory=CLEINT_DATA_PATH,
view=False,
cleanup=True)