def put_execute_request(self, remote_control_code, remote_control_data):
"""Add a request to send exception data to the send queue."""
remote_control_data = json.dumps([
JsonUtil.encode_object(remote_control_code),
JsonUtil.encode_object(remote_control_data)
])
self.put_request(b'EEEE', remote_control_data.encode())
def load_json(file_name):
"""
Load persistence data from file.
@return None if file did not yet exist.
"""
persistence_data = None
try:
persistence_file_handle = open_persistence_file(
file_name, os.O_RDONLY | os.O_NOFOLLOW)
persistence_data = os.read(persistence_file_handle,
os.fstat(persistence_file_handle).st_size)
persistence_data = str(persistence_data, 'utf-8')
os.close(persistence_file_handle)
except OSError as openOsError:
if openOsError.errno != errno.ENOENT:
logging.getLogger(DEBUG_LOG_NAME).error(openOsError)
raise openOsError
return None
result = None
try:
result = JsonUtil.load_json(persistence_data)
except ValueError as valueError:
msg = 'Corrupted data in %s' % file_name, valueError
logging.getLogger(DEBUG_LOG_NAME).error(msg)
raise Exception(msg)
return result
def storeJson(fileName, objectData): """Store persistency data to file.""" persistenceData = JsonUtil.dumpAsJson(objectData) fd = createTemporaryPersistenceFile(fileName) os.write(fd, bytes(persistenceData, 'utf-8')) replacePersistenceFile(fileName, fd) os.close(fd)
def store_json(file_name, object_data):
"""Store persistency data to file."""
persistence_data = JsonUtil.dump_as_json(object_data)
fd = create_temporary_persistence_file(file_name)
os.write(fd, bytes(persistence_data, 'utf-8'))
replace_persistence_file(file_name, fd)
os.close(fd)
def store_json(file_name, object_data):
"""Store persistence data to file."""
persistence_data = JsonUtil.dump_as_json(object_data)
# Create a temporary file within persistence directory to write new persistence data to it.
# Thus the old data is not modified, any error creating or writing the file will not harm the old state.
fd, _ = tempfile.mkstemp(dir=SecureOSFunctions.tmp_base_dir_path)
os.write(fd, bytes(persistence_data, 'utf-8'))
create_missing_directories(file_name)
replace_persistence_file(file_name, fd)
os.close(fd)
def loadJson(fileName):
"""Load persistency data from file.
@return None if file did not yet exist."""
persistenceData = None
try:
persistenceFileHandle = openPersistenceFile(fileName, os.O_RDONLY|os.O_NOFOLLOW)
persistenceData = os.read(persistenceFileHandle, os.fstat(persistenceFileHandle).st_size)
persistenceData = str(persistenceData, 'utf-8')
os.close(persistenceFileHandle)
except OSError as openOsError:
if openOsError.errno != errno.ENOENT:
raise openOsError
return None
result = None
try:
result = JsonUtil.loadJson(persistenceData)
except ValueError as valueError:
raise Exception('Corrupted data in %s' % fileName, valueError)
return result
def do_process(self, analysis_context):
"""Process the next request, if any."""
request_data = self.do_get()
if request_data is None:
return
request_type = request_data[4:8]
if request_type == b'EEEE':
json_remote_control_response = None
exception_data = None
try:
json_request_data = (json.loads(request_data[8:].decode()))
json_request_data = JsonUtil.decode_object(json_request_data)
if (json_request_data is None) or (not isinstance(json_request_data, list)) or (len(json_request_data) != 2):
msg = 'Invalid request data'
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
raise Exception(msg)
if json_request_data[0] and isinstance(json_request_data[0], bytes):
json_request_data[0] = json_request_data[0].decode()
if json_request_data[1]:
if isinstance(json_request_data[1], list):
new_list = []
for item in json_request_data[1]:
if isinstance(item, bytes):
new_list.append(item.decode())
else:
new_list.append(item)
json_request_data[1] = new_list
else:
json_request_data[1] = json_request_data[1].decode()
methods = AMinerRemoteControlExecutionMethods()
import aminer.analysis
exec_locals = {
'analysis_context': analysis_context, 'remote_control_data': json_request_data[1],
'print_current_config': methods.print_current_config, 'print_config_property': methods.print_config_property,
'print_attribute_of_registered_analysis_component': methods.print_attribute_of_registered_analysis_component,
'change_config_property': methods.change_config_property,
'change_attribute_of_registered_analysis_component': methods.change_attribute_of_registered_analysis_component,
'rename_registered_analysis_component': methods.rename_registered_analysis_component,
'add_handler_to_atom_filter_and_register_analysis_component':
methods.add_handler_to_atom_filter_and_register_analysis_component,
'save_current_config': methods.save_current_config,
'allowlist_event_in_component': methods.allowlist_event_in_component,
'blocklist_event_in_component': methods.blocklist_event_in_component,
'dump_events_from_history': methods.dump_events_from_history,
'ignore_events_from_history': methods.ignore_events_from_history,
'list_events_from_history': methods.list_events_from_history,
'allowlist_events_from_history': methods.allowlist_events_from_history,
'persist_all': methods.persist_all,
'list_backups': methods.list_backups,
'create_backup': methods.create_backup,
'EnhancedNewMatchPathValueComboDetector': aminer.analysis.EnhancedNewMatchPathValueComboDetector,
'EventCorrelationDetector': aminer.analysis.EventCorrelationDetector,
'EventTypeDetector': aminer.analysis.EventTypeDetector,
'HistogramAnalysis': aminer.analysis.HistogramAnalysis,
'PathDependentHistogramAnalysis': aminer.analysis.PathDependentHistogramAnalysis,
'MatchFilter': aminer.analysis.MatchFilter,
'MatchValueAverageChangeDetector': aminer.analysis.MatchValueAverageChangeDetector,
'MatchValueStreamWriter': aminer.analysis.MatchValueStreamWriter,
'MissingMatchPathValueDetector': aminer.analysis.MissingMatchPathValueDetector,
'NewMatchIdValueComboDetector': aminer.analysis.NewMatchIdValueComboDetector,
'NewMatchPathDetector': aminer.analysis.NewMatchPathDetector,
'NewMatchPathValueComboDetector': aminer.analysis.NewMatchPathValueComboDetector,
'NewMatchPathValueDetector': aminer.analysis.NewMatchPathValueDetector,
'ParserCount': aminer.analysis.ParserCount,
'Rules': aminer.analysis.Rules,
'TimeCorrelationDetector': aminer.analysis.TimeCorrelationDetector,
'TimeCorrelationViolationDetector': aminer.analysis.TimeCorrelationViolationDetector,
'TimestampCorrectionFilters': aminer.analysis.TimestampCorrectionFilters,
'TimestampsUnsortedDetector': aminer.analysis.TimestampsUnsortedDetector,
'VariableTypeDetector': aminer.analysis.VariableTypeDetector,
'AllowlistViolationDetector': aminer.analysis.AllowlistViolationDetector
}
logging.getLogger(AMinerConfig.REMOTE_CONTROL_LOG_NAME).log(15, json_request_data[0])
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).debug('Remote control: %s', json_request_data[0])
# skipcq: PYL-W0603
global suspended_flag
if json_request_data[0] in ('suspend_aminer()', 'suspend_aminer', 'suspend'):
suspended_flag = True
msg = methods.REMOTE_CONTROL_RESPONSE + 'OK. aminer is suspended now.'
json_remote_control_response = json.dumps(msg)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).info(msg)
elif json_request_data[0] in ('activate_aminer()', 'activate_aminer', 'activate'):
suspended_flag = False
msg = methods.REMOTE_CONTROL_RESPONSE + 'OK. aminer is activated now.'
json_remote_control_response = json.dumps(msg)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).info(msg)
else:
# skipcq: PYL-W0122
exec(json_request_data[0], {'__builtins__': None}, exec_locals)
json_remote_control_response = json.dumps(exec_locals.get('remoteControlResponse'))
if methods.REMOTE_CONTROL_RESPONSE == '':
methods.REMOTE_CONTROL_RESPONSE = None
if exec_locals.get('remoteControlResponse') is None:
json_remote_control_response = json.dumps(methods.REMOTE_CONTROL_RESPONSE)
else:
json_remote_control_response = json.dumps(
exec_locals.get('remoteControlResponse') + methods.REMOTE_CONTROL_RESPONSE)
# skipcq: FLK-E722
except:
exception_data = traceback.format_exc()
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).debug('Remote control exception data: %s', str(exception_data))
# This is little dirty but avoids having to pass over remoteControlResponse dumping again.
if json_remote_control_response is None:
json_remote_control_response = 'null'
json_response = '[%s, %s]' % (json.dumps(exception_data), json_remote_control_response)
if len(json_response) + 8 > self.max_control_packet_size:
# Damn: the response would be larger than packet size. Fake a secondary exception and return part of the json string
# included. Binary search of size could be more efficient, knowing the maximal size increase a string could have in json.
max_include_size = len(json_response)
min_include_size = 0
min_include_response_data = None
while True:
test_size = (max_include_size + min_include_size) >> 1
if test_size == min_include_size:
break
emergency_response_data = json.dumps(
['Exception: Response too large\nPartial response data: %s...' % json_response[:test_size], None])
if len(emergency_response_data) + 8 > self.max_control_packet_size:
max_include_size = test_size - 1
else:
min_include_size = test_size
min_include_response_data = emergency_response_data
json_response = min_include_response_data
# Now size is OK, send the data
json_response = json_response.encode()
self.output_buffer += struct.pack("!I", len(json_response) + 8) + b'RRRR' + json_response
else:
msg = 'Invalid request type %s' % repr(request_type)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
raise Exception(msg)
def do_process(self, analysis_context):
"""Process the next request, if any."""
request_data = self.do_get()
if request_data is None:
return
request_type = request_data[4:8]
if request_type == b'EEEE':
json_remote_control_response = None
exception_data = None
try:
json_request_data = (json.loads(request_data[8:].decode()))
json_request_data = JsonUtil.decode_object(json_request_data)
if (json_request_data is None) or (not isinstance(
json_request_data,
list)) or (len(json_request_data) != 2):
raise Exception('Invalid request data')
json_request_data[0] = shlex.quote(json_request_data[0])
json_request_data[1] = shlex.quote(json_request_data[1])
methods = AMinerRemoteControlExecutionMethods()
import aminer.analysis
exec_locals = {
'analysis_context':
analysis_context,
'remote_control_data':
json_request_data[1],
'print_current_config':
methods.print_current_config,
'print_config_property':
methods.print_config_property,
'print_attribute_of_registered_analysis_component':
methods.print_attribute_of_registered_analysis_component,
'change_config_property':
methods.change_config_property,
'change_attribute_of_registered_analysis_component':
methods.change_attribute_of_registered_analysis_component,
'rename_registered_analysis_component':
methods.rename_registered_analysis_component,
'add_handler_to_atom_filter_and_register_analysis_component':
methods.
add_handler_to_atom_filter_and_register_analysis_component,
'save_current_config':
methods.save_current_config,
'whitelist_event_in_component':
methods.whitelist_event_in_component,
'dump_events_from_history':
methods.dump_events_from_history,
'ignore_events_from_history':
methods.ignore_events_from_history,
'list_events_from_history':
methods.list_events_from_history,
'whitelist_events_from_history':
methods.whitelist_events_from_history,
'HistogramAnalysis':
aminer.analysis.HistogramAnalysis,
'MatchValueAverageChangeDetector':
aminer.analysis.MatchValueAverageChangeDetector,
'MatchValueStreamWriter':
aminer.analysis.MatchValueStreamWriter,
'MissingMatchPathValueDetector':
aminer.analysis.MissingMatchPathValueDetector,
'NewMatchPathDetector':
aminer.analysis.NewMatchPathDetector,
'NewMatchPathValueComboDetector':
aminer.analysis.NewMatchPathValueComboDetector,
'Rules':
aminer.analysis.Rules,
'TimeCorrelationDetector':
aminer.analysis.TimeCorrelationDetector,
'TimeCorrelationViolationDetector':
aminer.analysis.TimeCorrelationViolationDetector,
'TimestampCorrectionFilters':
aminer.analysis.TimestampCorrectionFilters,
'TimestampsUnsortedDetector':
aminer.analysis.TimestampsUnsortedDetector,
'WhitelistViolationDetector':
aminer.analysis.WhitelistViolationDetector
}
# write this to the log file!
logging.basicConfig(
filename=AMinerConfig.LOG_FILE,
level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s',
datefmt='%d.%m.%Y %H:%M:%S')
logging.addLevelName(15, "REMOTECONTROL")
logging.log(15, json_request_data[0].decode())
# skipcq: PYL-W0122
exec(json_request_data[0], {'__builtins__': None}, exec_locals)
json_remote_control_response = json.dumps(
exec_locals.get('remoteControlResponse', None))
if methods.REMOTE_CONTROL_RESPONSE == '':
methods.REMOTE_CONTROL_RESPONSE = None
if exec_locals.get('remoteControlResponse', None) is None:
json_remote_control_response = json.dumps(
methods.REMOTE_CONTROL_RESPONSE)
else:
json_remote_control_response = json.dumps(
exec_locals.get('remoteControlResponse', None) +
methods.REMOTE_CONTROL_RESPONSE)
# skipcq: FLK-E722
except:
exception_data = traceback.format_exc()
# This is little dirty but avoids having to pass over remoteControlResponse dumping again.
if json_remote_control_response is None:
json_remote_control_response = 'null'
json_response = '[%s, %s]' % (json.dumps(exception_data),
json_remote_control_response)
if len(json_response) + 8 > self.max_control_packet_size:
# Damn: the response would be larger than packet size. Fake a secondary exception and return part of the json string
# included. Binary search of size could be more efficient, knowing the maximal size increase a string could have in json.
max_include_size = len(json_response)
min_include_size = 0
min_include_response_data = None
while True:
test_size = (max_include_size + min_include_size) >> 1
if test_size == min_include_size:
break
emergency_response_data = json.dumps([
'Exception: Response too large\nPartial response data: %s...'
% json_response[:test_size], None
])
if len(emergency_response_data
) + 8 > self.max_control_packet_size:
max_include_size = test_size - 1
else:
min_include_size = test_size
min_include_response_data = emergency_response_data
json_response = min_include_response_data
# Now size is OK, send the data
json_response = json_response.encode()
self.output_buffer += struct.pack(
"!I",
len(json_response) + 8) + b'RRRR' + json_response
else:
raise Exception('Invalid request type %s' % repr(request_type))
def putExecuteRequest(self, remoteControlCode, remoteControlData):
"""Add a request to send exception data to the send queue."""
remoteControlData = json.dumps([JsonUtil.encodeObject(remoteControlCode), \
JsonUtil.encodeObject(remoteControlData)])
self.putRequest(b'EEEE', remoteControlData.encode())
